# HG changeset patch # User ymh # Date 1359987151 -3600 # Node ID 5f3e270580afb1862e81356a6764050deedd6959 # Parent 8dd5b0f370fc1b8488f838364831617c47fb7e77 First model or egonomy diff -r 8dd5b0f370fc -r 5f3e270580af .hgignore --- a/.hgignore Thu Jan 31 18:01:44 2013 +0100 +++ b/.hgignore Mon Feb 04 15:12:31 2013 +0100 @@ -1,13 +1,13 @@ syntax: regexp ^virtualenv/web/project-boot\.py$ ^virtualenv/web/env/ -syntax: regexp ^src/egonomy/config\.py$ -syntax: regexp ^web/static/site$ -syntax: regexp ^\.pydevproject$ ^\.project$ ^\.settings/org\.eclipse\.core\.resources\.prefs$ ^\.settings/org\.eclipse\.core\.runtime\.prefs$ +\.pyc$ +\.DS_Store$ ^web/static/media/cache$ + diff -r 8dd5b0f370fc -r 5f3e270580af .project --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.project Mon Feb 04 15:12:31 2013 +0100 @@ -0,0 +1,17 @@ + + + egonomy + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff -r 8dd5b0f370fc -r 5f3e270580af .pydevproject --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.pydevproject Mon Feb 04 15:12:31 2013 +0100 @@ -0,0 +1,8 @@ + + +python_egonomy +python 2.7 + +/egonomy/src + + diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/config.py.tmpl --- a/src/egonomy/config.py.tmpl Thu Jan 31 18:01:44 2013 +0100 +++ b/src/egonomy/config.py.tmpl Mon Feb 04 15:12:31 2013 +0100 @@ -33,13 +33,30 @@ SITE_ID = 1 BASE_STATIC_ROOT = os.path.abspath(BASE_DIR + "../../web/static/").rstrip("/")+"/" +BASE_STATIC_URL = WEB_URL + BASE_URL + 'static/' + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/var/www/example.com/media/" MEDIA_ROOT = BASE_STATIC_ROOT + "media/" -MEDIA_URL = BASE_URL + "static/media/" -# Absolute path to the directory that static files (js, css, swf...) -# DO NOT forget to do command line ./manage.py collectstatic to gather static media into the web/static folder + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://example.com/media/", "http://media.example.com/" +MEDIA_URL = BASE_STATIC_URL + "media/" + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/var/www/example.com/static/" STATIC_ROOT = BASE_STATIC_ROOT + "site/" +# URL prefix for static files. +# Example: "http://example.com/static/", "http://static.example.com/" +STATIC_URL = BASE_STATIC_URL + "site/" + + # Local path to rmn pictures +# TODO : remove RMN_PICT_ROOT = STATIC_ROOT + 'rmn/' diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/management/__init__.py diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/management/commands/__init__.py diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/management/commands/importRmn.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/egonomy/management/commands/importRmn.py Mon Feb 04 15:12:31 2013 +0100 @@ -0,0 +1,286 @@ +# -*- coding: utf-8 -*- +''' +Created on Jan 31, 2013 + +@author: ymh +''' + +from ..utils import show_progress +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings +from django.db import models, transaction +from egonomy.models import Image, ImageInfo, ImageMetadata +from optparse import make_option +import mimetypes +import csv +import decimal +import os.path +import sys +import shutil +import PIL.Image +import PIL.ExifTags +import json +import datetime + + +class Command(BaseCommand): + ''' + Import rmn csv files + ''' + + args = 'csv_file csv_file ...' + help = 'Import rmn csv files' + + option_list = BaseCommand.option_list + ( + make_option('--check-id', + action= 'store_true', + dest= 'check_id', + default= False, + help= 'check an image id before trying to insert it, may be a lot slower' + ), + make_option('-p', '--image-path', + dest= 'image_path', + default= None, + help= 'path to the root o image folder' + ), + make_option('-n', '--max-lines', + dest= 'max_lines', + type='int', + default= sys.maxint, + help= 'max number of line to process, -1 process all file' + ), + make_option('-b', '--batch-size', + dest= 'batch_size', + type='int', + default= 5000, + help= 'number of object to import in bulk operations' + ), + make_option('-e', '--encoding', + dest= 'encoding', + default= 'latin1', + help= 'csv files encoding' + ), + make_option('--skip', + dest= 'skip', + type='int', + default= 0, + help= 'number of entry to skip' + ), + make_option('--stop', + dest= 'cont', + action= 'store_false', + default= True, + help= 'stop on error' + ), + make_option('-l', '--log', + dest= 'log', + default= 'log.txt', + help= 'log file' + ), + ) + + def __safe_get(self, dict_arg, key, conv = lambda x: x, default= None): + val = dict_arg.get(key, default) + return conv(val) if val else default + + def __safe_decode(self, s): + if not isinstance(s, basestring): + return s + try: + return s.decode('utf8') + except: + try: + return s.decode('latin1') + except: + return s.decode('utf8','replace') + + def handle(self, *args, **options): + + #getting path to copy images + imageInfoModel = models.get_model('egonomy', 'ImageInfo') + upload_to = imageInfoModel._meta.get_field_by_name('image_file')[0].upload_to + media_root = getattr(settings, 'MEDIA_ROOT', None) + + if not media_root: + raise CommandError('The setting MEDIA_ROT must be set') + + image_root = os.path.abspath(os.path.join(media_root, upload_to)) + + print("Caching filenames...") + #map filenames + image_filemanes_map = {} + + root_img_dir = options.get('image_path', None) + + if not root_img_dir: + raise CommandError("No image path. the -p or --image-path options is compulsory") + + root_img_dir = os.path.abspath(root_img_dir) + + for f_triple in os.walk(root_img_dir, topdown = True): + for f in f_triple[2]: + full_path = os.path.join(f_triple[0],f) + rel_path = full_path[len(root_img_dir)+1:] + image_filemanes_map[os.path.splitext(f)[0]] = (full_path, rel_path) + #get the number of lines to process + + print("caching done. %d file found " % len(image_filemanes_map)) + + max_lines = options.get('max_lines', sys.maxint) + csv_files_dialect = {} + skip = options.get('skip', 0) + # calculating the number of lines to process + print("calculating number of line to process") + total = 0 + for csv_file_path in args: + with open(csv_file_path,'rb') as csv_file: + dialect = csv.Sniffer().sniff(csv_file.read(1024)) + dialect.doublequote = True + csv_files_dialect[csv_file_path] = dialect + csv_file.seek(0) + for _ in csv.DictReader(csv_file, dialect=dialect): + total += 1 + if total > max_lines: + break + + nb_lines = min(max_lines, total) + batch_size = options.get('batch_size', 5000) + + print("There is %d lines to process, starting processing now." % nb_lines) + counter = 0 + writer = None + img_objs = [] + img_objs_md = [] + img_objs_info = [] + check_id = options.get('check_id', False) + encoding = options.get('encoding', 'latin1') + log_path = options.get('log', "log.txt") + cont_on_error = options.get('cont', True) + + transaction.enter_transaction_management() + transaction.managed() + try: + for csv_file_path in args: + with open(csv_file_path,'rb') as csv_file: + dialect = csv_files_dialect.get(csv_file_path,None) + if not dialect: + dialect = csv.Sniffer().sniff(csv_file.read(1024)) + dialect.doublequote = True + csv_file.seek(0) + + dictreader = csv.DictReader(csv_file, dialect=dialect) + for row in dictreader: + try: + counter += 1 + if counter <= skip: + continue + if counter > nb_lines: + break + urow = dict([(k, v.decode(encoding, 'replace') if v else v) for k,v in row.items()]) + writer = show_progress(counter, nb_lines, u"%s - %s - %d/%d" % (urow['CLICHE'], urow['TITRE'], counter%batch_size, batch_size), 80, writer) + + if check_id and ImageMetadata.objects.filter(cliche=urow['CLICHE']).count(): + raise CommandError("Duplicate entry line %d of file %s" % (dictreader.line_num, csv_file_path)) + + img_id = urow['CLICHE'] + img_md_obj = ImageMetadata( + id = img_id, + cliche = img_id, + inventaire = self.__safe_get(urow, 'INVENTAIRE'), + titre = self.__safe_get(urow, 'TITRE'), + description = self.__safe_get(urow, 'DESCRIPTION'), + date = self.__safe_get(urow, 'DATE', int, None), + longueur = self.__safe_get(urow, 'LONGUEUR', decimal.Decimal, None), + hauteur = self.__safe_get(urow, 'HAUTEUR', decimal.Decimal, None), + profondeur = self.__safe_get(urow, 'PROFONDEUR', decimal.Decimal, None), + diametre = self.__safe_get(urow, 'DIAMETRE', decimal.Decimal, None), + photographe = self.__safe_get(urow, 'PHOTOGRAPE'), + auteur = self.__safe_get(urow, 'AUTEUR'), + droits = self.__safe_get(urow, 'DROITS'), + mentions = self.__safe_get(urow, 'MENTIONS'), + periode = self.__safe_get(urow, 'PERIODE'), + technique = self.__safe_get(urow, 'TECHNIQUE'), + site = self.__safe_get(urow, 'SITE'), + lieu = self.__safe_get(urow, 'LIEU'), + localisation = self.__safe_get(urow, 'LOCALISATION'), + mots_cles = self.__safe_get(urow, 'MOT_CLES') + ) + + img_info_obj = None + finfo = image_filemanes_map.get(img_id, None) + if finfo is not None: + # copy file + img_fullpath, img_relpath = finfo + dest_path = os.path.join(image_root, img_relpath) + d = os.path.dirname(dest_path) + if not os.path.exists(d): + os.makedirs(d) + shutil.copy(img_fullpath, dest_path) + mimestr = mimetypes.guess_type(dest_path, False)[0] + img = PIL.Image.open(dest_path) + width, height = img.size + raw_exif = img._getexif() + exif = dict((PIL.ExifTags.TAGS.get(k,k), self.__safe_decode(v)) for (k,v) in raw_exif.items()) if raw_exif else None + #create image info object + img_info_obj = ImageInfo( + id = img_id, + width = width, + height = height, + mimetype = mimestr, + exif = json.dumps(exif) if exif else None + ) + img_info_obj.image_file.name = os.path.join(upload_to, img_relpath) + + + img_obj = Image( + id = img_id, + metadata = img_md_obj, + info = img_info_obj + ) + + img_objs_md.append(img_md_obj) + if img_info_obj is not None: + img_objs_info.append(img_info_obj) + img_objs.append(img_obj) + + except Exception as e: + error_msg = "%s - Error treating line %d, file %s local %d : id %s - title : %s : %s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),counter, csv_file_path, dictreader.line_num, row['ID'] if (row and 'ID' in row and row['ID']) else 'n/a', row['TITRE'] if (row and 'TITRE' in row and row['TITRE']) else 'n/a', repr(e) ) + with open(log_path, 'a') as log_file: + log_file.write(error_msg) + if not cont_on_error: + raise + + + if not (counter%batch_size): + ImageMetadata.objects.bulk_create(img_objs_md) + ImageInfo.objects.bulk_create(img_objs_info) + Image.objects.bulk_create(img_objs) + img_objs = [] + img_objs_info = [] + img_objs_md = [] + transaction.commit() + + + if counter > nb_lines: + break + + if img_objs: + ImageMetadata.objects.bulk_create(img_objs_md) + ImageInfo.objects.bulk_create(img_objs_info) + Image.objects.bulk_create(img_objs) + transaction.commit() + + + no_img_req = Image.objects.filter(info=None) + + if no_img_req.count() > 0: + print "WARNING : the following images have no image files :" + for img_obj in no_img_req: + print "%s : %s" % (img_obj.metadata.id, img_obj.metadata.titre) + transaction.commit() + except: + transaction.rollback() + raise + finally: + transaction.leave_transaction_management() + \ No newline at end of file diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/management/utils.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/egonomy/management/utils.py Mon Feb 04 15:12:31 2013 +0100 @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +''' +Created on Feb 1, 2012 + +@author: ymh +''' +import sys +import codecs #@UnresolvedImport +import math + +def show_progress(current_line, total_line, label, width, writer=None): + + if writer is None: + writer = sys.stdout + if sys.stdout.encoding is not None: + writer = codecs.getwriter(sys.stdout.encoding)(sys.stdout) + + percent = (float(current_line) / float(total_line)) * 100.0 + + marks = math.floor(width * (percent / 100.0)) #@UndefinedVariable + spaces = math.floor(width - marks) #@UndefinedVariable + + loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']' + + s = u"%s %3d%% %*d/%d - %*s\r" % (loader, percent, len(str(total_line)), current_line, total_line, width, label[:width]) + + writer.write(s) #takes the header into account + if percent >= 100: + writer.write("\n") + writer.flush() + + return writer diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/migrations/__init__.py diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/models.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/egonomy/models.py Mon Feb 04 15:12:31 2013 +0100 @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +''' +Created on Jan 28, 2013 + +@author: ymh +''' + +from django.db import models +from django.contrib.auth.models import User + +class ImageMetadata(models.Model): + + id = models.CharField(null=False, blank=False, max_length=15, primary_key=True) + date_inserted = models.DateTimeField(null=False, blank=False, auto_now_add=True) + date_modified = models.DateTimeField(null=False, blank=False, auto_now=True) + + cliche = models.CharField(null=False, blank=False, max_length=15) + inventaire = models.TextField(null=True, blank=True) + titre = models.TextField(null=True, blank=True) + description = models.TextField(null=True, blank=True) + date = models.IntegerField(null=True, blank=True) + longueur = models.DecimalField(null=True, blank=True, max_digits=20, decimal_places=15) + hauteur = models.DecimalField(null=True, blank=True, max_digits=20, decimal_places=15) + profondeur = models.DecimalField(null=True, blank=True, max_digits=20, decimal_places=15) + diametre = models.DecimalField(null=True, blank=True, max_digits=20, decimal_places=15) + photographe = models.TextField(null=True, blank=True) + auteur = models.TextField(null=True, blank=True) + droits = models.TextField(null=True, blank=True) + mentions = models.TextField(null=True, blank=True) + periode = models.TextField(null=True, blank=True) + technique = models.TextField(null=True, blank=True) + site = models.TextField(null=True, blank=True) + lieu = models.TextField(null=True, blank=True) + localisation = models.TextField(null=True, blank=True) + mots_cles = models.TextField(null=True, blank=True) + + titre_pertimm = models.TextField(null=True, blank=True) + description_pertimm = models.TextField(null=True, blank=True) + thesaurus_pertimm = models.TextField(null=True, blank=True) + + +class ImageInfo(models.Model): + + id = models.CharField(null=False, blank=False, max_length=15, primary_key=True) + image_file = models.ImageField(width_field = "width", height_field= "height", upload_to="images/", max_length=2048) + width = models.IntegerField(null=False, blank=False) + height = models.IntegerField(null=False, blank=False) + mimetype = models.CharField(null=True, blank=True, max_length=1024) + exif = models.TextField(null=True, blank=True) #json value for exif data if available + + +class Image(models.Model): + + id = models.CharField(null=False, blank=False, max_length=15, primary_key=True) + metadata = models.ForeignKey(ImageMetadata) + info = models.ForeignKey(ImageInfo, null=True, blank=True) + + +class Fragment(models.Model): + + image = models.ForeignKey(Image, blank=False, null=False) + date_created = models.DateTimeField(blank=False, null=False, auto_now_add=True) + date_saved = models.DateTimeField(blank=False, null=False, auto_now=True) + coordinates = models.TextField(blank=False, null=False) + author = models.ForeignKey(User, blank=False, null=False) + title = models.CharField(max_length=2048, blank=True, null=True) + description = models.TextField(blank=True, null=True) + tags = models.TextField(blank=True, null=True) + \ No newline at end of file diff -r 8dd5b0f370fc -r 5f3e270580af src/egonomy/settings.py --- a/src/egonomy/settings.py Thu Jan 31 18:01:44 2013 +0100 +++ b/src/egonomy/settings.py Mon Feb 04 15:12:31 2013 +0100 @@ -118,6 +118,8 @@ 'django.contrib.staticfiles', 'django.contrib.admin', 'django.contrib.admindocs', + 'django_extensions', + 'south', 'sorl.thumbnail', 'egonomy', ) @@ -157,4 +159,4 @@ } } -from .config import * +from .config import * #@UnusedWildImport diff -r 8dd5b0f370fc -r 5f3e270580af virtualenv/res/lib/lib_create_env.py --- a/virtualenv/res/lib/lib_create_env.py Thu Jan 31 18:01:44 2013 +0100 +++ b/virtualenv/res/lib/lib_create_env.py Mon Feb 04 15:12:31 2013 +0100 @@ -54,7 +54,7 @@ else: URLS.update({ 'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.6.tar.gz', 'local':"psycopg2-2.4.6.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'PIL': {'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local':"Imaging-1.1.7.tar.gz", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}}, + 'PIL': {'setup': 'pil', 'url': 'https://github.com/python-imaging/Pillow/archive/1.7.8.tar.gz', 'local':"Pillow-1.7.8.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, }) diff -r 8dd5b0f370fc -r 5f3e270580af virtualenv/res/src/Imaging-1.1.7.tar.gz Binary file virtualenv/res/src/Imaging-1.1.7.tar.gz has changed diff -r 8dd5b0f370fc -r 5f3e270580af virtualenv/res/src/Pillow-1.7.8.tar.gz Binary file virtualenv/res/src/Pillow-1.7.8.tar.gz has changed