# HG changeset patch # User Riwad Salim # Date 1528801909 -7200 # Node ID cfd40849d24c9d202c67ff09aa66cae7b04f51d9 # Parent 9cc447bd52801caa9501ff22d6279737d0b1698e Turning iconolab-mcc into App to add specific import commands diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/__init__.py --- a/src/iconolab_mcc/__init__.py Thu Jun 07 17:12:48 2018 +0200 +++ b/src/iconolab_mcc/__init__.py Tue Jun 12 13:11:49 2018 +0200 @@ -45,4 +45,4 @@ __version__ = get_version(VERSION) -default_app_config = 'iconolab.apps.IconolabApp' +default_app_config = 'iconolab_mcc.apps.IconolabMccApp' diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/apps.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab_mcc/apps.py Tue Jun 12 13:11:49 2018 +0200 @@ -0,0 +1,8 @@ +from django.apps import AppConfig + +class IconolabMccApp(AppConfig): + name = 'iconolab_mcc' + verbose_name = 'Iconolab-mcc' + + # def ready(self): + # import iconolab.templatetags.iconolab_tags diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/fixtures/demo_data.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab_mcc/fixtures/demo_data.json Tue Jun 12 13:11:49 2018 +0200 @@ -0,0 +1,244 @@ +[ + { + "model": "auth.User", + "pk": 1, + "fields": { + "username": "contributeur1", + "password": "pbkdf2_sha256$24000$4t81p9toYpfc$ufZAWaUcF51dpNqHscABIoW7UyoXxYDCRlKFI87vQJM=" + } + },{ + "model": "auth.User", + "pk": 2, + "fields": { + "username": "contributeur2", + "password": "pbkdf2_sha256$24000$b9CPP5bLU4FM$1IYIBS5y2CIEFV9jHdTdZjvj59hzx+yH5Akc5LsHo+g=" + } + },{ + "model": "iconolab.Collection", + "pk": 1, + "fields": { + "name": "ingres", + "verbose_name": "Musée d'Ingres" + } + },{ + "model": "iconolab.Item", + "pk": 1, + "fields": { + "collection": 1, + "item_guid": "80858614-7988-494e-87f5-3a53d83ad889" + } + },{ + "model": "iconolab.ItemMetadata", + "pk": 1, + "fields": { + "item": 1, + "technics": "dessin", + "joconde_ref": "1" + } + },{ + "model": "iconolab.Image", + "pk": 1, + "fields": { + "image_guid" : "26aec320-dcfe-4cbc-b912-6a6c13e8916e", + "name" : "MIC.8.10.jpg", + "media" : "uploads/MIC.8.10.jpg", + "item": 1, + "height": 2478, + "width": 3744, + "created": "2016-03-23 14:13:44.913765+01" + } + },{ + "model": "iconolab.ImageStats", + "pk": 1, + "fields": { + "image": 1, + "annotations_count": 0, + "submitted_revisions_count": 0, + "comments_count": 0, + "folders_inclusion_count": 0, + "tag_count": 0 + } + },{ + "model": "iconolab.Item", + "pk": 2, + "fields": { + "collection": 1, + "item_guid": "b597b937-0d4b-4f48-814d-6de3e308ac76" + } + },{ + "model": "iconolab.ItemMetadata", + "pk": 2, + "fields": { + "item": 2, + "technics": "dessin", + "joconde_ref": "2" + } + },{ + "model": "iconolab.Image", + "pk": 2, + "fields": { + "image_guid" : "de734457-b600-4bed-bb60-a03d906b8e6e", + "name" : "MIC.1.7.jpg", + "media" : "uploads/MIC.1.7.jpg", + "item": 2, + "height": 4672, + "width": 6171, + "created": "2016-03-23 14:13:44.913765+01" + } + },{ + "model": "iconolab.ImageStats", + "pk": 2, + "fields": { + "image": 2, + "annotations_count": 0, + "submitted_revisions_count": 0, + "comments_count": 0, + "folders_inclusion_count": 0, + "tag_count": 0 + } + },{ + "model": "iconolab.Item", + "pk": 3, + "fields": { + "collection": 1, + "item_guid": "de301942-e60e-4e46-83c2-967bf3838f47" + } + },{ + "model": "iconolab.ItemMetadata", + "pk": 3, + "fields": { + "item": 3, + "technics": "dessin", + "designation": "dessin format paysage", + "joconde_ref": "12345" + } + },{ + "model": "iconolab.Image", + "pk": 3, + "fields": { + "image_guid" : "1a00cdd1-514e-4fd9-87ae-d9872b73143b", + "name" : "MIC.1.18.jpg", + "media" : "uploads/MIC.1.18.jpg", + "item": 3, + "height": 6208, + "width": 4704, + "created": "2016-03-23 14:13:44.913765+01" + } + },{ + "model": "iconolab.ImageStats", + "pk": 3, + "fields": { + "image": 3, + "annotations_count": 0, + "submitted_revisions_count": 0, + "comments_count": 0, + "folders_inclusion_count": 0, + "tag_count": 0 + } + },{ + "model": "iconolab.Collection", + "pk": 2, + "fields": { + "name": "stdie", + "verbose_name": "Musée Saint-Dié" + } + },{ + "model": "iconolab.Item", + "pk": 4, + "fields": { + "collection": 2, + "item_guid": "46ea6b45-caa4-47d5-bd0b-f1f22410b68e" + } + },{ + "model": "iconolab.ItemMetadata", + "pk": 4, + "fields": { + "item": 4, + "designation": "Médaille Athènes", + "joconde_ref": "1234567" + } + },{ + "model": "iconolab.Image", + "pk": 4, + "fields": { + "image_guid" : "7384a6af-e87b-42b5-b24b-30cadb8adebc", + "name" : "athene_medal1.jpg", + "media" : "uploads/athene_medal1.jpg", + "item": 4, + "height": 704, + "width": 704, + "created": "2016-03-23 14:13:44.913765+01" + } + },{ + "model": "iconolab.ImageStats", + "pk": 4, + "fields": { + "image": 4, + "annotations_count": 0, + "submitted_revisions_count": 0, + "comments_count": 0, + "folders_inclusion_count": 0, + "tag_count": 0 + } + },{ + "model": "iconolab.Image", + "pk": 5, + "fields": { + "image_guid" : "0ad0ae3e-06cf-4452-9d62-042471ddfdc3", + "name" : "athene_medal2.jpg", + "media" : "uploads/athene_medal2.jpg", + "item": 4, + "height": 704, + "width": 704, + "created": "2016-03-23 14:13:44.913765+01" + } + },{ + "model": "iconolab.ImageStats", + "pk": 5, + "fields": { + "image": 5, + "annotations_count": 0, + "submitted_revisions_count": 0, + "comments_count": 0, + "folders_inclusion_count": 0, + "tag_count": 0 + } + },{ + "model": "iconolab.MetaCategory", + "pk": 1, + "fields": { + "collection": 1, + "label": "Appel à contribution", + "triggers_notifications": 1 + } + },{ + "model": "iconolab.MetaCategory", + "pk": 2, + "fields": { + "collection": 1, + "label": "Appel à expertise", + "triggers_notifications": 3 + } + },{ + "model": "iconolab.MetaCategory", + "pk": 3, + "fields": { + "collection": 1, + "label": "Référence" + } + },{ + "model": "iconolab.MetaCategory", + "pk": 4, + "fields": { + "collection": 1, + "label": "Accord" + } + },{ + "model": "iconolab.MetaCategory", + "pk": 5, + "fields": { + "collection": 1, + "label": "Désaccord" + } + } +] \ No newline at end of file diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/management/__init__.py diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/management/commands/__init__.py diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/management/commands/importimages.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab_mcc/management/commands/importimages.py Tue Jun 12 13:11:49 2018 +0200 @@ -0,0 +1,447 @@ +# -*- coding: UTF-8 -*- +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from django.conf import settings +from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder +from PIL import Image as ImagePIL +from sorl.thumbnail import get_thumbnail +import os, csv, pprint, re, json, shutil, logging + +if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''): + logger = logging.getLogger(settings.IMPORT_LOGGER_NAME) +else: + logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = 'import images from a directory into the media folder and creates item and image objects' + + def add_arguments(self, parser): + parser.add_argument('csv_path') + parser.add_argument( + '--jpeg-quality', + dest='jpeg_quality', + default=settings.IMG_JPG_DEFAULT_QUALITY, + help='Jpeg default quality' + + ) + parser.add_argument( + '--encoding', + dest='encoding', + default='utf-8', + help='CSV file encoding' + + ) + parser.add_argument( + '--collection-json', + dest='collection_json', + default=False, + help='creates a new collection from a json file, must be an object with fields : '+ \ + '"name" (identifier), '+ \ + '"verbose_name" (proper title name), '+ \ + '"description" (description on homepage, html is supported), '+ \ + '"image" (image on homepages, must be "uploads/"), '+ \ + '"height" and "width" (height and width of the image)', + ) + parser.add_argument( + '--collection-id', + dest='collection_id', + default=False, + help='insert extracted data into the specified collection instead of trying to load a collection fixture', + ) + parser.add_argument( + '--metacategories-json', + dest='metacategories_json', + default=False, + help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)', + ) + parser.add_argument( + '--delimiter', + dest='csv_delimiter', + default=';', + help='csv file delimiter' + ) + parser.add_argument( + '--no-jpg-conversion', + dest='no-jpg-conversion', + default=False, + help='use this option if you only want the image copied and not converted' + ) + parser.add_argument( + '--img-filename-identifier', + dest='img_filename_identifier', + default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER, + help='codename of the csv field we\'ll try to match to find the related image to a given object' + ) + parser.add_argument( + '--filename-regexp-prefix', + dest='filename_regexp_prefix', + default=r'.*', + help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*' + ) + parser.add_argument( + '--filename-regexp-suffix', + dest='filename_regexp_suffix', + default=r'[\.\-_].*', + help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*' + ) + parser.add_argument( + '--folders', + dest='import_folders', + default=False, + action='store_const', + const=True, + help='option to create folders' + ) + parser.add_argument( + '--folders-regexp', + dest='folders_regexp', + default=False, + help='regexp used to extract the folder name/number' + ) + parser.add_argument( + '--folders-metadata', + dest='folders_metadata', + default='REF', + help='metadata from which to extract the folder name/number' + ) + def handle(self, *args, **options): + """ + Step-by-step for import: + + 1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files + 1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic: + * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas + to which we associate a list of column header that will identified as that metadata + * The cleaned_row_data list will associate the identifier with the actual value for its related column + 2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated + to each row for the filtered data. + 3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary + 4) For each item: + We create the object in the database + * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT + We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image + + Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command. + """ + try: + print('# Logging with logger '+logger.name) + logger.debug('# Initializing command with args: %r', options) + # Check we have a collection to store data into: + source_dir = os.path.dirname(os.path.realpath(options.get('csv_path'))) + print('# Checking collection args') + if options.get('collection_json'): + print('## Finding collection json data in '+source_dir) + collection_json_path = os.path.join(source_dir, options.get('collection_json')) + if not os.path.isfile(collection_json_path): + print('### No '+options.get('collection_json')+'.json file was found in the source directory') + raise ValueError('!!! Json file '+collection_json_path+' was not found !!!') + try: + with open(collection_json_path) as json_fixture_file: + collection_data = json.loads(json_fixture_file.read()) + for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']: + if not key in collection_data.keys(): + print('!!! Json file '+collection_json_path+' has no '+key+' field !!!') + raise ValueError() + if not collection_data.get('name', ''): + print('!!! Collection data key "name" is empty') + raise ValueError() + if Collection.objects.filter(name=collection_data.get('name')).exists(): + print('!!! A Collection with the provided name already exists!') + raise ValueError() + if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)): + print('!!! Collection data has an image but no height and width') + raise ValueError() + except ValueError as e: + raise ValueError('!!! JSON Data is invalid. !!!') + elif options.get('collection_id'): + print('## Finding collection with id '+options.get('collection_id')) + try: + collection = Collection.objects.get(pk=options.get('collection_id')) + except Collection.DoesNotExist: + raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!') + else: + raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!') + + if options.get('metacategories_json'): + print('## Finding metacategories fixture json data in '+source_dir) + metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json')) + if not os.path.isfile(metacategories_json_path): + print('### No '+options.get('metacategories_json')+'.json file was found in the source directory') + raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!') + with open(metacategories_json_path) as metacategories_json_file: + metacategories_data = json.loads(metacategories_json_file.read()) + for metacategory in metacategories_data: + if metacategory.get('label', None) is None: + raise ValueError('!!! Metacategory without label !!!') + + if options['import_folders'] and not options['folders_regexp']: + raise ValueError('!!! No regexp specified to extract folder name !!!') + + # We read the csv + delimiter = options.get('csv_delimiter') + if delimiter == '#9': + delimiter = chr(9) + if delimiter == '#29': + delimiter = chr(29) + if delimiter == '#30': + delimiter = chr(30) + if delimiter == '#31': + delimiter = chr(31) + csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter) + print('# Extracting data from csv file and storing it in standardized format') + # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT + cleaned_csv_data=[] + duplicate_rows=[] + for row in csvreader: + cleaned_row_data = {} + for key in settings.IMPORT_FIELDS_DICT.keys(): + cleaned_row_data[key] = '' + for row_key in row.keys(): + if row_key in settings.IMPORT_FIELDS_DICT[key]: + if key == 'REF': + ref_number, _, _ = row[row_key].partition(';') + cleaned_row_data[key] = ref_number.rstrip() + else: + cleaned_row_data[key] = row[row_key] + break + if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]: + print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line") + duplicate_rows.append(cleaned_row_data) + else: + cleaned_csv_data.append(cleaned_row_data) + # Listing image files in csv directory + image_list = [ + f for f in os.listdir(source_dir) + if os.path.isfile(os.path.join(source_dir, f)) + and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png')) + ] # Maybe check if image another way + filtered_csv_data = [] + no_image_rows = [] + no_data_images = [] + assigned_images = [] + # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image + for item in cleaned_csv_data: + item['SRC_IMG_FILES'] = [] + has_image = False + for image in image_list: + img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix') + if re.match(img_name_pattern, image): + item['SRC_IMG_FILES'].append(image) + assigned_images.append(image) + has_image = True + if has_image: + filtered_csv_data.append(item) + else: + # We keep track of the entries that don't have any corresponding image + no_image_rows.append(item) + # We keep track of the images that don't have any corresponding entry + for image in image_list: + if image not in assigned_images: + no_data_images.append(image) + + print('## found ' + str(len(filtered_csv_data))+' items with at least one image') + print('# Importing data into Iconolab') + if options.get('collection_json'): + print('## Loading collection json') + collection = Collection.objects.create( + name = collection_data.get('name'), + verbose_name = collection_data.get('verbose_name', ''), + description = collection_data.get('description', ''), + image = collection_data.get('image', ''), + height = collection_data.get('height', 0), + width = collection_data.get('width', 0), + ) + if collection.image: + collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image)) + if not os.path.isfile(collection_image_path): + print('### Moving collection image') + _ , collection_image_name = os.path.split(collection_image_path) + try: + col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name)) + print('##### Generating or copying jpeg for '+collection_image_name) + col_im.thumbnail(col_im.size) + col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY)) + except Exception as e: + print(e) + if options.get('metacategories_json'): + for metacategory in metacategories_data: + MetaCategory.objects.create( + collection = collection, + label = metacategory.get('label'), + triggers_notifications = metacategory.get('triggers_notifications', 0) + ) + print('## Converting image and moving it to static dir, creating Image and Item objects') + target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads') + print('### Images will be stored in '+target_dir) + for item in filtered_csv_data: + print('#### Computing metadatas for item '+item['REF']+' (natural key)') + if not item['REF']: + print('#### No Natural key, skipping') + continue + item_authors = item['AUTR'] + item_school = item['ECOLE'] + item_designation = '' + if item.get('TITR', ''): + item_designation = item['TITR'] + elif item.get('DENO', ''): + item_designation = item['DENO'] + elif item.get('APPL', ''): + item_designation = item['APPL'] + item_datation = '' + if item.get('PERI', ''): + item_datation = item['PERI'] + elif item.get('MILL', ''): + item_datation = item['MILL'] + elif item.get('EPOQ', ''): + item_datation = item['EPOQ'] + item_technics = item['TECH'] + item_field = item['DOM'] + item_measurements = item['DIMS'] + item_create_or_usage_location = item['LIEUX'] + item_discovery_context = item['DECV'] + item_conservation_location = item['LOCA'] + item_photo_credits = item['PHOT'] + item_inventory_number = item['INV'] + item_joconde_ref = item['REF'] + if ItemMetadata.objects.filter(item__collection = collection, natural_key = item_joconde_ref).exists(): + print('#### An item with '+item['REF']+' for natural key, already exists in database in the import collection') + + if options['import_folders']: + + # Extract folder name from natural key + m = re.search(options['folders_regexp'], item[options['folders_metadata']]) + folder_id = m.group(1) + + if not Folder.objects.filter(original_id=folder_id).exists(): + print('#### Creating folder "'+folder_id+'"') + folder = Folder.objects.create( + collection = collection, + name = 'Dossier '+folder_id, + original_id = folder_id + ) + else: + print('#### Folder "'+folder_id+'" already exists') + folder = Folder.objects.get(original_id=folder_id) + + item_metadata = ItemMetadata.objects.get(item__collection = collection, natural_key = item_joconde_ref) + item = item_metadata.item + + item.folders.add(folder) + + else: + print('#### Creating item '+item['REF']+' (natural key) in database') + item_object = Item.objects.create( + collection = collection + ) + + new_metadata = { + "authors" : item_authors, + "school" : item_school, + "designation" : item_designation, + "field" : item_field, + "datation" : item_datation, + "technics" : item_technics, + "measurements" : item_measurements, + "create_or_usage_location" : item_create_or_usage_location, + "discovery_context" : item_discovery_context, + "conservation_location" : item_conservation_location, + "photo_credits" : item_photo_credits, + "inventory_number" : item_inventory_number, + "joconde_ref" : item_joconde_ref + } + ItemMetadata.objects.create( + item = item_object, + metadata = json.dumps(new_metadata), + natural_key = item_joconde_ref + ) + + print('#### Computing item image(s)') + for image in item['SRC_IMG_FILES']: + (image_name, ext) = os.path.splitext(image) + if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS: + print('##### Copying file '+str(image)+' without converting') + image_path = os.path.join(target_dir, image) + new_image_name = image + shutil.copy(os.path.join(source_dir, image), target_dir) + try: + im = ImagePIL.open(os.path.join(target_dir, image)) + im_width, im_height = im.size + except Exception as e: + print(e) + continue + else: + image_path = os.path.join(target_dir, image_name) + '.jpg' + new_image_name = image_name+'.jpg' + if os.path.isfile(image_path): + print('##### A jpeg file already exists in target dir for '+ image) + try: + im = ImagePIL.open(image_path) + im_width, im_height = im.size + except Exception as e: + print(e) + continue + else: + jpeg_img_path = image_path + try: + im = ImagePIL.open(os.path.join(source_dir, image)) + print('##### Generating or copying jpeg for '+image) + im.thumbnail(im.size) + im.save(jpeg_img_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY)) + im_width, im_height = im.size + except Exception as e: + print(e) + continue + new_image = Image.objects.create( + item = item_object, + media = 'uploads/'+new_image_name, + name = new_image_name, + height = im_height, + width = im_width + ) + ImageStats.objects.create( + image = new_image + ) + print('### Generating thumbnails for item '+item['REF']) + for image in item_object.images.all(): + for size in settings.PREGENERATE_THUMBNAILS_SIZES: + print('#### Thumbnail for size '+size) + get_thumbnail(image.media, size, crop=False) + + print('# All done!') + + logger.debug('# Recap for import command: ') + print('# Images without data: ') + logger.debug('## Checking images left without data') + collection_image_file = os.path.split(str(collection.image))[1] + if no_data_images and collection_image_file in no_data_images: + no_data_images.remove(collection_image_file) + + if no_data_images: + for image in no_data_images: + logger.debug('### %r', image) + print('## '+image) + else: + print('## Each image has one corresponding row!') + logger.debug('### Each image has one corresponding row!') + print('# CSV Items without image') + logger.debug('## Checking csv rows left without image') + if no_image_rows: + for item in no_image_rows: + logger.debug('### %r', item['REF']) + print('## Natural key: '+item['REF']) + else: + print('## Each row found at least one corresponding image!') + logger.debug('### Each row found at least one corresponding image!') + print('# Duplicate rows in csv') + logger.debug('## Checking duplicate rows in csv') + if duplicate_rows: + for item in no_image_rows: + logger.debug('### %r: %r', options.get('img_filename_identifier'), item[options.get('img_filename_identifier')]) + print('## '+options.get('img_filename_identifier')+': '+item[options.get('img_filename_identifier')]) + else: + print('## Each row found at least one corresponding image!') + logger.debug('### Each row found at least one corresponding image!') + except FileNotFoundError: + print('!!! File '+options.get('csv_path')+' does not exist. !!!') + except ValueError as e: + print(str(e)) diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/management/commands/updatecollection.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab_mcc/management/commands/updatecollection.py Tue Jun 12 13:11:49 2018 +0200 @@ -0,0 +1,144 @@ +# -*- coding: UTF-8 -*- +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from django.conf import settings +from iconolab.models import Collection, Item, ItemMetadata, MetaCategory +from sorl.thumbnail import get_thumbnail +import os, csv, pprint, re, json, shutil + +class Command(BaseCommand): + help = "import images from a directory into the media folder and creates item and image objects" + + def add_arguments(self, parser): + parser.add_argument("csv_path") + parser.add_argument( + '--encoding', + dest='encoding', + default='utf-8', + help='CSV file encoding' + + ) + parser.add_argument( + '--collection-id', + dest='collection_id', + default=False, + help='insert extracted data into the specified collection instead of trying to load a collection fixture', + ) + parser.add_argument( + '--delimiter', + dest='csv_delimiter', + default=';', + help='csv file delimiter' + ) + + def handle(self, *args, **options): + try: + # Check we have a collection to store data into: + source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) + print("# Checking collection args") + if options.get("collection_id"): + print("## Finding collection with id "+options.get("collection_id")) + try: + collection = Collection.objects.get(pk=options.get("collection_id")) + except Collection.DoesNotExist: + raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!") + else: + raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!") + + # We read the csv + delimiter = options.get('csv_delimiter') + if delimiter == "#9": + delimiter = chr(9) + if delimiter == "#29": + delimiter = chr(29) + if delimiter == "#30": + delimiter = chr(30) + if delimiter == "#31": + delimiter = chr(31) + csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter) + print("# Extracting data from csv file and storing it in standardized format") + # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT + cleaned_csv_data=[] + for row in csvreader: + cleaned_row_data = {} + for key in settings.IMPORT_FIELDS_DICT.keys(): + cleaned_row_data[key] = "" + for row_key in row.keys(): + if row_key in settings.IMPORT_FIELDS_DICT[key]: + # Handling the multiple natural keys exports nonsense + if key == "REF": + natural_key_number, _, _ = row[row_key].partition(";") + cleaned_row_data[key] = natural_key_number.rstrip() + else: + cleaned_row_data[key] = row[row_key] + break + cleaned_csv_data.append(cleaned_row_data) + # Listing image files in csv directory + filtered_csv_data = [] + # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key) + for item in cleaned_csv_data: + if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists(): + filtered_csv_data.append(item) + print("## found " + str(len(filtered_csv_data))+" items to update") + print("# Updating data from Iconolab") + for item in filtered_csv_data: + item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first() + if not item["REF"]: + print("#### No Natural key, skipping") + continue + item_authors = item["AUTR"] + item_school = item["ECOLE"] + item_designation = "" + if item.get("TITR", ""): + item_designation = item["TITR"] + elif item.get("DENO", ""): + item_designation = item["DENO"] + elif item.get("APPL", ""): + item_designation = item["APPL"] + item_datation = "" + if item.get("PERI", ""): + item_datation = item["PERI"] + elif item.get("MILL", ""): + item_datation = item["MILL"] + elif item.get("EPOQ", ""): + item_datation = item["EPOQ"] + item_technics = item["TECH"] + item_field = item["DOM"] + item_measurements = item["DIMS"] + item_create_or_usage_location = item["LIEUX"] + item_discovery_context = item["DECV"] + item_conservation_location = item["LOCA"] + item_photo_credits = item["PHOT"] + item_inventory_number = item["INV"] + item_joconde_ref = item["REF"] + # Updating metadatas + + new_metadata = { + "authors" : item_authors, + "school" : item_school, + "designation" : item_designation, + "field" : item_field, + "datation" : item_datation, + "technics" : item_technics, + "measurements" : item_measurements, + "create_or_usage_location" : item_create_or_usage_location, + "discovery_context" : item_discovery_context, + "conservation_location" : item_conservation_location, + "photo_credits" : item_photo_credits, + "inventory_number" : item_inventory_number, + "joconde_ref" : item_joconde_ref + } + + item_metadatas.metadata = json.dumps(new_metadata) + item_metadatas.natural_key = item_joconde_ref + item_metadatas.save() + print('### Generating thumbnails for item '+item['REF']) + for image in item_metadatas.item.images.all(): + for size in settings.PREGENERATE_THUMBNAILS_SIZES: + print('#### Thumbnail for size '+size) + get_thumbnail(image.media, size, crop=False) + print("# All done!") + except FileNotFoundError: + print("!!! File "+options.get("csv_path")+" does not exist. !!!") + except ValueError as e: + print(str(e)) diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/settings/__init__.py --- a/src/iconolab_mcc/settings/__init__.py Thu Jun 07 17:12:48 2018 +0200 +++ b/src/iconolab_mcc/settings/__init__.py Tue Jun 12 13:11:49 2018 +0200 @@ -44,6 +44,7 @@ # Application definition INSTALLED_APPS = [ + 'iconolab_mcc', 'iconolab.apps.IconolabApp', 'django.contrib.admin', 'django.contrib.auth', diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/templates/iconolab/home.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab_mcc/templates/iconolab/home.html Tue Jun 12 13:11:49 2018 +0200 @@ -0,0 +1,139 @@ +{% extends 'iconolab_base.html' %} + +{% load i18n %} +{% load staticfiles %} +{% load thumbnail %} +{% load iconolab_tags %} +{% load humanize %} + +{% block content %} +
+ +
+
+

Les collections

+
+
+
+ + {% for collection in collections_primary %} +
+
+ {% thumbnail collection.image "450x250" crop="center" as im %} +
+ + + +
+

{{collection.verbose_name}}

+
+
+ {{ collection.completed_percent }}% +
+
+ + Contribuer + +
+

{{collection.description|safe}}

+ {% endthumbnail %} +
+
+ {% endfor %} +
+
+
+

Les dernières annotations

+ {% for annotation in latest_annotations %} +
+
+ +
+
+
+ {% thumbnail annotation.image.media "100x100" crop=False as im %} + + + + + + + + + {% endthumbnail %} +
+
+
+ +

+ {{ annotation.current_revision.title }} + {{ annotation.current_revision.created|naturaltime }} +

+

+ {% for contributor in annotation.stats.contributors.all %} + {{ contributor.username }} + {% endfor %} +

+

{{ annotation.current_revision.description }}

+

+ {% for tagging_info in annotation.current_revision.tagginginfo_set.all %} +  {{ tagging_info.tag.label }} + {% endfor %} +

+
+
+
+
+ {% endfor %} +
+
+

Les meilleurs contributeurs

+ + +

Les mots-clé les plus pertinents

+ + +
+
+ {% for collection in collections_secondary %} +
+
+
+ {% thumbnail collection.image "350x350" crop=False as im %} + + {% endthumbnail %} +
+
+

{{ collection.verbose_name }}

+

+ {{collection.description | safe}} +

+ Contribuer +
+
+
+ {% endfor %} +
+{% endblock %} diff -r 9cc447bd5280 -r cfd40849d24c src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt --- a/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt Thu Jun 07 17:12:48 2018 +0200 +++ b/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt Tue Jun 12 13:11:49 2018 +0200 @@ -1,4 +1,4 @@ -{{ object.metadatas.metadata_obj.meschool }} +{{ object.metadatas.metadata_obj.school }} {{ object.metadatas.metadata_obj.authors }} {{ object.metadatas.metadata_obj.designation }} {{ object.metadatas.metadata_obj.datation }}