Turning iconolab-mcc into App to add specific import commands
authorRiwad Salim
Tue, 12 Jun 2018 13:11:49 +0200
changeset 5 cfd40849d24c
parent 4 9cc447bd5280
child 6 a676152d6bc5
Turning iconolab-mcc into App to add specific import commands
src/iconolab_mcc/__init__.py
src/iconolab_mcc/apps.py
src/iconolab_mcc/fixtures/demo_data.json
src/iconolab_mcc/management/__init__.py
src/iconolab_mcc/management/commands/__init__.py
src/iconolab_mcc/management/commands/importimages.py
src/iconolab_mcc/management/commands/updatecollection.py
src/iconolab_mcc/settings/__init__.py
src/iconolab_mcc/templates/iconolab/home.html
src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt
--- a/src/iconolab_mcc/__init__.py	Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/__init__.py	Tue Jun 12 13:11:49 2018 +0200
@@ -45,4 +45,4 @@
 
 __version__ = get_version(VERSION)
 
-default_app_config = 'iconolab.apps.IconolabApp'
+default_app_config = 'iconolab_mcc.apps.IconolabMccApp'
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/apps.py	Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+class IconolabMccApp(AppConfig):
+    name = 'iconolab_mcc'
+    verbose_name = 'Iconolab-mcc'
+
+    # def ready(self):
+    #     import iconolab.templatetags.iconolab_tags
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/fixtures/demo_data.json	Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,244 @@
+[
+	{
+	    "model": "auth.User",
+	    "pk": 1,
+	    "fields": {
+	    	"username": "contributeur1",
+	    	"password": "pbkdf2_sha256$24000$4t81p9toYpfc$ufZAWaUcF51dpNqHscABIoW7UyoXxYDCRlKFI87vQJM="   
+	    }
+	},{
+	    "model": "auth.User",
+	    "pk": 2,
+	    "fields": {
+	    	"username": "contributeur2",
+	    	"password": "pbkdf2_sha256$24000$b9CPP5bLU4FM$1IYIBS5y2CIEFV9jHdTdZjvj59hzx+yH5Akc5LsHo+g="
+	    }
+	},{
+	    "model": "iconolab.Collection",
+	    "pk": 1,
+	    "fields": {
+	    	"name": "ingres",
+	    	"verbose_name": "Musée d'Ingres"
+	    }
+	},{
+	    "model": "iconolab.Item",
+	    "pk": 1,
+	    "fields": {
+	    	"collection": 1,
+	    	"item_guid": "80858614-7988-494e-87f5-3a53d83ad889"
+	    }
+	},{
+	    "model": "iconolab.ItemMetadata",
+	    "pk": 1,
+	    "fields": {
+	    	"item": 1,
+	    	"technics": "dessin",
+	    	"joconde_ref": "1"
+	    }
+	},{
+	    "model": "iconolab.Image",
+	    "pk": 1,
+	    "fields": {
+	    	"image_guid" : "26aec320-dcfe-4cbc-b912-6a6c13e8916e",
+	    	"name" : "MIC.8.10.jpg",
+			"media" : "uploads/MIC.8.10.jpg",
+			"item": 1,
+			"height": 2478,
+			"width": 3744,
+			"created": "2016-03-23 14:13:44.913765+01"
+	    }
+	},{
+	    "model": "iconolab.ImageStats",
+	    "pk": 1,
+	    "fields": {
+	    	"image": 1,
+	    	"annotations_count": 0,
+			"submitted_revisions_count": 0,
+			"comments_count": 0,
+			"folders_inclusion_count": 0,
+			"tag_count": 0
+	    }
+	},{
+	    "model": "iconolab.Item",
+	    "pk": 2,
+	    "fields": {
+	    	"collection": 1,
+	    	"item_guid": "b597b937-0d4b-4f48-814d-6de3e308ac76"
+	    }
+	},{
+	    "model": "iconolab.ItemMetadata",
+	    "pk": 2,
+	    "fields": {
+	    	"item": 2,
+	    	"technics": "dessin",
+	    	"joconde_ref": "2"
+	    }
+	},{
+	    "model": "iconolab.Image",
+	    "pk": 2,
+	    "fields": {
+	    	"image_guid" : "de734457-b600-4bed-bb60-a03d906b8e6e",
+	    	"name" : "MIC.1.7.jpg",
+			"media" : "uploads/MIC.1.7.jpg",
+			"item": 2,
+			"height": 4672,
+			"width": 6171,
+			"created": "2016-03-23 14:13:44.913765+01"
+	    }
+	},{
+	    "model": "iconolab.ImageStats",
+	    "pk": 2,
+	    "fields": {
+	    	"image": 2,
+	    	"annotations_count": 0,
+			"submitted_revisions_count": 0,
+			"comments_count": 0,
+			"folders_inclusion_count": 0,
+			"tag_count": 0
+	    }
+	},{
+	    "model": "iconolab.Item",
+	    "pk": 3,
+	    "fields": {
+	    	"collection": 1,
+	    	"item_guid": "de301942-e60e-4e46-83c2-967bf3838f47"
+	    }
+	},{
+	    "model": "iconolab.ItemMetadata",
+	    "pk": 3,
+	    "fields": {
+	    	"item": 3,
+	    	"technics": "dessin",
+	    	"designation": "dessin format paysage",
+	    	"joconde_ref": "12345"
+	    }
+	},{
+	    "model": "iconolab.Image",
+	    "pk": 3,
+	    "fields": {
+	    	"image_guid" : "1a00cdd1-514e-4fd9-87ae-d9872b73143b",
+	    	"name" : "MIC.1.18.jpg",
+			"media" : "uploads/MIC.1.18.jpg",
+			"item": 3,
+			"height": 6208,
+			"width": 4704,
+			"created": "2016-03-23 14:13:44.913765+01"
+	    }
+	},{
+	    "model": "iconolab.ImageStats",
+	    "pk": 3,
+	    "fields": {
+	    	"image": 3,
+	    	"annotations_count": 0,
+			"submitted_revisions_count": 0,
+			"comments_count": 0,
+			"folders_inclusion_count": 0,
+			"tag_count": 0
+	    }
+	},{
+	    "model": "iconolab.Collection",
+	    "pk": 2,
+	    "fields": {
+	    	"name": "stdie",
+	    	"verbose_name": "Musée Saint-Dié"
+	    }
+	},{
+	    "model": "iconolab.Item",
+	    "pk": 4,
+	    "fields": {
+	    	"collection": 2,
+	    	"item_guid": "46ea6b45-caa4-47d5-bd0b-f1f22410b68e"
+	    }
+	},{
+	    "model": "iconolab.ItemMetadata",
+	    "pk": 4,
+	    "fields": {
+	    	"item": 4,
+	    	"designation": "Médaille Athènes",
+	    	"joconde_ref": "1234567"
+	    }
+	},{
+	    "model": "iconolab.Image",
+	    "pk": 4,
+	    "fields": {
+	    	"image_guid" : "7384a6af-e87b-42b5-b24b-30cadb8adebc",
+	    	"name" : "athene_medal1.jpg",
+			"media" : "uploads/athene_medal1.jpg",
+			"item": 4,
+			"height": 704,
+			"width": 704,
+			"created": "2016-03-23 14:13:44.913765+01"
+	    }
+	},{
+	    "model": "iconolab.ImageStats",
+	    "pk": 4,
+	    "fields": {
+	    	"image": 4,
+	    	"annotations_count": 0,
+			"submitted_revisions_count": 0,
+			"comments_count": 0,
+			"folders_inclusion_count": 0,
+			"tag_count": 0
+	    }
+	},{
+	    "model": "iconolab.Image",
+	    "pk": 5,
+	    "fields": {
+	    	"image_guid" : "0ad0ae3e-06cf-4452-9d62-042471ddfdc3",
+	    	"name" : "athene_medal2.jpg",
+			"media" : "uploads/athene_medal2.jpg",
+			"item": 4,
+			"height": 704,
+			"width": 704,
+			"created": "2016-03-23 14:13:44.913765+01"
+	    }
+	},{
+	    "model": "iconolab.ImageStats",
+	    "pk": 5,
+	    "fields": {
+	    	"image": 5,
+	    	"annotations_count": 0,
+			"submitted_revisions_count": 0,
+			"comments_count": 0,
+			"folders_inclusion_count": 0,
+			"tag_count": 0
+	    }
+	},{
+	    "model": "iconolab.MetaCategory",
+	    "pk": 1,
+	    "fields": {
+	    	"collection": 1,
+	    	"label": "Appel à contribution",
+	    	"triggers_notifications": 1
+	    }
+	},{
+	    "model": "iconolab.MetaCategory",
+	    "pk": 2,
+	    "fields": {
+	    	"collection": 1,
+	    	"label": "Appel à expertise",
+	    	"triggers_notifications": 3
+	    }
+	},{
+	    "model": "iconolab.MetaCategory",
+	    "pk": 3,
+	    "fields": {
+	    	"collection": 1,
+	    	"label": "Référence"
+	    }
+	},{
+	    "model": "iconolab.MetaCategory",
+	    "pk": 4,
+	    "fields": {
+	    	"collection": 1,
+	    	"label": "Accord"
+	    }
+	},{
+	    "model": "iconolab.MetaCategory",
+	    "pk": 5,
+	    "fields": {
+	    	"collection": 1,
+	    	"label": "Désaccord"
+	    }
+	}
+]
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/management/commands/importimages.py	Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,447 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder
+from PIL import Image as ImagePIL
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil, logging
+
+if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
+    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
+else:
+    logger = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+    help = 'import images from a directory into the media folder and creates item and image objects'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_path')
+        parser.add_argument(
+            '--jpeg-quality',
+            dest='jpeg_quality',
+            default=settings.IMG_JPG_DEFAULT_QUALITY,
+            help='Jpeg default quality'
+
+        )
+        parser.add_argument(
+            '--encoding',
+            dest='encoding',
+            default='utf-8',
+            help='CSV file encoding'
+
+        )
+        parser.add_argument(
+            '--collection-json',
+            dest='collection_json',
+            default=False,
+            help='creates a new collection from a json file, must be an object with fields : '+ \
+                 '"name" (identifier), '+ \
+                 '"verbose_name" (proper title name), '+ \
+                 '"description" (description on homepage, html is supported), '+ \
+                 '"image" (image on homepages, must be "uploads/<imgname>"), '+ \
+                 '"height" and "width" (height and width of the image)',
+        )
+        parser.add_argument(
+            '--collection-id',
+            dest='collection_id',
+            default=False,
+            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+        )
+        parser.add_argument(
+            '--metacategories-json',
+            dest='metacategories_json',
+            default=False,
+            help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',
+        )
+        parser.add_argument(
+            '--delimiter',
+            dest='csv_delimiter',
+            default=';',
+            help='csv file delimiter'
+        )
+        parser.add_argument(
+            '--no-jpg-conversion',
+            dest='no-jpg-conversion',
+            default=False,
+            help='use this option if you only want the image copied and not converted'
+        )
+        parser.add_argument(
+            '--img-filename-identifier',
+            dest='img_filename_identifier',
+            default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
+            help='codename of the csv field we\'ll try to match to find the related image to a given object'
+        )
+        parser.add_argument(
+            '--filename-regexp-prefix',
+            dest='filename_regexp_prefix',
+            default=r'.*',
+            help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'
+        )
+        parser.add_argument(
+            '--filename-regexp-suffix',
+            dest='filename_regexp_suffix',
+            default=r'[\.\-_].*',
+            help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'
+        )
+        parser.add_argument(
+            '--folders',
+            dest='import_folders',
+            default=False,
+            action='store_const',
+            const=True,
+            help='option to create folders'
+        )
+        parser.add_argument(
+            '--folders-regexp',
+            dest='folders_regexp',
+            default=False,
+            help='regexp used to extract the folder name/number'
+        )
+        parser.add_argument(
+            '--folders-metadata',
+            dest='folders_metadata',
+            default='REF',
+            help='metadata from which to extract the folder name/number'
+        )
+    def handle(self, *args, **options):
+        """
+            Step-by-step for import:
+
+            1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files
+            1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic:
+                * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas
+                to which we associate a list of column header that will identified as that metadata
+                * The cleaned_row_data list will associate the identifier with the actual value for its related column
+            2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated
+            to each row for the filtered data.
+            3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary
+            4) For each item:
+                We create the object in the database
+                * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT
+                We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image
+
+            Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command.
+        """
+        try:
+            print('# Logging with logger '+logger.name)
+            logger.debug('# Initializing command with args: %r', options)
+            # Check we have a collection to store data into:
+            source_dir = os.path.dirname(os.path.realpath(options.get('csv_path')))
+            print('# Checking collection args')
+            if options.get('collection_json'):
+                print('## Finding collection json data in '+source_dir)
+                collection_json_path = os.path.join(source_dir, options.get('collection_json'))
+                if not os.path.isfile(collection_json_path):
+                    print('### No '+options.get('collection_json')+'.json file was found in the source directory')
+                    raise ValueError('!!! Json file '+collection_json_path+' was not found !!!')
+                try:
+                    with open(collection_json_path) as json_fixture_file:
+                        collection_data = json.loads(json_fixture_file.read())
+                        for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:
+                            if not key in collection_data.keys():
+                                print('!!! Json file '+collection_json_path+' has no '+key+' field !!!')
+                                raise ValueError()
+                        if not collection_data.get('name', ''):
+                            print('!!! Collection data key "name" is empty')
+                            raise ValueError()
+                        if Collection.objects.filter(name=collection_data.get('name')).exists():
+                            print('!!! A Collection with the provided name already exists!')
+                            raise ValueError()
+                        if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):
+                            print('!!! Collection data has an image but no height and width')
+                            raise ValueError()
+                except ValueError as e:
+                    raise ValueError('!!! JSON Data is invalid. !!!')
+            elif options.get('collection_id'):
+                print('## Finding collection with id '+options.get('collection_id'))
+                try:
+                    collection = Collection.objects.get(pk=options.get('collection_id'))
+                except Collection.DoesNotExist:
+                    raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!')
+            else:
+                raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')
+
+            if options.get('metacategories_json'):
+                print('## Finding metacategories fixture json data in '+source_dir)
+                metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json'))
+                if not os.path.isfile(metacategories_json_path):
+                    print('### No '+options.get('metacategories_json')+'.json file was found in the source directory')
+                    raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!')
+                with open(metacategories_json_path) as metacategories_json_file:
+                    metacategories_data = json.loads(metacategories_json_file.read())
+                    for metacategory in metacategories_data:
+                        if metacategory.get('label', None) is None:
+                            raise ValueError('!!! Metacategory without label !!!')
+
+            if options['import_folders'] and not options['folders_regexp']:
+                raise ValueError('!!! No regexp specified to extract folder name !!!')
+
+            # We read the csv
+            delimiter = options.get('csv_delimiter')
+            if delimiter == '#9':
+                delimiter = chr(9)
+            if delimiter == '#29':
+                delimiter = chr(29)
+            if delimiter == '#30':
+                delimiter = chr(30)
+            if delimiter == '#31':
+                delimiter = chr(31)
+            csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter)
+            print('# Extracting data from csv file and storing it in standardized format')
+            # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
+            cleaned_csv_data=[]
+            duplicate_rows=[]
+            for row in csvreader:
+                cleaned_row_data = {}
+                for key in settings.IMPORT_FIELDS_DICT.keys():
+                    cleaned_row_data[key] = ''
+                    for row_key in row.keys():
+                        if row_key in settings.IMPORT_FIELDS_DICT[key]:
+                            if key == 'REF':
+                                ref_number, _, _ = row[row_key].partition(';')
+                                cleaned_row_data[key] = ref_number.rstrip()
+                            else:
+                                cleaned_row_data[key] = row[row_key]
+                            break
+                if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]:
+                    print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line")
+                    duplicate_rows.append(cleaned_row_data)
+                else:
+                    cleaned_csv_data.append(cleaned_row_data)
+            # Listing image files in csv directory
+            image_list = [
+                f for f in os.listdir(source_dir)
+                if os.path.isfile(os.path.join(source_dir, f))
+                and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png'))
+            ] # Maybe check if image another way
+            filtered_csv_data = []
+            no_image_rows = []
+            no_data_images = []
+            assigned_images = []
+            # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image
+            for item in cleaned_csv_data:
+                item['SRC_IMG_FILES'] = []
+                has_image = False
+                for image in image_list:
+                    img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix')
+                    if re.match(img_name_pattern, image):
+                        item['SRC_IMG_FILES'].append(image)
+                        assigned_images.append(image)
+                        has_image = True
+                if has_image:
+                    filtered_csv_data.append(item)
+                else:
+                    # We keep track of the entries that don't have any corresponding image
+                    no_image_rows.append(item)
+            # We keep track of the images that don't have any corresponding entry
+            for image in image_list:
+                if image not in assigned_images:
+                    no_data_images.append(image)
+
+            print('## found ' + str(len(filtered_csv_data))+' items with at least one image')
+            print('# Importing data into Iconolab')
+            if options.get('collection_json'):
+                print('## Loading collection json')
+                collection = Collection.objects.create(
+                    name = collection_data.get('name'),
+                    verbose_name = collection_data.get('verbose_name', ''),
+                    description = collection_data.get('description', ''),
+                    image = collection_data.get('image', ''),
+                    height = collection_data.get('height', 0),
+                    width = collection_data.get('width', 0),
+                )
+                if collection.image:
+                    collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))
+                    if not os.path.isfile(collection_image_path):
+                        print('### Moving collection image')
+                        _ , collection_image_name = os.path.split(collection_image_path)
+                        try:
+                            col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name))
+                            print('##### Generating or copying jpeg for '+collection_image_name)
+                            col_im.thumbnail(col_im.size)
+                            col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
+                        except Exception as e:
+                            print(e)
+            if options.get('metacategories_json'):
+                for metacategory in metacategories_data:
+                    MetaCategory.objects.create(
+                        collection = collection,
+                        label = metacategory.get('label'),
+                        triggers_notifications = metacategory.get('triggers_notifications', 0)
+                    )
+            print('## Converting image and moving it to static dir, creating Image and Item objects')
+            target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads')
+            print('### Images will be stored in '+target_dir)
+            for item in filtered_csv_data:
+                print('#### Computing metadatas for item '+item['REF']+' (natural key)')
+                if not item['REF']:
+                    print('#### No Natural key, skipping')
+                    continue
+                item_authors = item['AUTR']
+                item_school = item['ECOLE']
+                item_designation = ''
+                if item.get('TITR', ''):
+                    item_designation = item['TITR']
+                elif item.get('DENO', ''):
+                    item_designation = item['DENO']
+                elif item.get('APPL', ''):
+                    item_designation = item['APPL']
+                item_datation = ''
+                if item.get('PERI', ''):
+                    item_datation = item['PERI']
+                elif item.get('MILL', ''):
+                    item_datation = item['MILL']
+                elif item.get('EPOQ', ''):
+                    item_datation = item['EPOQ']
+                item_technics = item['TECH']
+                item_field = item['DOM']
+                item_measurements = item['DIMS']
+                item_create_or_usage_location = item['LIEUX']
+                item_discovery_context = item['DECV']
+                item_conservation_location = item['LOCA']
+                item_photo_credits = item['PHOT']
+                item_inventory_number = item['INV']
+                item_joconde_ref = item['REF']
+                if ItemMetadata.objects.filter(item__collection = collection, natural_key = item_joconde_ref).exists():
+                    print('#### An item with '+item['REF']+' for natural key, already exists in database in the import collection')
+
+                    if options['import_folders']:
+
+                        # Extract folder name from natural key
+                        m = re.search(options['folders_regexp'], item[options['folders_metadata']])
+                        folder_id = m.group(1)
+
+                        if not Folder.objects.filter(original_id=folder_id).exists():
+                            print('#### Creating folder "'+folder_id+'"')
+                            folder = Folder.objects.create(
+                                collection = collection,
+                                name = 'Dossier '+folder_id,
+                                original_id = folder_id
+                            )
+                        else:
+                            print('#### Folder "'+folder_id+'" already exists')
+                            folder = Folder.objects.get(original_id=folder_id)
+
+                        item_metadata = ItemMetadata.objects.get(item__collection = collection, natural_key = item_joconde_ref)
+                        item = item_metadata.item
+
+                        item.folders.add(folder)
+
+                else:
+                    print('#### Creating item '+item['REF']+' (natural key) in database')
+                    item_object = Item.objects.create(
+                        collection = collection
+                    )
+
+                    new_metadata = {
+                        "authors" : item_authors,
+                        "school" : item_school,
+                        "designation" : item_designation,
+                        "field" : item_field,
+                        "datation" : item_datation,
+                        "technics" : item_technics,
+                        "measurements" : item_measurements,
+                        "create_or_usage_location" : item_create_or_usage_location,
+                        "discovery_context" : item_discovery_context,
+                        "conservation_location" : item_conservation_location,
+                        "photo_credits" : item_photo_credits,
+                        "inventory_number" : item_inventory_number,
+                        "joconde_ref" : item_joconde_ref
+                    }
+                    ItemMetadata.objects.create(
+                        item = item_object,
+                        metadata = json.dumps(new_metadata),
+                        natural_key = item_joconde_ref
+                        )
+
+                    print('#### Computing item image(s)')
+                    for image in item['SRC_IMG_FILES']:
+                        (image_name, ext) = os.path.splitext(image)
+                        if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS:
+                            print('##### Copying file '+str(image)+' without converting')
+                            image_path = os.path.join(target_dir, image)
+                            new_image_name = image
+                            shutil.copy(os.path.join(source_dir, image), target_dir)
+                            try:
+                                im = ImagePIL.open(os.path.join(target_dir, image))
+                                im_width, im_height = im.size
+                            except Exception as e:
+                                print(e)
+                                continue
+                        else:
+                            image_path = os.path.join(target_dir, image_name) + '.jpg'
+                            new_image_name = image_name+'.jpg'
+                            if os.path.isfile(image_path):
+                                print('##### A jpeg file already exists in target dir for '+ image)
+                                try:
+                                    im = ImagePIL.open(image_path)
+                                    im_width, im_height = im.size
+                                except Exception as e:
+                                    print(e)
+                                    continue
+                            else:
+                                jpeg_img_path = image_path
+                                try:
+                                    im = ImagePIL.open(os.path.join(source_dir, image))
+                                    print('##### Generating or copying jpeg for '+image)
+                                    im.thumbnail(im.size)
+                                    im.save(jpeg_img_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
+                                    im_width, im_height = im.size
+                                except Exception as e:
+                                    print(e)
+                                    continue
+                        new_image = Image.objects.create(
+                            item = item_object,
+                            media = 'uploads/'+new_image_name,
+                            name = new_image_name,
+                            height = im_height,
+                            width = im_width
+                        )
+                        ImageStats.objects.create(
+                            image = new_image
+                        )
+                    print('### Generating thumbnails for item '+item['REF'])
+                    for image in item_object.images.all():
+                        for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+                            print('#### Thumbnail for size '+size)
+                            get_thumbnail(image.media, size, crop=False)
+
+            print('# All done!')
+
+            logger.debug('# Recap for import command: ')
+            print('# Images without data: ')
+            logger.debug('## Checking images left without data')
+            collection_image_file = os.path.split(str(collection.image))[1]
+            if no_data_images and collection_image_file in no_data_images:
+                no_data_images.remove(collection_image_file)
+
+            if no_data_images:
+                for image in no_data_images:
+                    logger.debug('### %r', image)
+                    print('## '+image)
+            else:
+                print('## Each image has one corresponding row!')
+                logger.debug('### Each image has one corresponding row!')
+            print('# CSV Items without image')
+            logger.debug('## Checking csv rows left without image')
+            if no_image_rows:
+                for item in no_image_rows:
+                    logger.debug('### %r', item['REF'])
+                    print('## Natural key: '+item['REF'])
+            else:
+                print('## Each row found at least one corresponding image!')
+                logger.debug('### Each row found at least one corresponding image!')
+            print('# Duplicate rows in csv')
+            logger.debug('## Checking duplicate rows in csv')
+            if duplicate_rows:
+                for item in no_image_rows:
+                    logger.debug('### %r: %r', options.get('img_filename_identifier'), item[options.get('img_filename_identifier')])
+                    print('## '+options.get('img_filename_identifier')+': '+item[options.get('img_filename_identifier')])
+            else:
+                print('## Each row found at least one corresponding image!')
+                logger.debug('### Each row found at least one corresponding image!')
+        except FileNotFoundError:
+            print('!!! File '+options.get('csv_path')+' does not exist. !!!')
+        except ValueError as e:
+            print(str(e))
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/management/commands/updatecollection.py	Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,144 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil
+
+class Command(BaseCommand):
+    help = "import images from a directory into the media folder and creates item and image objects"
+
+    def add_arguments(self, parser):
+        parser.add_argument("csv_path")
+        parser.add_argument(
+            '--encoding',
+            dest='encoding',
+            default='utf-8',
+            help='CSV file encoding'
+
+        )
+        parser.add_argument(
+            '--collection-id',
+            dest='collection_id',
+            default=False,
+            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+        )
+        parser.add_argument(
+            '--delimiter',
+            dest='csv_delimiter',
+            default=';',
+            help='csv file delimiter'
+        )
+
+    def handle(self, *args, **options):
+        try:
+            # Check we have a collection to store data into:
+            source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
+            print("# Checking collection args")
+            if options.get("collection_id"):
+                print("## Finding collection with id "+options.get("collection_id"))
+                try:
+                    collection = Collection.objects.get(pk=options.get("collection_id"))
+                except Collection.DoesNotExist:
+                    raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
+            else:
+                raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
+
+            # We read the csv
+            delimiter = options.get('csv_delimiter')
+            if delimiter == "#9":
+                delimiter = chr(9)
+            if delimiter == "#29":
+                delimiter = chr(29)
+            if delimiter == "#30":
+                delimiter = chr(30)
+            if delimiter == "#31":
+                delimiter = chr(31)
+            csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
+            print("# Extracting data from csv file and storing it in standardized format")
+            # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
+            cleaned_csv_data=[]
+            for row in csvreader:
+                cleaned_row_data = {}
+                for key in settings.IMPORT_FIELDS_DICT.keys():
+                    cleaned_row_data[key] = ""
+                    for row_key in row.keys():
+                        if row_key in settings.IMPORT_FIELDS_DICT[key]:
+                            # Handling the multiple natural keys exports nonsense
+                            if key == "REF":
+                                natural_key_number, _, _ = row[row_key].partition(";")
+                                cleaned_row_data[key] = natural_key_number.rstrip()
+                            else:
+                                cleaned_row_data[key] = row[row_key]
+                            break
+                cleaned_csv_data.append(cleaned_row_data)
+            # Listing image files in csv directory
+            filtered_csv_data = []
+            # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key)
+            for item in cleaned_csv_data:
+                if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists():
+                    filtered_csv_data.append(item)
+            print("## found " + str(len(filtered_csv_data))+" items to update")
+            print("# Updating data from Iconolab")
+            for item in filtered_csv_data:
+                item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first()
+                if not item["REF"]:
+                    print("#### No Natural key, skipping")
+                    continue
+                item_authors = item["AUTR"]
+                item_school = item["ECOLE"]
+                item_designation = ""
+                if item.get("TITR", ""):
+                    item_designation = item["TITR"]
+                elif item.get("DENO", ""):
+                    item_designation = item["DENO"]
+                elif item.get("APPL", ""):
+                    item_designation = item["APPL"]
+                item_datation = ""
+                if item.get("PERI", ""):
+                    item_datation = item["PERI"]
+                elif item.get("MILL", ""):
+                    item_datation = item["MILL"]
+                elif item.get("EPOQ", ""):
+                    item_datation = item["EPOQ"]
+                item_technics = item["TECH"]
+                item_field = item["DOM"]
+                item_measurements = item["DIMS"]
+                item_create_or_usage_location = item["LIEUX"]
+                item_discovery_context = item["DECV"]
+                item_conservation_location = item["LOCA"]
+                item_photo_credits = item["PHOT"]
+                item_inventory_number = item["INV"]
+                item_joconde_ref = item["REF"]
+                # Updating metadatas
+
+                new_metadata = {
+                    "authors" : item_authors,
+                    "school" : item_school,
+                    "designation" : item_designation,
+                    "field" : item_field,
+                    "datation" : item_datation,
+                    "technics" : item_technics,
+                    "measurements" : item_measurements,
+                    "create_or_usage_location" : item_create_or_usage_location,
+                    "discovery_context" : item_discovery_context,
+                    "conservation_location" : item_conservation_location,
+                    "photo_credits" : item_photo_credits,
+                    "inventory_number" : item_inventory_number,
+                    "joconde_ref" : item_joconde_ref
+                }
+
+                item_metadatas.metadata = json.dumps(new_metadata)
+                item_metadatas.natural_key = item_joconde_ref
+                item_metadatas.save()
+                print('### Generating thumbnails for item '+item['REF'])
+                for image in item_metadatas.item.images.all():
+                    for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+                        print('#### Thumbnail for size '+size)
+                        get_thumbnail(image.media, size, crop=False)
+            print("# All done!")
+        except FileNotFoundError:
+            print("!!! File "+options.get("csv_path")+" does not exist. !!!")
+        except ValueError as e:
+            print(str(e))
--- a/src/iconolab_mcc/settings/__init__.py	Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/settings/__init__.py	Tue Jun 12 13:11:49 2018 +0200
@@ -44,6 +44,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    'iconolab_mcc',
     'iconolab.apps.IconolabApp',
     'django.contrib.admin',
     'django.contrib.auth',
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/templates/iconolab/home.html	Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,139 @@
+{% extends 'iconolab_base.html' %}
+
+{% load i18n %}
+{% load staticfiles %}
+{% load thumbnail %}
+{% load iconolab_tags %}
+{% load humanize %}
+
+{% block content %}
+<div id="homepage-main">
+  <div class="alert alert-warning" role="alert">
+    <p><strong>Bienvenue sur iconolab. La plateforme
+    est actuellement en cours de développement.</strong></p>
+    <p>Nous entrons en phase d'expérimentation ouverte au grand public. Pour plus d'informations, prière de contacter <a href="mailto:{{ contact }}">{{ contact }}</a>.</p>
+  </div>
+  <div class="row">
+    <div class="col-xs-12">
+      <h3>Les collections</h3>
+    </div>
+  </div>
+  <div id="main-panel" class="row">
+
+    {% for collection in collections_primary %}
+    <div class="col-md-3">
+      <div class="home-collection">
+        {% thumbnail collection.image "450x250" crop="center" as im %}
+        <div>
+          <a href="{% url 'collection_home' collection.name %}">
+            <img src="{{ im.url }}" class="img-responsive">
+          </a>
+        </div>
+        <h4 class="text-center">{{collection.verbose_name}}</h4>
+        <div class="progress">
+          <div class="progress-bar progress-bar-info progress-bar-striped" role="progressbar"
+            aria-valuenow="{{ collection.completed_percent }}" aria-valuemin="0" aria-valuemax="100"
+            style="width: {{ collection.completed_percent }}%;">
+            {{ collection.completed_percent }}%
+          </div>
+        </div>
+        <a class="btn btn-default btn-primary btn-block" href="{% url 'collection_home' collection.name %}">
+          Contribuer
+        </a>
+        <hr>
+        <p class="home-collection-description" title="{{collection.description|safe}}">{{collection.description|safe}}</p>
+        {% endthumbnail %}
+      </div>
+    </div>
+    {% endfor %}
+  </div>
+  <div class="row">
+    <div class="col-md-6">
+      <h3>Les dernières annotations</h3>
+      {% for annotation in latest_annotations %}
+      <div class="panel panel-default">
+        <div class="panel-body">
+
+          <div class="row">
+            <div class="col-md-4">
+              <div class="fragment-container" style="position: relative">
+                {% thumbnail annotation.image.media "100x100" crop=False as im %}
+                  <a href="{% url 'annotation_detail' annotation.image.item.collection.name annotation.image.image_guid annotation.annotation_guid %}">
+                    <img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
+                    <svg width="{{ im.width }}" height="{{ im.height }}" version="1.1" style="position:absolute; top:0px; left: 0px">
+                      <g transform="matrix({% transform_matrix im_width=im.width im_height=im.height max_x=100 max_y=100 %})">
+                        <path d="{{ annotation.current_revision.fragment|clean_path }}" opacity="0.7" fill="orange"></path>
+                      </g>
+                    </svg>
+                  </a>
+                {% endthumbnail %}
+              </div>
+            </div>
+            <div class="col-md-8">
+
+              <h4>
+                {{ annotation.current_revision.title }}
+                <small class="pull-right">{{ annotation.current_revision.created|naturaltime }}</small>
+              </h4>
+              <p>
+                {% for contributor in annotation.stats.contributors.all %}
+                <a href="{% url 'user_home' slug=contributor.username %}">{{ contributor.username }}</a>
+                {% endfor %}
+              </p>
+              <p>{{ annotation.current_revision.description }}</p>
+              <p>
+                {% for tagging_info in annotation.current_revision.tagginginfo_set.all %}
+                <span class="label label-default"><i class="fa fa-tag"></i> {{ tagging_info.tag.label }}</span>
+                {% endfor %}
+              </p>
+            </div>
+          </div>
+        </div>
+      </div>
+      {% endfor %}
+    </div>
+    <div class="col-md-6">
+      <h3>Les meilleurs contributeurs</h3>
+      <ul class="list-group">
+      {% for best_contributor in best_contributors %}
+        <li class="list-group-item">
+          <span class="badge">{{ best_contributor.contributions }}</span>
+          <a href="{% url 'user_home' best_contributor.author.username %}">{{ best_contributor.author.username }}</a>
+        </li>
+      {% endfor %}
+      </ul>
+
+      <h3>Les mots-clé les plus pertinents</h3>
+      <ul class="list-group">
+        {% for most_accurate_tag in most_accurate_tags %}
+        <li class="list-group-item">
+          <span class="badge">{{ most_accurate_tag.annotation_count }}</span>
+          <a href="{% url 'search_indexes:model_search' 'annotations' %}?q={{ most_accurate_tag.tag.label }}&tags=1">
+            {{ most_accurate_tag.tag.label }}
+          </a>
+        </li>
+        {% endfor %}
+      </ul>
+
+    </div>
+  </div>
+  {% for collection in collections_secondary %}
+  <div id="collection-panel-{{collection.name}}" class="container collection-container panel panel-default">
+    <div class="row">
+      <div class="col-md-4">
+        {% thumbnail collection.image "350x350" crop=False as im %}
+          <img src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}">
+        {% endthumbnail %}
+      </div>
+      <div class="col-md-8">
+        <h3 class="collection-title">{{ collection.verbose_name }}</h3>
+        <p id="collection-description-short-{{collection.name}}" class="text-justify collection-description">
+          {{collection.description | safe}}
+        </p>
+        <a href="{% url 'collection_home' collection.name %}" class="btn btn-primary btn">Contribuer</a>
+      </div>
+    </div>
+  </div>
+  {% endfor %}
+</div>
+{% endblock %}
--- a/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt	Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt	Tue Jun 12 13:11:49 2018 +0200
@@ -1,4 +1,4 @@
-{{ object.metadatas.metadata_obj.meschool }}
+{{ object.metadatas.metadata_obj.school }}
 {{ object.metadatas.metadata_obj.authors }}
 {{ object.metadatas.metadata_obj.designation }}
 {{ object.metadatas.metadata_obj.datation }}