--- a/src/iconolab_mcc/__init__.py Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/__init__.py Tue Jun 12 13:11:49 2018 +0200
@@ -45,4 +45,4 @@
__version__ = get_version(VERSION)
-default_app_config = 'iconolab.apps.IconolabApp'
+default_app_config = 'iconolab_mcc.apps.IconolabMccApp'
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/apps.py Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+class IconolabMccApp(AppConfig):
+ name = 'iconolab_mcc'
+ verbose_name = 'Iconolab-mcc'
+
+ # def ready(self):
+ # import iconolab.templatetags.iconolab_tags
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/fixtures/demo_data.json Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,244 @@
+[
+ {
+ "model": "auth.User",
+ "pk": 1,
+ "fields": {
+ "username": "contributeur1",
+ "password": "pbkdf2_sha256$24000$4t81p9toYpfc$ufZAWaUcF51dpNqHscABIoW7UyoXxYDCRlKFI87vQJM="
+ }
+ },{
+ "model": "auth.User",
+ "pk": 2,
+ "fields": {
+ "username": "contributeur2",
+ "password": "pbkdf2_sha256$24000$b9CPP5bLU4FM$1IYIBS5y2CIEFV9jHdTdZjvj59hzx+yH5Akc5LsHo+g="
+ }
+ },{
+ "model": "iconolab.Collection",
+ "pk": 1,
+ "fields": {
+ "name": "ingres",
+ "verbose_name": "Musée d'Ingres"
+ }
+ },{
+ "model": "iconolab.Item",
+ "pk": 1,
+ "fields": {
+ "collection": 1,
+ "item_guid": "80858614-7988-494e-87f5-3a53d83ad889"
+ }
+ },{
+ "model": "iconolab.ItemMetadata",
+ "pk": 1,
+ "fields": {
+ "item": 1,
+ "technics": "dessin",
+ "joconde_ref": "1"
+ }
+ },{
+ "model": "iconolab.Image",
+ "pk": 1,
+ "fields": {
+ "image_guid" : "26aec320-dcfe-4cbc-b912-6a6c13e8916e",
+ "name" : "MIC.8.10.jpg",
+ "media" : "uploads/MIC.8.10.jpg",
+ "item": 1,
+ "height": 2478,
+ "width": 3744,
+ "created": "2016-03-23 14:13:44.913765+01"
+ }
+ },{
+ "model": "iconolab.ImageStats",
+ "pk": 1,
+ "fields": {
+ "image": 1,
+ "annotations_count": 0,
+ "submitted_revisions_count": 0,
+ "comments_count": 0,
+ "folders_inclusion_count": 0,
+ "tag_count": 0
+ }
+ },{
+ "model": "iconolab.Item",
+ "pk": 2,
+ "fields": {
+ "collection": 1,
+ "item_guid": "b597b937-0d4b-4f48-814d-6de3e308ac76"
+ }
+ },{
+ "model": "iconolab.ItemMetadata",
+ "pk": 2,
+ "fields": {
+ "item": 2,
+ "technics": "dessin",
+ "joconde_ref": "2"
+ }
+ },{
+ "model": "iconolab.Image",
+ "pk": 2,
+ "fields": {
+ "image_guid" : "de734457-b600-4bed-bb60-a03d906b8e6e",
+ "name" : "MIC.1.7.jpg",
+ "media" : "uploads/MIC.1.7.jpg",
+ "item": 2,
+ "height": 4672,
+ "width": 6171,
+ "created": "2016-03-23 14:13:44.913765+01"
+ }
+ },{
+ "model": "iconolab.ImageStats",
+ "pk": 2,
+ "fields": {
+ "image": 2,
+ "annotations_count": 0,
+ "submitted_revisions_count": 0,
+ "comments_count": 0,
+ "folders_inclusion_count": 0,
+ "tag_count": 0
+ }
+ },{
+ "model": "iconolab.Item",
+ "pk": 3,
+ "fields": {
+ "collection": 1,
+ "item_guid": "de301942-e60e-4e46-83c2-967bf3838f47"
+ }
+ },{
+ "model": "iconolab.ItemMetadata",
+ "pk": 3,
+ "fields": {
+ "item": 3,
+ "technics": "dessin",
+ "designation": "dessin format paysage",
+ "joconde_ref": "12345"
+ }
+ },{
+ "model": "iconolab.Image",
+ "pk": 3,
+ "fields": {
+ "image_guid" : "1a00cdd1-514e-4fd9-87ae-d9872b73143b",
+ "name" : "MIC.1.18.jpg",
+ "media" : "uploads/MIC.1.18.jpg",
+ "item": 3,
+ "height": 6208,
+ "width": 4704,
+ "created": "2016-03-23 14:13:44.913765+01"
+ }
+ },{
+ "model": "iconolab.ImageStats",
+ "pk": 3,
+ "fields": {
+ "image": 3,
+ "annotations_count": 0,
+ "submitted_revisions_count": 0,
+ "comments_count": 0,
+ "folders_inclusion_count": 0,
+ "tag_count": 0
+ }
+ },{
+ "model": "iconolab.Collection",
+ "pk": 2,
+ "fields": {
+ "name": "stdie",
+ "verbose_name": "Musée Saint-Dié"
+ }
+ },{
+ "model": "iconolab.Item",
+ "pk": 4,
+ "fields": {
+ "collection": 2,
+ "item_guid": "46ea6b45-caa4-47d5-bd0b-f1f22410b68e"
+ }
+ },{
+ "model": "iconolab.ItemMetadata",
+ "pk": 4,
+ "fields": {
+ "item": 4,
+ "designation": "Médaille Athènes",
+ "joconde_ref": "1234567"
+ }
+ },{
+ "model": "iconolab.Image",
+ "pk": 4,
+ "fields": {
+ "image_guid" : "7384a6af-e87b-42b5-b24b-30cadb8adebc",
+ "name" : "athene_medal1.jpg",
+ "media" : "uploads/athene_medal1.jpg",
+ "item": 4,
+ "height": 704,
+ "width": 704,
+ "created": "2016-03-23 14:13:44.913765+01"
+ }
+ },{
+ "model": "iconolab.ImageStats",
+ "pk": 4,
+ "fields": {
+ "image": 4,
+ "annotations_count": 0,
+ "submitted_revisions_count": 0,
+ "comments_count": 0,
+ "folders_inclusion_count": 0,
+ "tag_count": 0
+ }
+ },{
+ "model": "iconolab.Image",
+ "pk": 5,
+ "fields": {
+ "image_guid" : "0ad0ae3e-06cf-4452-9d62-042471ddfdc3",
+ "name" : "athene_medal2.jpg",
+ "media" : "uploads/athene_medal2.jpg",
+ "item": 4,
+ "height": 704,
+ "width": 704,
+ "created": "2016-03-23 14:13:44.913765+01"
+ }
+ },{
+ "model": "iconolab.ImageStats",
+ "pk": 5,
+ "fields": {
+ "image": 5,
+ "annotations_count": 0,
+ "submitted_revisions_count": 0,
+ "comments_count": 0,
+ "folders_inclusion_count": 0,
+ "tag_count": 0
+ }
+ },{
+ "model": "iconolab.MetaCategory",
+ "pk": 1,
+ "fields": {
+ "collection": 1,
+ "label": "Appel à contribution",
+ "triggers_notifications": 1
+ }
+ },{
+ "model": "iconolab.MetaCategory",
+ "pk": 2,
+ "fields": {
+ "collection": 1,
+ "label": "Appel à expertise",
+ "triggers_notifications": 3
+ }
+ },{
+ "model": "iconolab.MetaCategory",
+ "pk": 3,
+ "fields": {
+ "collection": 1,
+ "label": "Référence"
+ }
+ },{
+ "model": "iconolab.MetaCategory",
+ "pk": 4,
+ "fields": {
+ "collection": 1,
+ "label": "Accord"
+ }
+ },{
+ "model": "iconolab.MetaCategory",
+ "pk": 5,
+ "fields": {
+ "collection": 1,
+ "label": "Désaccord"
+ }
+ }
+]
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/management/commands/importimages.py Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,447 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder
+from PIL import Image as ImagePIL
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil, logging
+
+if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
+ logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
+else:
+ logger = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+ help = 'import images from a directory into the media folder and creates item and image objects'
+
+ def add_arguments(self, parser):
+ parser.add_argument('csv_path')
+ parser.add_argument(
+ '--jpeg-quality',
+ dest='jpeg_quality',
+ default=settings.IMG_JPG_DEFAULT_QUALITY,
+ help='Jpeg default quality'
+
+ )
+ parser.add_argument(
+ '--encoding',
+ dest='encoding',
+ default='utf-8',
+ help='CSV file encoding'
+
+ )
+ parser.add_argument(
+ '--collection-json',
+ dest='collection_json',
+ default=False,
+ help='creates a new collection from a json file, must be an object with fields : '+ \
+ '"name" (identifier), '+ \
+ '"verbose_name" (proper title name), '+ \
+ '"description" (description on homepage, html is supported), '+ \
+ '"image" (image on homepages, must be "uploads/<imgname>"), '+ \
+ '"height" and "width" (height and width of the image)',
+ )
+ parser.add_argument(
+ '--collection-id',
+ dest='collection_id',
+ default=False,
+ help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+ )
+ parser.add_argument(
+ '--metacategories-json',
+ dest='metacategories_json',
+ default=False,
+ help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',
+ )
+ parser.add_argument(
+ '--delimiter',
+ dest='csv_delimiter',
+ default=';',
+ help='csv file delimiter'
+ )
+ parser.add_argument(
+ '--no-jpg-conversion',
+ dest='no-jpg-conversion',
+ default=False,
+ help='use this option if you only want the image copied and not converted'
+ )
+ parser.add_argument(
+ '--img-filename-identifier',
+ dest='img_filename_identifier',
+ default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
+ help='codename of the csv field we\'ll try to match to find the related image to a given object'
+ )
+ parser.add_argument(
+ '--filename-regexp-prefix',
+ dest='filename_regexp_prefix',
+ default=r'.*',
+ help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'
+ )
+ parser.add_argument(
+ '--filename-regexp-suffix',
+ dest='filename_regexp_suffix',
+ default=r'[\.\-_].*',
+ help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'
+ )
+ parser.add_argument(
+ '--folders',
+ dest='import_folders',
+ default=False,
+ action='store_const',
+ const=True,
+ help='option to create folders'
+ )
+ parser.add_argument(
+ '--folders-regexp',
+ dest='folders_regexp',
+ default=False,
+ help='regexp used to extract the folder name/number'
+ )
+ parser.add_argument(
+ '--folders-metadata',
+ dest='folders_metadata',
+ default='REF',
+ help='metadata from which to extract the folder name/number'
+ )
+ def handle(self, *args, **options):
+ """
+ Step-by-step for import:
+
+ 1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files
+ 1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic:
+ * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas
+ to which we associate a list of column header that will identified as that metadata
+ * The cleaned_row_data list will associate the identifier with the actual value for its related column
+ 2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated
+ to each row for the filtered data.
+ 3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary
+ 4) For each item:
+ We create the object in the database
+ * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT
+ We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image
+
+ Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command.
+ """
+ try:
+ print('# Logging with logger '+logger.name)
+ logger.debug('# Initializing command with args: %r', options)
+ # Check we have a collection to store data into:
+ source_dir = os.path.dirname(os.path.realpath(options.get('csv_path')))
+ print('# Checking collection args')
+ if options.get('collection_json'):
+ print('## Finding collection json data in '+source_dir)
+ collection_json_path = os.path.join(source_dir, options.get('collection_json'))
+ if not os.path.isfile(collection_json_path):
+ print('### No '+options.get('collection_json')+'.json file was found in the source directory')
+ raise ValueError('!!! Json file '+collection_json_path+' was not found !!!')
+ try:
+ with open(collection_json_path) as json_fixture_file:
+ collection_data = json.loads(json_fixture_file.read())
+ for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:
+ if not key in collection_data.keys():
+ print('!!! Json file '+collection_json_path+' has no '+key+' field !!!')
+ raise ValueError()
+ if not collection_data.get('name', ''):
+ print('!!! Collection data key "name" is empty')
+ raise ValueError()
+ if Collection.objects.filter(name=collection_data.get('name')).exists():
+ print('!!! A Collection with the provided name already exists!')
+ raise ValueError()
+ if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):
+ print('!!! Collection data has an image but no height and width')
+ raise ValueError()
+ except ValueError as e:
+ raise ValueError('!!! JSON Data is invalid. !!!')
+ elif options.get('collection_id'):
+ print('## Finding collection with id '+options.get('collection_id'))
+ try:
+ collection = Collection.objects.get(pk=options.get('collection_id'))
+ except Collection.DoesNotExist:
+ raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!')
+ else:
+ raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')
+
+ if options.get('metacategories_json'):
+ print('## Finding metacategories fixture json data in '+source_dir)
+ metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json'))
+ if not os.path.isfile(metacategories_json_path):
+ print('### No '+options.get('metacategories_json')+'.json file was found in the source directory')
+ raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!')
+ with open(metacategories_json_path) as metacategories_json_file:
+ metacategories_data = json.loads(metacategories_json_file.read())
+ for metacategory in metacategories_data:
+ if metacategory.get('label', None) is None:
+ raise ValueError('!!! Metacategory without label !!!')
+
+ if options['import_folders'] and not options['folders_regexp']:
+ raise ValueError('!!! No regexp specified to extract folder name !!!')
+
+ # We read the csv
+ delimiter = options.get('csv_delimiter')
+ if delimiter == '#9':
+ delimiter = chr(9)
+ if delimiter == '#29':
+ delimiter = chr(29)
+ if delimiter == '#30':
+ delimiter = chr(30)
+ if delimiter == '#31':
+ delimiter = chr(31)
+ csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter)
+ print('# Extracting data from csv file and storing it in standardized format')
+ # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
+ cleaned_csv_data=[]
+ duplicate_rows=[]
+ for row in csvreader:
+ cleaned_row_data = {}
+ for key in settings.IMPORT_FIELDS_DICT.keys():
+ cleaned_row_data[key] = ''
+ for row_key in row.keys():
+ if row_key in settings.IMPORT_FIELDS_DICT[key]:
+ if key == 'REF':
+ ref_number, _, _ = row[row_key].partition(';')
+ cleaned_row_data[key] = ref_number.rstrip()
+ else:
+ cleaned_row_data[key] = row[row_key]
+ break
+ if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]:
+ print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line")
+ duplicate_rows.append(cleaned_row_data)
+ else:
+ cleaned_csv_data.append(cleaned_row_data)
+ # Listing image files in csv directory
+ image_list = [
+ f for f in os.listdir(source_dir)
+ if os.path.isfile(os.path.join(source_dir, f))
+ and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png'))
+ ] # Maybe check if image another way
+ filtered_csv_data = []
+ no_image_rows = []
+ no_data_images = []
+ assigned_images = []
+ # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image
+ for item in cleaned_csv_data:
+ item['SRC_IMG_FILES'] = []
+ has_image = False
+ for image in image_list:
+ img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix')
+ if re.match(img_name_pattern, image):
+ item['SRC_IMG_FILES'].append(image)
+ assigned_images.append(image)
+ has_image = True
+ if has_image:
+ filtered_csv_data.append(item)
+ else:
+ # We keep track of the entries that don't have any corresponding image
+ no_image_rows.append(item)
+ # We keep track of the images that don't have any corresponding entry
+ for image in image_list:
+ if image not in assigned_images:
+ no_data_images.append(image)
+
+ print('## found ' + str(len(filtered_csv_data))+' items with at least one image')
+ print('# Importing data into Iconolab')
+ if options.get('collection_json'):
+ print('## Loading collection json')
+ collection = Collection.objects.create(
+ name = collection_data.get('name'),
+ verbose_name = collection_data.get('verbose_name', ''),
+ description = collection_data.get('description', ''),
+ image = collection_data.get('image', ''),
+ height = collection_data.get('height', 0),
+ width = collection_data.get('width', 0),
+ )
+ if collection.image:
+ collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))
+ if not os.path.isfile(collection_image_path):
+ print('### Moving collection image')
+ _ , collection_image_name = os.path.split(collection_image_path)
+ try:
+ col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name))
+ print('##### Generating or copying jpeg for '+collection_image_name)
+ col_im.thumbnail(col_im.size)
+ col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
+ except Exception as e:
+ print(e)
+ if options.get('metacategories_json'):
+ for metacategory in metacategories_data:
+ MetaCategory.objects.create(
+ collection = collection,
+ label = metacategory.get('label'),
+ triggers_notifications = metacategory.get('triggers_notifications', 0)
+ )
+ print('## Converting image and moving it to static dir, creating Image and Item objects')
+ target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads')
+ print('### Images will be stored in '+target_dir)
+ for item in filtered_csv_data:
+ print('#### Computing metadatas for item '+item['REF']+' (natural key)')
+ if not item['REF']:
+ print('#### No Natural key, skipping')
+ continue
+ item_authors = item['AUTR']
+ item_school = item['ECOLE']
+ item_designation = ''
+ if item.get('TITR', ''):
+ item_designation = item['TITR']
+ elif item.get('DENO', ''):
+ item_designation = item['DENO']
+ elif item.get('APPL', ''):
+ item_designation = item['APPL']
+ item_datation = ''
+ if item.get('PERI', ''):
+ item_datation = item['PERI']
+ elif item.get('MILL', ''):
+ item_datation = item['MILL']
+ elif item.get('EPOQ', ''):
+ item_datation = item['EPOQ']
+ item_technics = item['TECH']
+ item_field = item['DOM']
+ item_measurements = item['DIMS']
+ item_create_or_usage_location = item['LIEUX']
+ item_discovery_context = item['DECV']
+ item_conservation_location = item['LOCA']
+ item_photo_credits = item['PHOT']
+ item_inventory_number = item['INV']
+ item_joconde_ref = item['REF']
+ if ItemMetadata.objects.filter(item__collection = collection, natural_key = item_joconde_ref).exists():
+ print('#### An item with '+item['REF']+' for natural key, already exists in database in the import collection')
+
+ if options['import_folders']:
+
+ # Extract folder name from natural key
+ m = re.search(options['folders_regexp'], item[options['folders_metadata']])
+ folder_id = m.group(1)
+
+ if not Folder.objects.filter(original_id=folder_id).exists():
+ print('#### Creating folder "'+folder_id+'"')
+ folder = Folder.objects.create(
+ collection = collection,
+ name = 'Dossier '+folder_id,
+ original_id = folder_id
+ )
+ else:
+ print('#### Folder "'+folder_id+'" already exists')
+ folder = Folder.objects.get(original_id=folder_id)
+
+ item_metadata = ItemMetadata.objects.get(item__collection = collection, natural_key = item_joconde_ref)
+ item = item_metadata.item
+
+ item.folders.add(folder)
+
+ else:
+ print('#### Creating item '+item['REF']+' (natural key) in database')
+ item_object = Item.objects.create(
+ collection = collection
+ )
+
+ new_metadata = {
+ "authors" : item_authors,
+ "school" : item_school,
+ "designation" : item_designation,
+ "field" : item_field,
+ "datation" : item_datation,
+ "technics" : item_technics,
+ "measurements" : item_measurements,
+ "create_or_usage_location" : item_create_or_usage_location,
+ "discovery_context" : item_discovery_context,
+ "conservation_location" : item_conservation_location,
+ "photo_credits" : item_photo_credits,
+ "inventory_number" : item_inventory_number,
+ "joconde_ref" : item_joconde_ref
+ }
+ ItemMetadata.objects.create(
+ item = item_object,
+ metadata = json.dumps(new_metadata),
+ natural_key = item_joconde_ref
+ )
+
+ print('#### Computing item image(s)')
+ for image in item['SRC_IMG_FILES']:
+ (image_name, ext) = os.path.splitext(image)
+ if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS:
+ print('##### Copying file '+str(image)+' without converting')
+ image_path = os.path.join(target_dir, image)
+ new_image_name = image
+ shutil.copy(os.path.join(source_dir, image), target_dir)
+ try:
+ im = ImagePIL.open(os.path.join(target_dir, image))
+ im_width, im_height = im.size
+ except Exception as e:
+ print(e)
+ continue
+ else:
+ image_path = os.path.join(target_dir, image_name) + '.jpg'
+ new_image_name = image_name+'.jpg'
+ if os.path.isfile(image_path):
+ print('##### A jpeg file already exists in target dir for '+ image)
+ try:
+ im = ImagePIL.open(image_path)
+ im_width, im_height = im.size
+ except Exception as e:
+ print(e)
+ continue
+ else:
+ jpeg_img_path = image_path
+ try:
+ im = ImagePIL.open(os.path.join(source_dir, image))
+ print('##### Generating or copying jpeg for '+image)
+ im.thumbnail(im.size)
+ im.save(jpeg_img_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
+ im_width, im_height = im.size
+ except Exception as e:
+ print(e)
+ continue
+ new_image = Image.objects.create(
+ item = item_object,
+ media = 'uploads/'+new_image_name,
+ name = new_image_name,
+ height = im_height,
+ width = im_width
+ )
+ ImageStats.objects.create(
+ image = new_image
+ )
+ print('### Generating thumbnails for item '+item['REF'])
+ for image in item_object.images.all():
+ for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+ print('#### Thumbnail for size '+size)
+ get_thumbnail(image.media, size, crop=False)
+
+ print('# All done!')
+
+ logger.debug('# Recap for import command: ')
+ print('# Images without data: ')
+ logger.debug('## Checking images left without data')
+ collection_image_file = os.path.split(str(collection.image))[1]
+ if no_data_images and collection_image_file in no_data_images:
+ no_data_images.remove(collection_image_file)
+
+ if no_data_images:
+ for image in no_data_images:
+ logger.debug('### %r', image)
+ print('## '+image)
+ else:
+ print('## Each image has one corresponding row!')
+ logger.debug('### Each image has one corresponding row!')
+ print('# CSV Items without image')
+ logger.debug('## Checking csv rows left without image')
+ if no_image_rows:
+ for item in no_image_rows:
+ logger.debug('### %r', item['REF'])
+ print('## Natural key: '+item['REF'])
+ else:
+ print('## Each row found at least one corresponding image!')
+ logger.debug('### Each row found at least one corresponding image!')
+ print('# Duplicate rows in csv')
+ logger.debug('## Checking duplicate rows in csv')
+ if duplicate_rows:
+ for item in no_image_rows:
+ logger.debug('### %r: %r', options.get('img_filename_identifier'), item[options.get('img_filename_identifier')])
+ print('## '+options.get('img_filename_identifier')+': '+item[options.get('img_filename_identifier')])
+ else:
+ print('## Each row found at least one corresponding image!')
+ logger.debug('### Each row found at least one corresponding image!')
+ except FileNotFoundError:
+ print('!!! File '+options.get('csv_path')+' does not exist. !!!')
+ except ValueError as e:
+ print(str(e))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/management/commands/updatecollection.py Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,144 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil
+
+class Command(BaseCommand):
+ help = "import images from a directory into the media folder and creates item and image objects"
+
+ def add_arguments(self, parser):
+ parser.add_argument("csv_path")
+ parser.add_argument(
+ '--encoding',
+ dest='encoding',
+ default='utf-8',
+ help='CSV file encoding'
+
+ )
+ parser.add_argument(
+ '--collection-id',
+ dest='collection_id',
+ default=False,
+ help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+ )
+ parser.add_argument(
+ '--delimiter',
+ dest='csv_delimiter',
+ default=';',
+ help='csv file delimiter'
+ )
+
+ def handle(self, *args, **options):
+ try:
+ # Check we have a collection to store data into:
+ source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
+ print("# Checking collection args")
+ if options.get("collection_id"):
+ print("## Finding collection with id "+options.get("collection_id"))
+ try:
+ collection = Collection.objects.get(pk=options.get("collection_id"))
+ except Collection.DoesNotExist:
+ raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
+ else:
+ raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
+
+ # We read the csv
+ delimiter = options.get('csv_delimiter')
+ if delimiter == "#9":
+ delimiter = chr(9)
+ if delimiter == "#29":
+ delimiter = chr(29)
+ if delimiter == "#30":
+ delimiter = chr(30)
+ if delimiter == "#31":
+ delimiter = chr(31)
+ csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
+ print("# Extracting data from csv file and storing it in standardized format")
+ # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
+ cleaned_csv_data=[]
+ for row in csvreader:
+ cleaned_row_data = {}
+ for key in settings.IMPORT_FIELDS_DICT.keys():
+ cleaned_row_data[key] = ""
+ for row_key in row.keys():
+ if row_key in settings.IMPORT_FIELDS_DICT[key]:
+ # Handling the multiple natural keys exports nonsense
+ if key == "REF":
+ natural_key_number, _, _ = row[row_key].partition(";")
+ cleaned_row_data[key] = natural_key_number.rstrip()
+ else:
+ cleaned_row_data[key] = row[row_key]
+ break
+ cleaned_csv_data.append(cleaned_row_data)
+ # Listing image files in csv directory
+ filtered_csv_data = []
+ # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key)
+ for item in cleaned_csv_data:
+ if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists():
+ filtered_csv_data.append(item)
+ print("## found " + str(len(filtered_csv_data))+" items to update")
+ print("# Updating data from Iconolab")
+ for item in filtered_csv_data:
+ item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first()
+ if not item["REF"]:
+ print("#### No Natural key, skipping")
+ continue
+ item_authors = item["AUTR"]
+ item_school = item["ECOLE"]
+ item_designation = ""
+ if item.get("TITR", ""):
+ item_designation = item["TITR"]
+ elif item.get("DENO", ""):
+ item_designation = item["DENO"]
+ elif item.get("APPL", ""):
+ item_designation = item["APPL"]
+ item_datation = ""
+ if item.get("PERI", ""):
+ item_datation = item["PERI"]
+ elif item.get("MILL", ""):
+ item_datation = item["MILL"]
+ elif item.get("EPOQ", ""):
+ item_datation = item["EPOQ"]
+ item_technics = item["TECH"]
+ item_field = item["DOM"]
+ item_measurements = item["DIMS"]
+ item_create_or_usage_location = item["LIEUX"]
+ item_discovery_context = item["DECV"]
+ item_conservation_location = item["LOCA"]
+ item_photo_credits = item["PHOT"]
+ item_inventory_number = item["INV"]
+ item_joconde_ref = item["REF"]
+ # Updating metadatas
+
+ new_metadata = {
+ "authors" : item_authors,
+ "school" : item_school,
+ "designation" : item_designation,
+ "field" : item_field,
+ "datation" : item_datation,
+ "technics" : item_technics,
+ "measurements" : item_measurements,
+ "create_or_usage_location" : item_create_or_usage_location,
+ "discovery_context" : item_discovery_context,
+ "conservation_location" : item_conservation_location,
+ "photo_credits" : item_photo_credits,
+ "inventory_number" : item_inventory_number,
+ "joconde_ref" : item_joconde_ref
+ }
+
+ item_metadatas.metadata = json.dumps(new_metadata)
+ item_metadatas.natural_key = item_joconde_ref
+ item_metadatas.save()
+ print('### Generating thumbnails for item '+item['REF'])
+ for image in item_metadatas.item.images.all():
+ for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+ print('#### Thumbnail for size '+size)
+ get_thumbnail(image.media, size, crop=False)
+ print("# All done!")
+ except FileNotFoundError:
+ print("!!! File "+options.get("csv_path")+" does not exist. !!!")
+ except ValueError as e:
+ print(str(e))
--- a/src/iconolab_mcc/settings/__init__.py Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/settings/__init__.py Tue Jun 12 13:11:49 2018 +0200
@@ -44,6 +44,7 @@
# Application definition
INSTALLED_APPS = [
+ 'iconolab_mcc',
'iconolab.apps.IconolabApp',
'django.contrib.admin',
'django.contrib.auth',
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab_mcc/templates/iconolab/home.html Tue Jun 12 13:11:49 2018 +0200
@@ -0,0 +1,139 @@
+{% extends 'iconolab_base.html' %}
+
+{% load i18n %}
+{% load staticfiles %}
+{% load thumbnail %}
+{% load iconolab_tags %}
+{% load humanize %}
+
+{% block content %}
+<div id="homepage-main">
+ <div class="alert alert-warning" role="alert">
+ <p><strong>Bienvenue sur iconolab. La plateforme
+ est actuellement en cours de développement.</strong></p>
+ <p>Nous entrons en phase d'expérimentation ouverte au grand public. Pour plus d'informations, prière de contacter <a href="mailto:{{ contact }}">{{ contact }}</a>.</p>
+ </div>
+ <div class="row">
+ <div class="col-xs-12">
+ <h3>Les collections</h3>
+ </div>
+ </div>
+ <div id="main-panel" class="row">
+
+ {% for collection in collections_primary %}
+ <div class="col-md-3">
+ <div class="home-collection">
+ {% thumbnail collection.image "450x250" crop="center" as im %}
+ <div>
+ <a href="{% url 'collection_home' collection.name %}">
+ <img src="{{ im.url }}" class="img-responsive">
+ </a>
+ </div>
+ <h4 class="text-center">{{collection.verbose_name}}</h4>
+ <div class="progress">
+ <div class="progress-bar progress-bar-info progress-bar-striped" role="progressbar"
+ aria-valuenow="{{ collection.completed_percent }}" aria-valuemin="0" aria-valuemax="100"
+ style="width: {{ collection.completed_percent }}%;">
+ {{ collection.completed_percent }}%
+ </div>
+ </div>
+ <a class="btn btn-default btn-primary btn-block" href="{% url 'collection_home' collection.name %}">
+ Contribuer
+ </a>
+ <hr>
+ <p class="home-collection-description" title="{{collection.description|safe}}">{{collection.description|safe}}</p>
+ {% endthumbnail %}
+ </div>
+ </div>
+ {% endfor %}
+ </div>
+ <div class="row">
+ <div class="col-md-6">
+ <h3>Les dernières annotations</h3>
+ {% for annotation in latest_annotations %}
+ <div class="panel panel-default">
+ <div class="panel-body">
+
+ <div class="row">
+ <div class="col-md-4">
+ <div class="fragment-container" style="position: relative">
+ {% thumbnail annotation.image.media "100x100" crop=False as im %}
+ <a href="{% url 'annotation_detail' annotation.image.item.collection.name annotation.image.image_guid annotation.annotation_guid %}">
+ <img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
+ <svg width="{{ im.width }}" height="{{ im.height }}" version="1.1" style="position:absolute; top:0px; left: 0px">
+ <g transform="matrix({% transform_matrix im_width=im.width im_height=im.height max_x=100 max_y=100 %})">
+ <path d="{{ annotation.current_revision.fragment|clean_path }}" opacity="0.7" fill="orange"></path>
+ </g>
+ </svg>
+ </a>
+ {% endthumbnail %}
+ </div>
+ </div>
+ <div class="col-md-8">
+
+ <h4>
+ {{ annotation.current_revision.title }}
+ <small class="pull-right">{{ annotation.current_revision.created|naturaltime }}</small>
+ </h4>
+ <p>
+ {% for contributor in annotation.stats.contributors.all %}
+ <a href="{% url 'user_home' slug=contributor.username %}">{{ contributor.username }}</a>
+ {% endfor %}
+ </p>
+ <p>{{ annotation.current_revision.description }}</p>
+ <p>
+ {% for tagging_info in annotation.current_revision.tagginginfo_set.all %}
+ <span class="label label-default"><i class="fa fa-tag"></i> {{ tagging_info.tag.label }}</span>
+ {% endfor %}
+ </p>
+ </div>
+ </div>
+ </div>
+ </div>
+ {% endfor %}
+ </div>
+ <div class="col-md-6">
+ <h3>Les meilleurs contributeurs</h3>
+ <ul class="list-group">
+ {% for best_contributor in best_contributors %}
+ <li class="list-group-item">
+ <span class="badge">{{ best_contributor.contributions }}</span>
+ <a href="{% url 'user_home' best_contributor.author.username %}">{{ best_contributor.author.username }}</a>
+ </li>
+ {% endfor %}
+ </ul>
+
+ <h3>Les mots-clé les plus pertinents</h3>
+ <ul class="list-group">
+ {% for most_accurate_tag in most_accurate_tags %}
+ <li class="list-group-item">
+ <span class="badge">{{ most_accurate_tag.annotation_count }}</span>
+ <a href="{% url 'search_indexes:model_search' 'annotations' %}?q={{ most_accurate_tag.tag.label }}&tags=1">
+ {{ most_accurate_tag.tag.label }}
+ </a>
+ </li>
+ {% endfor %}
+ </ul>
+
+ </div>
+ </div>
+ {% for collection in collections_secondary %}
+ <div id="collection-panel-{{collection.name}}" class="container collection-container panel panel-default">
+ <div class="row">
+ <div class="col-md-4">
+ {% thumbnail collection.image "350x350" crop=False as im %}
+ <img src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}">
+ {% endthumbnail %}
+ </div>
+ <div class="col-md-8">
+ <h3 class="collection-title">{{ collection.verbose_name }}</h3>
+ <p id="collection-description-short-{{collection.name}}" class="text-justify collection-description">
+ {{collection.description | safe}}
+ </p>
+ <a href="{% url 'collection_home' collection.name %}" class="btn btn-primary btn">Contribuer</a>
+ </div>
+ </div>
+ </div>
+ {% endfor %}
+</div>
+{% endblock %}
--- a/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt Thu Jun 07 17:12:48 2018 +0200
+++ b/src/iconolab_mcc/templates/iconolab/search/indexes/iconolab/item_text.txt Tue Jun 12 13:11:49 2018 +0200
@@ -1,4 +1,4 @@
-{{ object.metadatas.metadata_obj.meschool }}
+{{ object.metadatas.metadata_obj.school }}
{{ object.metadatas.metadata_obj.authors }}
{{ object.metadatas.metadata_obj.designation }}
{{ object.metadatas.metadata_obj.datation }}