--- a/src/iconolab/fixtures/demo_data.json Thu Jun 07 16:07:52 2018 +0200
+++ b/src/iconolab/fixtures/demo_data.json Tue Jun 12 13:33:18 2018 +0200
@@ -4,7 +4,7 @@
"pk": 1,
"fields": {
"username": "contributeur1",
- "password": "pbkdf2_sha256$24000$4t81p9toYpfc$ufZAWaUcF51dpNqHscABIoW7UyoXxYDCRlKFI87vQJM="
+ "password": "pbkdf2_sha256$24000$4t81p9toYpfc$ufZAWaUcF51dpNqHscABIoW7UyoXxYDCRlKFI87vQJM="
}
},{
"model": "auth.User",
@@ -17,35 +17,33 @@
"model": "iconolab.Collection",
"pk": 1,
"fields": {
- "name": "ingres",
- "verbose_name": "Musée d'Ingres"
+ "name": "",
+ "verbose_name": ""
}
},{
"model": "iconolab.Item",
"pk": 1,
"fields": {
"collection": 1,
- "item_guid": "80858614-7988-494e-87f5-3a53d83ad889"
+ "item_guid": ""
}
},{
"model": "iconolab.ItemMetadata",
"pk": 1,
"fields": {
- "item": 1,
- "technics": "dessin",
- "joconde_ref": "1"
+ "item": 1
}
},{
"model": "iconolab.Image",
"pk": 1,
"fields": {
- "image_guid" : "26aec320-dcfe-4cbc-b912-6a6c13e8916e",
- "name" : "MIC.8.10.jpg",
- "media" : "uploads/MIC.8.10.jpg",
+ "image_guid" : "",
+ "name" : "",
+ "media" : "",
"item": 1,
"height": 2478,
"width": 3744,
- "created": "2016-03-23 14:13:44.913765+01"
+ "created": ""
}
},{
"model": "iconolab.ImageStats",
@@ -63,27 +61,25 @@
"pk": 2,
"fields": {
"collection": 1,
- "item_guid": "b597b937-0d4b-4f48-814d-6de3e308ac76"
+ "item_guid": ""
}
},{
"model": "iconolab.ItemMetadata",
"pk": 2,
"fields": {
- "item": 2,
- "technics": "dessin",
- "joconde_ref": "2"
+ "item": 2
}
},{
"model": "iconolab.Image",
"pk": 2,
"fields": {
- "image_guid" : "de734457-b600-4bed-bb60-a03d906b8e6e",
- "name" : "MIC.1.7.jpg",
- "media" : "uploads/MIC.1.7.jpg",
+ "image_guid" : "",
+ "name" : "",
+ "media" : "",
"item": 2,
"height": 4672,
"width": 6171,
- "created": "2016-03-23 14:13:44.913765+01"
+ "created": ""
}
},{
"model": "iconolab.ImageStats",
@@ -101,28 +97,25 @@
"pk": 3,
"fields": {
"collection": 1,
- "item_guid": "de301942-e60e-4e46-83c2-967bf3838f47"
+ "item_guid": ""
}
},{
"model": "iconolab.ItemMetadata",
"pk": 3,
"fields": {
- "item": 3,
- "technics": "dessin",
- "designation": "dessin format paysage",
- "joconde_ref": "12345"
+ "item": 3
}
},{
"model": "iconolab.Image",
"pk": 3,
"fields": {
- "image_guid" : "1a00cdd1-514e-4fd9-87ae-d9872b73143b",
- "name" : "MIC.1.18.jpg",
- "media" : "uploads/MIC.1.18.jpg",
+ "image_guid" : "",
+ "name" : "",
+ "media" : "",
"item": 3,
"height": 6208,
"width": 4704,
- "created": "2016-03-23 14:13:44.913765+01"
+ "created": ""
}
},{
"model": "iconolab.ImageStats",
@@ -139,35 +132,33 @@
"model": "iconolab.Collection",
"pk": 2,
"fields": {
- "name": "stdie",
- "verbose_name": "Musée Saint-Dié"
+ "name": "",
+ "verbose_name": ""
}
},{
"model": "iconolab.Item",
"pk": 4,
"fields": {
"collection": 2,
- "item_guid": "46ea6b45-caa4-47d5-bd0b-f1f22410b68e"
+ "item_guid": ""
}
},{
"model": "iconolab.ItemMetadata",
"pk": 4,
"fields": {
- "item": 4,
- "designation": "Médaille Athènes",
- "joconde_ref": "1234567"
+ "item": 4
}
},{
"model": "iconolab.Image",
"pk": 4,
"fields": {
- "image_guid" : "7384a6af-e87b-42b5-b24b-30cadb8adebc",
- "name" : "athene_medal1.jpg",
- "media" : "uploads/athene_medal1.jpg",
+ "image_guid" : "",
+ "name" : "",
+ "media" : "",
"item": 4,
"height": 704,
"width": 704,
- "created": "2016-03-23 14:13:44.913765+01"
+ "created": ""
}
},{
"model": "iconolab.ImageStats",
@@ -184,13 +175,13 @@
"model": "iconolab.Image",
"pk": 5,
"fields": {
- "image_guid" : "0ad0ae3e-06cf-4452-9d62-042471ddfdc3",
- "name" : "athene_medal2.jpg",
- "media" : "uploads/athene_medal2.jpg",
+ "image_guid" : "",
+ "name" : "",
+ "media" : "",
"item": 4,
"height": 704,
"width": 704,
- "created": "2016-03-23 14:13:44.913765+01"
+ "created": ""
}
},{
"model": "iconolab.ImageStats",
@@ -241,4 +232,4 @@
"label": "Désaccord"
}
}
-]
\ No newline at end of file
+]
--- a/src/iconolab/management/commands/importimages.py Thu Jun 07 16:07:52 2018 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,441 +0,0 @@
-# -*- coding: UTF-8 -*-
-from django.core.management.base import BaseCommand, CommandError
-from django.core.management import call_command
-from django.conf import settings
-from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder
-from PIL import Image as ImagePIL
-from sorl.thumbnail import get_thumbnail
-import os, csv, pprint, re, json, shutil, logging
-
-if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
- logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
-else:
- logger = logging.getLogger(__name__)
-
-class Command(BaseCommand):
- help = 'import images from a directory into the media folder and creates item and image objects'
-
- def add_arguments(self, parser):
- parser.add_argument('csv_path')
- parser.add_argument(
- '--jpeg-quality',
- dest='jpeg_quality',
- default=settings.IMG_JPG_DEFAULT_QUALITY,
- help='Jpeg default quality'
-
- )
- parser.add_argument(
- '--encoding',
- dest='encoding',
- default='utf-8',
- help='CSV file encoding'
-
- )
- parser.add_argument(
- '--collection-json',
- dest='collection_json',
- default=False,
- help='creates a new collection from a json file, must be an object with fields : '+ \
- '"name" (identifier), '+ \
- '"verbose_name" (proper title name), '+ \
- '"description" (description on homepage, html is supported), '+ \
- '"image" (image on homepages, must be "uploads/<imgname>"), '+ \
- '"height" and "width" (height and width of the image)',
- )
- parser.add_argument(
- '--collection-id',
- dest='collection_id',
- default=False,
- help='insert extracted data into the specified collection instead of trying to load a collection fixture',
- )
- parser.add_argument(
- '--metacategories-json',
- dest='metacategories_json',
- default=False,
- help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',
- )
- parser.add_argument(
- '--delimiter',
- dest='csv_delimiter',
- default=';',
- help='csv file delimiter'
- )
- parser.add_argument(
- '--no-jpg-conversion',
- dest='no-jpg-conversion',
- default=False,
- help='use this option if you only want the image copied and not converted'
- )
- parser.add_argument(
- '--img-filename-identifier',
- dest='img_filename_identifier',
- default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
- help='codename of the csv field we\'ll try to match to find the related image to a given object'
- )
- parser.add_argument(
- '--filename-regexp-prefix',
- dest='filename_regexp_prefix',
- default=r'.*',
- help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'
- )
- parser.add_argument(
- '--filename-regexp-suffix',
- dest='filename_regexp_suffix',
- default=r'[\.\-_].*',
- help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'
- )
- parser.add_argument(
- '--folders',
- dest='import_folders',
- default=False,
- action='store_const',
- const=True,
- help='option to create folders'
- )
- parser.add_argument(
- '--folders-regexp',
- dest='folders_regexp',
- default=False,
- help='regexp used to extract the folder name/number'
- )
- parser.add_argument(
- '--folders-metadata',
- dest='folders_metadata',
- default='INV',
- help='metadata from which to extract the folder name/number'
- )
- def handle(self, *args, **options):
- """
- Step-by-step for import:
-
- 1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files
- 1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic:
- * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas
- to which we associate a list of column header that will identified as that metadata
- * The cleaned_row_data list will associate the identifier with the actual value for its related column
- 2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated
- to each row for the filtered data.
- 3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary
- 4) For each item:
- We create the object in the database
- * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT
- We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image
-
- Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command.
- """
- try:
- print('# Logging with logger '+logger.name)
- logger.debug('# Initializing command with args: %r', options)
- # Check we have a collection to store data into:
- source_dir = os.path.dirname(os.path.realpath(options.get('csv_path')))
- print('# Checking collection args')
- if options.get('collection_json'):
- print('## Finding collection json data in '+source_dir)
- collection_json_path = os.path.join(source_dir, options.get('collection_json'))
- if not os.path.isfile(collection_json_path):
- print('### No '+options.get('collection_json')+'.json file was found in the source directory')
- raise ValueError('!!! Json file '+collection_json_path+' was not found !!!')
- try:
- with open(collection_json_path) as json_fixture_file:
- collection_data = json.loads(json_fixture_file.read())
- for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:
- if not key in collection_data.keys():
- print('!!! Json file '+collection_json_path+' has no '+key+' field !!!')
- raise ValueError()
- if not collection_data.get('name', ''):
- print('!!! Collection data key "name" is empty')
- raise ValueError()
- if Collection.objects.filter(name=collection_data.get('name')).exists():
- print('!!! A Collection with the provided name already exists!')
- raise ValueError()
- if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):
- print('!!! Collection data has an image but no height and width')
- raise ValueError()
- except ValueError as e:
- raise ValueError('!!! JSON Data is invalid. !!!')
- elif options.get('collection_id'):
- print('## Finding collection with id '+options.get('collection_id'))
- try:
- collection = Collection.objects.get(pk=options.get('collection_id'))
- except Collection.DoesNotExist:
- raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!')
- else:
- raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')
-
- if options.get('metacategories_json'):
- print('## Finding metacategories fixture json data in '+source_dir)
- metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json'))
- if not os.path.isfile(metacategories_json_path):
- print('### No '+options.get('metacategories_json')+'.json file was found in the source directory')
- raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!')
- with open(metacategories_json_path) as metacategories_json_file:
- metacategories_data = json.loads(metacategories_json_file.read())
- for metacategory in metacategories_data:
- if metacategory.get('label', None) is None:
- raise ValueError('!!! Metacategory without label !!!')
-
- if options['import_folders'] and not options['folders_regexp']:
- raise ValueError('!!! No regexp specified to extract folder name !!!')
-
- # We read the csv
- delimiter = options.get('csv_delimiter')
- if delimiter == '#9':
- delimiter = chr(9)
- if delimiter == '#29':
- delimiter = chr(29)
- if delimiter == '#30':
- delimiter = chr(30)
- if delimiter == '#31':
- delimiter = chr(31)
- csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter)
- print('# Extracting data from csv file and storing it in standardized format')
- # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
- cleaned_csv_data=[]
- duplicate_rows=[]
- for row in csvreader:
- cleaned_row_data = {}
- for key in settings.IMPORT_FIELDS_DICT.keys():
- cleaned_row_data[key] = ''
- for row_key in row.keys():
- if row_key in settings.IMPORT_FIELDS_DICT[key]:
- if key == 'INV':
- inv_number, _, _ = row[row_key].partition(';')
- cleaned_row_data[key] = inv_number.rstrip()
- else:
- cleaned_row_data[key] = row[row_key]
- break
- if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]:
- print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line")
- duplicate_rows.append(cleaned_row_data)
- else:
- cleaned_csv_data.append(cleaned_row_data)
- # Listing image files in csv directory
- image_list = [
- f for f in os.listdir(source_dir)
- if os.path.isfile(os.path.join(source_dir, f))
- and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png'))
- ] # Maybe check if image another way
- filtered_csv_data = []
- no_image_rows = []
- no_data_images = []
- assigned_images = []
- # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image
- for item in cleaned_csv_data:
- item['SRC_IMG_FILES'] = []
- has_image = False
- for image in image_list:
- img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix')
- if re.match(img_name_pattern, image):
- item['SRC_IMG_FILES'].append(image)
- assigned_images.append(image)
- has_image = True
- if has_image:
- filtered_csv_data.append(item)
- else:
- # We keep track of the entries that don't have any corresponding image
- no_image_rows.append(item)
- # We keep track of the images that don't have any corresponding entry
- for image in image_list:
- if image not in assigned_images:
- no_data_images.append(image)
-
- print('## found ' + str(len(filtered_csv_data))+' items with at least one image')
- print('# Importing data into Iconolab')
- if options.get('collection_json'):
- print('## Loading collection json')
- collection = Collection.objects.create(
- name = collection_data.get('name'),
- verbose_name = collection_data.get('verbose_name', ''),
- description = collection_data.get('description', ''),
- image = collection_data.get('image', ''),
- height = collection_data.get('height', 0),
- width = collection_data.get('width', 0),
- )
- if collection.image:
- collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))
- if not os.path.isfile(collection_image_path):
- print('### Moving collection image')
- _ , collection_image_name = os.path.split(collection_image_path)
- try:
- col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name))
- print('##### Generating or copying jpeg for '+collection_image_name)
- col_im.thumbnail(col_im.size)
- col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
- except Exception as e:
- print(e)
- if options.get('metacategories_json'):
- for metacategory in metacategories_data:
- MetaCategory.objects.create(
- collection = collection,
- label = metacategory.get('label'),
- triggers_notifications = metacategory.get('triggers_notifications', 0)
- )
- print('## Converting image and moving it to static dir, creating Image and Item objects')
- target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads')
- print('### Images will be stored in '+target_dir)
- for item in filtered_csv_data:
- print('#### Computing metadatas for item '+item['INV']+' (inv number)')
- if not item['INV']:
- print('#### No INV number, skipping')
- continue
- item_authors = item['AUTR']
- item_school = item['ECOLE']
- item_designation = ''
- if item.get('TITR', ''):
- item_designation = item['TITR']
- elif item.get('DENO', ''):
- item_designation = item['DENO']
- elif item.get('APPL', ''):
- item_designation = item['APPL']
- item_datation = ''
- if item.get('PERI', ''):
- item_datation = item['PERI']
- elif item.get('MILL', ''):
- item_datation = item['MILL']
- elif item.get('EPOQ', ''):
- item_datation = item['EPOQ']
- item_technics = item['TECH']
- item_field = item['DOM']
- item_measurements = item['DIMS']
- item_create_or_usage_location = item['LIEUX']
- item_discovery_context = item['DECV']
- item_conservation_location = item['LOCA']
- item_photo_credits = item['PHOT']
- item_inventory_number = item['INV']
- item_joconde_ref = item['REF']
- if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists():
- print('#### An item with '+item['INV']+' for inventory number, already exists in database in the import collection')
-
- if options['import_folders']:
-
- # Extract folder name from inventory number
- m = re.search(options['folders_regexp'], item[options['folders_metadata']])
- folder_id = m.group(1)
-
- if not Folder.objects.filter(original_id=folder_id).exists():
- print('#### Creating folder "'+folder_id+'"')
- folder = Folder.objects.create(
- collection = collection,
- name = 'Dossier '+folder_id,
- original_id = folder_id
- )
- else:
- print('#### Folder "'+folder_id+'" already exists')
- folder = Folder.objects.get(original_id=folder_id)
-
- item_metadata = ItemMetadata.objects.get(item__collection = collection, inventory_number = item_inventory_number)
- item = item_metadata.item
-
- item.folders.add(folder)
-
- else:
- print('#### Creating item '+item['INV']+' (inv number) in database')
- item_object = Item.objects.create(
- collection = collection
- )
- ItemMetadata.objects.create(
- item = item_object,
- authors = item_authors,
- school = item_school,
- designation = item_designation,
- datation = item_datation,
- technics = item_technics,
- measurements = item_measurements,
- create_or_usage_location = item_create_or_usage_location,
- field = item_field,
- discovery_context = item_discovery_context,
- conservation_location = item_conservation_location,
- photo_credits = item_photo_credits,
- inventory_number = item_inventory_number,
- joconde_ref = item_joconde_ref
- )
- print('#### Computing item image(s)')
- for image in item['SRC_IMG_FILES']:
- (image_name, ext) = os.path.splitext(image)
- if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS:
- print('##### Copying file '+str(image)+' without converting')
- image_path = os.path.join(target_dir, image)
- new_image_name = image
- shutil.copy(os.path.join(source_dir, image), target_dir)
- try:
- im = ImagePIL.open(os.path.join(target_dir, image))
- im_width, im_height = im.size
- except Exception as e:
- print(e)
- continue
- else:
- image_path = os.path.join(target_dir, image_name) + '.jpg'
- new_image_name = image_name+'.jpg'
- if os.path.isfile(image_path):
- print('##### A jpeg file already exists in target dir for '+ image)
- try:
- im = ImagePIL.open(image_path)
- im_width, im_height = im.size
- except Exception as e:
- print(e)
- continue
- else:
- jpeg_img_path = image_path
- try:
- im = ImagePIL.open(os.path.join(source_dir, image))
- print('##### Generating or copying jpeg for '+image)
- im.thumbnail(im.size)
- im.save(jpeg_img_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
- im_width, im_height = im.size
- except Exception as e:
- print(e)
- continue
- new_image = Image.objects.create(
- item = item_object,
- media = 'uploads/'+new_image_name,
- name = new_image_name,
- height = im_height,
- width = im_width
- )
- ImageStats.objects.create(
- image = new_image
- )
- print('### Generating thumbnails for item '+item['INV'])
- for image in item_object.images.all():
- for size in settings.PREGENERATE_THUMBNAILS_SIZES:
- print('#### Thumbnail for size '+size)
- get_thumbnail(image.media, size, crop=False)
-
- print('# All done!')
-
- logger.debug('# Recap for import command: ')
- print('# Images without data: ')
- logger.debug('## Checking images left without data')
- collection_image_file = os.path.split(str(collection.image))[1]
- if no_data_images and collection_image_file in no_data_images:
- no_data_images.remove(collection_image_file)
-
- if no_data_images:
- for image in no_data_images:
- logger.debug('### %r', image)
- print('## '+image)
- else:
- print('## Each image has one corresponding row!')
- logger.debug('### Each image has one corresponding row!')
- print('# CSV Items without image')
- logger.debug('## Checking csv rows left without image')
- if no_image_rows:
- for item in no_image_rows:
- logger.debug('### %r', item['INV'])
- print('## Inv number: '+item['INV'])
- else:
- print('## Each row found at least one corresponding image!')
- logger.debug('### Each row found at least one corresponding image!')
- print('# Duplicate rows in csv')
- logger.debug('## Checking duplicate rows in csv')
- if duplicate_rows:
- for item in no_image_rows:
- logger.debug('### %r: %r', options.get('img_filename_identifier'), item[options.get('img_filename_identifier')])
- print('## '+options.get('img_filename_identifier')+': '+item[options.get('img_filename_identifier')])
- else:
- print('## Each row found at least one corresponding image!')
- logger.debug('### Each row found at least one corresponding image!')
- except FileNotFoundError:
- print('!!! File '+options.get('csv_path')+' does not exist. !!!')
- except ValueError as e:
- print(str(e))
--- a/src/iconolab/management/commands/updatecollection.py Thu Jun 07 16:07:52 2018 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,137 +0,0 @@
-# -*- coding: UTF-8 -*-
-from django.core.management.base import BaseCommand, CommandError
-from django.core.management import call_command
-from django.conf import settings
-from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
-from sorl.thumbnail import get_thumbnail
-import os, csv, pprint, re, json, shutil
-
-class Command(BaseCommand):
- help = "import images from a directory into the media folder and creates item and image objects"
-
- def add_arguments(self, parser):
- parser.add_argument("csv_path")
- parser.add_argument(
- '--encoding',
- dest='encoding',
- default='utf-8',
- help='CSV file encoding'
-
- )
- parser.add_argument(
- '--collection-id',
- dest='collection_id',
- default=False,
- help='insert extracted data into the specified collection instead of trying to load a collection fixture',
- )
- parser.add_argument(
- '--delimiter',
- dest='csv_delimiter',
- default=';',
- help='csv file delimiter'
- )
-
- def handle(self, *args, **options):
- try:
- # Check we have a collection to store data into:
- source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
- print("# Checking collection args")
- if options.get("collection_id"):
- print("## Finding collection with id "+options.get("collection_id"))
- try:
- collection = Collection.objects.get(pk=options.get("collection_id"))
- except Collection.DoesNotExist:
- raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
- else:
- raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
-
- # We read the csv
- delimiter = options.get('csv_delimiter')
- if delimiter == "#9":
- delimiter = chr(9)
- if delimiter == "#29":
- delimiter = chr(29)
- if delimiter == "#30":
- delimiter = chr(30)
- if delimiter == "#31":
- delimiter = chr(31)
- csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
- print("# Extracting data from csv file and storing it in standardized format")
- # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
- cleaned_csv_data=[]
- for row in csvreader:
- cleaned_row_data = {}
- for key in settings.IMPORT_FIELDS_DICT.keys():
- cleaned_row_data[key] = ""
- for row_key in row.keys():
- if row_key in settings.IMPORT_FIELDS_DICT[key]:
- # Handling the multiple inventory numbers exports nonsense
- if key == "INV":
- inv_number, _, _ = row[row_key].partition(";")
- cleaned_row_data[key] = inv_number.rstrip()
- else:
- cleaned_row_data[key] = row[row_key]
- break
- cleaned_csv_data.append(cleaned_row_data)
- # Listing image files in csv directory
- filtered_csv_data = []
- # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the inventory number)
- for item in cleaned_csv_data:
- if item.get("INV", "") and ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).exists():
- filtered_csv_data.append(item)
- print("## found " + str(len(filtered_csv_data))+" items to update")
- print("# Updating data from Iconolab")
- for item in filtered_csv_data:
- item_metadatas = ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).first()
- if not item["INV"]:
- print("#### No INV number, skipping")
- continue
- item_authors = item["AUTR"]
- item_school = item["ECOLE"]
- item_designation = ""
- if item.get("TITR", ""):
- item_designation = item["TITR"]
- elif item.get("DENO", ""):
- item_designation = item["DENO"]
- elif item.get("APPL", ""):
- item_designation = item["APPL"]
- item_datation = ""
- if item.get("PERI", ""):
- item_datation = item["PERI"]
- elif item.get("MILL", ""):
- item_datation = item["MILL"]
- elif item.get("EPOQ", ""):
- item_datation = item["EPOQ"]
- item_technics = item["TECH"]
- item_field = item["DOM"]
- item_measurements = item["DIMS"]
- item_create_or_usage_location = item["LIEUX"]
- item_discovery_context = item["DECV"]
- item_conservation_location = item["LOCA"]
- item_photo_credits = item["PHOT"]
- item_inventory_number = item["INV"]
- item_joconde_ref = item["REF"]
- # Updating metadatas
- item_metadatas.authors = item_authors
- item_metadatas.school = item_school
- item_metadatas.designation = item_designation
- item_metadatas.datation = item_datation
- item_metadatas.technics = item_technics
- item_metadatas.measurements = item_measurements
- item_metadatas.create_or_usage_location = item_create_or_usage_location
- item_metadatas.field = item_field
- item_metadatas.discovery_context = item_discovery_context
- item_metadatas.conservation_location = item_conservation_location
- item_metadatas.photo_credits = item_photo_credits
- item_metadatas.joconde_ref = item_joconde_ref
- item_metadatas.save()
- print('### Generating thumbnails for item '+item['INV'])
- for image in item_metadatas.item.images.all():
- for size in settings.PREGENERATE_THUMBNAILS_SIZES:
- print('#### Thumbnail for size '+size)
- get_thumbnail(image.media, size, crop=False)
- print("# All done!")
- except FileNotFoundError:
- print("!!! File "+options.get("csv_path")+" does not exist. !!!")
- except ValueError as e:
- print(str(e))
--- a/src/iconolab/models.py Thu Jun 07 16:07:52 2018 +0200
+++ b/src/iconolab/models.py Tue Jun 12 13:33:18 2018 +0200
@@ -125,14 +125,12 @@
class ItemMetadata(models.Model):
"""
- Metadata object for the item class. Each field represents what we can
- import from the provided .csv files
-
+ Metadata object for the item class.
"""
item = models.OneToOneField('Item', related_name='metadatas', on_delete=models.PROTECT)
- natural_key = models.CharField(max_length=1024, default="", unique=True)
+ natural_key = models.CharField(max_length=1024, default="")
"""
JSON field integration
--- a/src/iconolab/templates/iconolab/home.html Thu Jun 07 16:07:52 2018 +0200
+++ b/src/iconolab/templates/iconolab/home.html Tue Jun 12 13:33:18 2018 +0200
@@ -8,11 +8,6 @@
{% block content %}
<div id="homepage-main">
- <div class="alert alert-warning" role="alert">
- <p><strong>Bienvenue sur iconolab. La plateforme
- est actuellement en cours de développement.</strong></p>
- <p>Nous entrons en phase d'expérimentation ouverte au grand public. Pour plus d'informations, prière de contacter <a href="mailto:{{ contact }}">{{ contact }}</a>.</p>
- </div>
<div class="row">
<div class="col-xs-12">
<h3>Les collections</h3>
--- a/src/iconolab/templates/iconolab/misc/help.html Thu Jun 07 16:07:52 2018 +0200
+++ b/src/iconolab/templates/iconolab/misc/help.html Tue Jun 12 13:33:18 2018 +0200
@@ -8,9 +8,7 @@
{% block content %}
<div class="row">
- <div class="alert alert-warning">
- <b>Avertissement</b> : Le contenu de cette page est temporaire et sera amené à être modifié dans les versions ultérieures de la plateforme.
- </div><div class="col-md-9">
+ <div class="col-md-9">
<h1>Le projet</h1>
<p class="text-justify">Iconolab est une plateforme contributive grâce à laquelle les utilisateurs peuvent explorer et annoter des <a href="/glossary/">collections (fonds)</a>.
<br><br>
--- a/src/iconolab/views/objects.py Thu Jun 07 16:07:52 2018 +0200
+++ b/src/iconolab/views/objects.py Tue Jun 12 13:33:18 2018 +0200
@@ -436,7 +436,7 @@
else:
page = 1
per_page = 10
- annotations_paginator = Paginator(image.annotations.all(), per_page)
+ annotations_paginator = Paginator(image.annotations.all().order_by('created'), per_page)
try:
annotations = annotations_paginator.page(page)
except PageNotAnInteger: