# HG changeset patch # User Alexandre Segura # Date 1492185264 -7200 # Node ID 4f6d30adf06a1309d45cbd83a233691232614f1b # Parent ce9947e45d042bdfa89903d0dcdee4a439a9fc88 Manage folder import for existing collections. diff -r ce9947e45d04 -r 4f6d30adf06a src/iconolab/management/commands/importimages.py --- a/src/iconolab/management/commands/importimages.py Fri Apr 14 16:35:42 2017 +0200 +++ b/src/iconolab/management/commands/importimages.py Fri Apr 14 17:54:24 2017 +0200 @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.core.management import call_command from django.conf import settings -from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory +from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder from PIL import Image as ImagePIL from sorl.thumbnail import get_thumbnail import os, csv, pprint, re, json, shutil, logging @@ -87,20 +87,20 @@ def handle(self, *args, **options): """ Step-by-step for import: - + 1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files 1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic: - * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas + * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas to which we associate a list of column header that will identified as that metadata - * The cleaned_row_data list will associate the identifier with the actual value for its related column + * The cleaned_row_data list will associate the identifier with the actual value for its related column 2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated - to each row for the filtered data. + to each row for the filtered data. 3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary 4) For each item: We create the object in the database * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image - + Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command. """ try: @@ -188,8 +188,8 @@ cleaned_csv_data.append(cleaned_row_data) # Listing image files in csv directory image_list = [ - f for f in os.listdir(source_dir) - if os.path.isfile(os.path.join(source_dir, f)) + f for f in os.listdir(source_dir) + if os.path.isfile(os.path.join(source_dir, f)) and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png')) ] # Maybe check if image another way filtered_csv_data = [] @@ -200,7 +200,7 @@ for item in cleaned_csv_data: item['SRC_IMG_FILES'] = [] has_image = False - for image in image_list: + for image in image_list: img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix') if re.match(img_name_pattern, image): item['SRC_IMG_FILES'].append(image) @@ -208,14 +208,14 @@ has_image = True if has_image: filtered_csv_data.append(item) - else: + else: # We keep track of the entries that don't have any corresponding image no_image_rows.append(item) # We keep track of the images that don't have any corresponding entry for image in image_list: if image not in assigned_images: no_data_images.append(image) - + print('## found ' + str(len(filtered_csv_data))+' items with at least one image') print('# Importing data into Iconolab') if options.get('collection_json'): @@ -282,6 +282,27 @@ item_joconde_ref = item['REF'] if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists(): print('#### An item with '+item['INV']+' for inventory number, already exists in database in the import collection') + + # Extract folder name from inventory number + m = re.search('^.*\.(.*)\..*$', item['INV']) + folder_id = m.group(1) + + if not Folder.objects.filter(original_id=folder_id).exists(): + print('#### Creating folder "'+folder_id+'"') + folder = Folder.objects.create( + collection = collection, + name = 'Dossier '+folder_id, + original_id = folder_id + ) + else: + print('#### Folder "'+folder_id+'" already exists') + folder = Folder.objects.get(original_id=folder_id) + + item_metadata = ItemMetadata.objects.get(item__collection = collection, inventory_number = item_inventory_number) + item = item_metadata.item + + item.folders.add(folder) + else: print('#### Creating item '+item['INV']+' (inv number) in database') item_object = Item.objects.create( @@ -354,20 +375,20 @@ for size in settings.PREGENERATE_THUMBNAILS_SIZES: print('#### Thumbnail for size '+size) get_thumbnail(image.media, size, crop=False) - + print('# All done!') - + logger.debug('# Recap for import command: ') print('# Images without data: ') logger.debug('## Checking images left without data') collection_image_file = os.path.split(str(collection.image))[1] if no_data_images and collection_image_file in no_data_images: no_data_images.remove(collection_image_file) - + if no_data_images: for image in no_data_images: logger.debug('### %r', image) - print('## '+image) + print('## '+image) else: print('## Each image has one corresponding row!') logger.debug('### Each image has one corresponding row!')