# HG changeset patch # User durandn # Date 1480340047 -3600 # Node ID c3ff8c5b28e6ff8499dfa3d113e9c3cc925c24b7 # Parent cd89d284079e5275a7e559647daed21099260b29 Pregeneration of thumbnail in import command + added "field" metadata for items + added updatecollection command diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/management/commands/generatethumbnails.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab/management/commands/generatethumbnails.py Mon Nov 28 14:34:07 2016 +0100 @@ -0,0 +1,39 @@ +# -*- coding: UTF-8 -*- +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from django.conf import settings +from iconolab.models import Collection, Item, ItemMetadata, MetaCategory +from sorl.thumbnail import get_thumbnail +import os, csv, pprint, re, json, shutil + +class Command(BaseCommand): + help = "import images from a directory into the media folder and creates item and image objects" + + def add_arguments(self, parser): + parser.add_argument( + '--collection-id', + dest='collection_id', + default=False, + help='insert extracted data into the specified collection instead of trying to load a collection fixture', + ) + + def handle(self, *args, **options): + try: + if options.get("collection_id"): + print("## Finding collection with id "+options.get("collection_id")) + try: + collection = Collection.objects.get(pk=options.get("collection_id")) + except Collection.DoesNotExist: + raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!") + else: + raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!") + for item in collection.items.all(): + print("# Generating thumbnails for item: "+str(item.metadatas.inventory_number)) + for image in item.images.all(): + print("## Processing image "+str(image.name)) + for size in settings.PREGENERATE_THUMBNAILS_SIZES: + print("### Size: "+size) + get_thumbnail(image.media, size, crop=False) + print("## All done!") + except ValueError as e: + print(str(e)) \ No newline at end of file diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/management/commands/importimages.py --- a/src/iconolab/management/commands/importimages.py Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/management/commands/importimages.py Mon Nov 28 14:34:07 2016 +0100 @@ -4,7 +4,8 @@ from django.conf import settings from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory from PIL import Image as ImagePIL -import os, csv, pprint, re, json +from sorl.thumbnail import get_thumbnail +import os, csv, pprint, re, json, shutil class Command(BaseCommand): help = "import images from a directory into the media folder and creates item and image objects" @@ -55,14 +56,30 @@ help='csv file delimiter' ) parser.add_argument( + '--no-jpg-conversion', + dest='no-jpg-conversion', + default=False, + help='use this option if you only want the image copied and not converted' + ) + parser.add_argument( '--img-filename-identifier', dest='img_filename_identifier', default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER, help='codename of the csv field we\'ll try to match to find the related image to a given object' ) - + parser.add_argument( + '--filename-regexp-prefix', + dest='filename_regexp_prefix', + default=r'.*', + help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*' + ) + parser.add_argument( + '--filename-regexp-suffix', + dest='filename_regexp_suffix', + default=r'[\.\-_].*', + help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*' + ) def handle(self, *args, **options): - pp = pprint.PrettyPrinter(indent=4) try: # Check we have a collection to store data into: source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) @@ -132,23 +149,42 @@ cleaned_row_data[key] = "" for row_key in row.keys(): if row_key in settings.IMPORT_FIELDS_DICT[key]: - cleaned_row_data[key] = row[row_key] + if key == "INV": + inv_number, _, _ = row[row_key].partition(";") + cleaned_row_data[key] = inv_number.rstrip() + else: + cleaned_row_data[key] = row[row_key] break cleaned_csv_data.append(cleaned_row_data) # Listing image files in csv directory - image_list = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f)) and not f.endswith(".csv")] + image_list = [ + f for f in os.listdir(source_dir) + if os.path.isfile(os.path.join(source_dir, f)) + and (f.endswith(".jpg") or f.endswith(".tif") or f.endswith(".bmp") or f.endswith(".png")) + ] # Maybe check if image another way filtered_csv_data = [] + no_image_rows = [] + no_data_images = [] + assigned_images = [] # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image for item in cleaned_csv_data: item["SRC_IMG_FILES"] = [] has_image = False for image in image_list: - img_name_pattern = r'.*'+re.escape(item[options.get("img_filename_identifier")])+r'[\.\-_].*' + img_name_pattern = options.get("filename_regexp_prefix")+re.escape(item[options.get("img_filename_identifier")])+options.get("filename_regexp_suffix") if re.match(img_name_pattern, image): item["SRC_IMG_FILES"].append(image) + assigned_images.append(image) has_image = True if has_image: filtered_csv_data.append(item) + else: + # We keep track of the entries that don't have any corresponding image + no_image_rows.append(item) + # We keep track of the images that don't have any corresponding entry + for image in image_list: + if image not in assigned_images: + no_data_images.append(image) print("## found " + str(len(filtered_csv_data))+" items with at least one image") print("# Importing data into Iconolab") @@ -206,6 +242,7 @@ elif item.get("EPOQ", ""): item_datation = item["EPOQ"] item_technics = item["TECH"] + item_field = item["DOM"] item_measurements = item["DIMS"] item_create_or_usage_location = item["LIEUX"] item_discovery_context = item["DECV"] @@ -214,9 +251,9 @@ item_inventory_number = item["INV"] item_joconde_ref = item["REF"] if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists(): - print("#### An item with "+item["INV"]+" for inventory number, already exists in databse in the import collection") + print("#### An item with "+item["INV"]+" for inventory number, already exists in database in the import collection") else: - print("#### Creating item "+item["INV"]+" (inv number) in databse") + print("#### Creating item "+item["INV"]+" (inv number) in database") item_object = Item.objects.create( collection = collection ) @@ -229,6 +266,7 @@ technics = item_technics, measurements = item_measurements, create_or_usage_location = item_create_or_usage_location, + field = item_field, discovery_context = item_discovery_context, conservation_location = item_conservation_location, photo_credits = item_photo_credits, @@ -238,37 +276,71 @@ print("#### Computing item image(s)") for image in item["SRC_IMG_FILES"]: (image_name, ext) = os.path.splitext(image) - image_path = os.path.join(target_dir, image_name) + ".jpg" - if os.path.isfile(image_path): - print("##### A jpeg file already exists in target dir for "+ image) + if options.get("no-jpg-conversion") or ext in settings.NO_IMG_CONVERSION_EXTS: + print("##### Copying file "+str(image)+" without converting") + image_path = os.path.join(target_dir, image) + new_image_name = image + shutil.copy(os.path.join(source_dir, image), target_dir) try: - im = ImagePIL.open(image_path) + im = ImagePIL.open(os.path.join(target_dir, image)) im_width, im_height = im.size except Exception as e: print(e) continue else: - jpeg_img_path = image_path - try: - im = ImagePIL.open(os.path.join(source_dir, image)) - print("##### Generating or copying jpeg for "+image) - im.thumbnail(im.size) - im.save(jpeg_img_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY)) - im_width, im_height = im.size - except Exception as e: - print(e) - continue + image_path = os.path.join(target_dir, image_name) + ".jpg" + new_image_name = image_name+".jpg" + if os.path.isfile(image_path): + print("##### A jpeg file already exists in target dir for "+ image) + try: + im = ImagePIL.open(image_path) + im_width, im_height = im.size + except Exception as e: + print(e) + continue + else: + jpeg_img_path = image_path + try: + im = ImagePIL.open(os.path.join(source_dir, image)) + print("##### Generating or copying jpeg for "+image) + im.thumbnail(im.size) + im.save(jpeg_img_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY)) + im_width, im_height = im.size + except Exception as e: + print(e) + continue new_image = Image.objects.create( item = item_object, - media = "uploads/"+image_name+".jpg", - name = image_name+".jpg", + media = "uploads/"+new_image_name, + name = new_image_name, height = im_height, width = im_width ) ImageStats.objects.create( image = new_image ) + print("### Generating thumbnails for item "+item["INV"]) + for image in item_object.images.all(): + for size in settings.PREGENERATE_THUMBNAILS_SIZES: + print("#### Thumbnail for size "+size) + get_thumbnail(image.media, size, crop=False) + print("# All done!") + print("# Images without data: ") + collection_image_file = os.path.split(str(collection.image))[1] + if no_data_images and collection_image_file in no_data_images: + no_data_images.remove(collection_image_file) + if no_data_images: + for image in no_data_images: + print("## "+image) + else: + print('## Each image has one corresponding row!') + print('# CSV Items without image') + if no_image_rows: + for item in no_image_rows: + print('## Inv number: '+item["INV"]) + else: + print('## Each row found at least one corresponding image!') except FileNotFoundError: print("!!! File "+options.get("csv_path")+" does not exist. !!!") except ValueError as e: diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/management/commands/updatecollection.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab/management/commands/updatecollection.py Mon Nov 28 14:34:07 2016 +0100 @@ -0,0 +1,136 @@ +# -*- coding: UTF-8 -*- +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from django.conf import settings +from iconolab.models import Collection, Item, ItemMetadata, MetaCategory +from sorl.thumbnail import get_thumbnail +import os, csv, pprint, re, json, shutil + +class Command(BaseCommand): + help = "import images from a directory into the media folder and creates item and image objects" + + def add_arguments(self, parser): + parser.add_argument("csv_path") + parser.add_argument( + '--encoding', + dest='encoding', + default='utf-8', + help='CSV file encoding' + + ) + parser.add_argument( + '--collection-id', + dest='collection_id', + default=False, + help='insert extracted data into the specified collection instead of trying to load a collection fixture', + ) + parser.add_argument( + '--delimiter', + dest='csv_delimiter', + default=';', + help='csv file delimiter' + ) + + def handle(self, *args, **options): + try: + # Check we have a collection to store data into: + source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) + print("# Checking collection args") + if options.get("collection_id"): + print("## Finding collection with id "+options.get("collection_id")) + try: + collection = Collection.objects.get(pk=options.get("collection_id")) + except Collection.DoesNotExist: + raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!") + else: + raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!") + + # We read the csv + delimiter = options.get('csv_delimiter') + if delimiter == "#9": + delimiter = chr(9) + if delimiter == "#29": + delimiter = chr(29) + if delimiter == "#30": + delimiter = chr(30) + if delimiter == "#31": + delimiter = chr(31) + csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter) + print("# Extracting data from csv file and storing it in standardized format") + # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT + cleaned_csv_data=[] + for row in csvreader: + cleaned_row_data = {} + for key in settings.IMPORT_FIELDS_DICT.keys(): + cleaned_row_data[key] = "" + for row_key in row.keys(): + if row_key in settings.IMPORT_FIELDS_DICT[key]: + # Handling the multiple inventory numbers exports nonsense + if key == "INV": + inv_number, _, _ = row[row_key].partition(";") + cleaned_row_data[key] = inv_number.rstrip() + else: + cleaned_row_data[key] = row[row_key] + break + cleaned_csv_data.append(cleaned_row_data) + # Listing image files in csv directory + filtered_csv_data = [] + # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the inventory number) + for item in cleaned_csv_data: + if item.get("INV", "") and ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).exists(): + filtered_csv_data.append(item) + print("## found " + str(len(filtered_csv_data))+" items to update") + print("# Updating data from Iconolab") + for item in filtered_csv_data: + item_metadatas = ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).first() + if not item["INV"]: + print("#### No INV number, skipping") + continue + item_authors = item["AUTR"] + item_school = item["ECOLE"] + item_designation = "" + if item.get("TITR", ""): + item_designation = item["TITR"] + elif item.get("DENO", ""): + item_designation = item["DENO"] + elif item.get("APPL", ""): + item_designation = item["APPL"] + item_datation = "" + if item.get("PERI", ""): + item_datation = item["PERI"] + elif item.get("MILL", ""): + item_datation = item["MILL"] + elif item.get("EPOQ", ""): + item_datation = item["EPOQ"] + item_technics = item["TECH"] + item_field = item["DOM"] + item_measurements = item["DIMS"] + item_create_or_usage_location = item["LIEUX"] + item_discovery_context = item["DECV"] + item_conservation_location = item["LOCA"] + item_photo_credits = item["PHOT"] + item_inventory_number = item["INV"] + item_joconde_ref = item["REF"] + # Updating metadatas + item_metadatas.authors = item_authors + item_metadatas.school = item_school + item_metadatas.designation = item_designation + item_metadatas.datation = item_datation + item_metadatas.technics = item_technics + item_metadatas.measurements = item_measurements + item_metadatas.create_or_usage_location = item_create_or_usage_location + item_metadatas.field = item_field + item_metadatas.discovery_context = item_discovery_context + item_metadatas.conservation_location = item_conservation_location + item_metadatas.photo_credits = item_photo_credits + item_metadatas.joconde_ref = item_joconde_ref + item_metadatas.save() + print("### Generating thumbnails for item") + for image in item_metadatas.item.images: + for size in settings.PREGENERATE_THUMBNAILS_SIZES: + get_thumbnail(image.media, size, crop=False) + print("# All done!") + except FileNotFoundError: + print("!!! File "+options.get("csv_path")+" does not exist. !!!") + except ValueError as e: + print(str(e)) diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/models.py --- a/src/iconolab/models.py Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/models.py Mon Nov 28 14:34:07 2016 +0100 @@ -61,6 +61,7 @@ item = models.OneToOneField('Item', related_name='metadatas') authors = models.CharField(max_length=255, default="") school = models.CharField(max_length=255, default="") + field = models.CharField(max_length=255, default="") designation = models.CharField(max_length=255, default="") datation = models.CharField(max_length=255, default="") technics = models.CharField(max_length=255, default="") diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/settings/__init__.py --- a/src/iconolab/settings/__init__.py Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/settings/__init__.py Mon Nov 28 14:34:07 2016 +0100 @@ -149,6 +149,7 @@ "ECOLE": [], "TITR": ["Titre"], "DENO": [], + "DOM": ["Domaine"], "APPL": [], "PERI": ["Période"], "MILL": [], @@ -160,7 +161,7 @@ "DECV": [], "LOCA": ["Localisation"], "PHOT": ["Photo"], - "INV": ["No inventaire",], + "INV": ["No inventaire"], "REF": ["REFERENCE"], } @@ -174,3 +175,19 @@ RELEVANT_TAGS_MIN_SCORE = 3 ACCURATE_TAGS_MIN_SCORE = 3 + +# The different thumbnail sizes that we want to pre-generate when importing or when updating collections using commands +# This allows to pre-calculate thumbnails for media-heavy pages such as collection_home +PREGENERATE_THUMBNAILS_SIZES = [#{ +# # Thumbnails that will always be generated without taking image format into account +# "all": [], +# # Thumbnails for images in portrait format +# "portrait": [], +# # Thumbnails for images in landscape format +# "landscape": [], +# } + + # item_images_preview.html + "250x250", + "100x100", +] diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/settings/dev.py.tmpl --- a/src/iconolab/settings/dev.py.tmpl Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/settings/dev.py.tmpl Mon Nov 28 14:34:07 2016 +0100 @@ -239,6 +239,7 @@ "ECOLE": [], "TITR": ["Titre"], "DENO": [], + "DOM": ["Domaine"], "APPL": [], "PERI": ["Période"], "MILL": [], diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/static/iconolab/css/iconolab.css --- a/src/iconolab/static/iconolab/css/iconolab.css Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/static/iconolab/css/iconolab.css Mon Nov 28 14:34:07 2016 +0100 @@ -187,7 +187,7 @@ li.image-list-li{ margin-bottom: 5px; width: 370px; - height: 350px; + height: 400px; vertical-align:middle; padding:5px; } diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/templates/iconolab/detail_item.html --- a/src/iconolab/templates/iconolab/detail_item.html Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/templates/iconolab/detail_item.html Mon Nov 28 14:34:07 2016 +0100 @@ -23,6 +23,7 @@
{% if item.metadatas.designation %}

Désignation : {{item.metadatas.designation}}

{% endif %} {% if item.metadatas.authors %}

Auteur(s) : {{item.metadatas.designation}}

{% endif %} + {% if item.metadatas.field %}

Domaine : {{item.metadatas.field}}

{% endif %} {% if item.metadatas.conservation_location %}

Conservé à : {{item.metadatas.conservation_location}}

{% endif %} {% if item.metadatas.datation %}

Datation : {{item.metadatas.datation}}

{% endif %} {% if item.metadatas.technics %}
Techniques : {{item.metadatas.technics}}
{% endif %} diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/templates/iconolab/user_home.html --- a/src/iconolab/templates/iconolab/user_home.html Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/templates/iconolab/user_home.html Mon Nov 28 14:34:07 2016 +0100 @@ -120,8 +120,9 @@ {% if profile_user.profile.managed_collections %}
+

Administration de Fonds:

{% for collection in profile_user.profile.managed_collections.all %} -

Administration du Fonds {{collection.verbose_name}}

+

{{collection.verbose_name}}

Tableau de bord
diff -r cd89d284079e -r c3ff8c5b28e6 src/iconolab/templates/partials/item_images_preview.html --- a/src/iconolab/templates/partials/item_images_preview.html Tue Nov 15 15:48:19 2016 +0100 +++ b/src/iconolab/templates/partials/item_images_preview.html Mon Nov 28 14:34:07 2016 +0100 @@ -5,7 +5,7 @@ {% with item.images_sorted_by_name.first as main_image %} {% if main_image.wh_ratio < 1.3 %}
- {% thumbnail main_image.media "175x300" crop=False as im %} + {% thumbnail main_image.media "250x250" crop=False as im %}
@@ -15,7 +15,7 @@
{% for secondary_image in item.images.all %} {% if secondary_image != main_image and forloop.counter <= 4 %} - {% thumbnail secondary_image.media "165x90" crop=False as im %} + {% thumbnail secondary_image.media "100x100" crop=False as im %}
@@ -28,7 +28,7 @@
{% else %}
- {% thumbnail main_image.media "350x150" crop=False as im %} + {% thumbnail main_image.media "250x250" crop=False as im %}
@@ -38,7 +38,7 @@
{% for secondary_image in item.images.all %} {% if secondary_image != main_image and forloop.counter <= 3 %} - {% thumbnail secondary_image.media "140x140" crop=False as im %} + {% thumbnail secondary_image.media "100x100" crop=False as im %}
@@ -54,7 +54,7 @@ {% else %} {% with item.images.first as main_image %}
- {% thumbnail main_image.media "350x210" crop=False as im %} + {% thumbnail main_image.media "250x250" crop=False as im %}