Pregeneration of thumbnail in import command + added "field" metadata for items + added updatecollection command
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab/management/commands/generatethumbnails.py Mon Nov 28 14:34:07 2016 +0100
@@ -0,0 +1,39 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil
+
+class Command(BaseCommand):
+ help = "import images from a directory into the media folder and creates item and image objects"
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '--collection-id',
+ dest='collection_id',
+ default=False,
+ help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+ )
+
+ def handle(self, *args, **options):
+ try:
+ if options.get("collection_id"):
+ print("## Finding collection with id "+options.get("collection_id"))
+ try:
+ collection = Collection.objects.get(pk=options.get("collection_id"))
+ except Collection.DoesNotExist:
+ raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
+ else:
+ raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
+ for item in collection.items.all():
+ print("# Generating thumbnails for item: "+str(item.metadatas.inventory_number))
+ for image in item.images.all():
+ print("## Processing image "+str(image.name))
+ for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+ print("### Size: "+size)
+ get_thumbnail(image.media, size, crop=False)
+ print("## All done!")
+ except ValueError as e:
+ print(str(e))
\ No newline at end of file
--- a/src/iconolab/management/commands/importimages.py Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/management/commands/importimages.py Mon Nov 28 14:34:07 2016 +0100
@@ -4,7 +4,8 @@
from django.conf import settings
from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory
from PIL import Image as ImagePIL
-import os, csv, pprint, re, json
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil
class Command(BaseCommand):
help = "import images from a directory into the media folder and creates item and image objects"
@@ -55,14 +56,30 @@
help='csv file delimiter'
)
parser.add_argument(
+ '--no-jpg-conversion',
+ dest='no-jpg-conversion',
+ default=False,
+ help='use this option if you only want the image copied and not converted'
+ )
+ parser.add_argument(
'--img-filename-identifier',
dest='img_filename_identifier',
default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
help='codename of the csv field we\'ll try to match to find the related image to a given object'
)
-
+ parser.add_argument(
+ '--filename-regexp-prefix',
+ dest='filename_regexp_prefix',
+ default=r'.*',
+ help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'
+ )
+ parser.add_argument(
+ '--filename-regexp-suffix',
+ dest='filename_regexp_suffix',
+ default=r'[\.\-_].*',
+ help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'
+ )
def handle(self, *args, **options):
- pp = pprint.PrettyPrinter(indent=4)
try:
# Check we have a collection to store data into:
source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
@@ -132,23 +149,42 @@
cleaned_row_data[key] = ""
for row_key in row.keys():
if row_key in settings.IMPORT_FIELDS_DICT[key]:
- cleaned_row_data[key] = row[row_key]
+ if key == "INV":
+ inv_number, _, _ = row[row_key].partition(";")
+ cleaned_row_data[key] = inv_number.rstrip()
+ else:
+ cleaned_row_data[key] = row[row_key]
break
cleaned_csv_data.append(cleaned_row_data)
# Listing image files in csv directory
- image_list = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f)) and not f.endswith(".csv")]
+ image_list = [
+ f for f in os.listdir(source_dir)
+ if os.path.isfile(os.path.join(source_dir, f))
+ and (f.endswith(".jpg") or f.endswith(".tif") or f.endswith(".bmp") or f.endswith(".png"))
+ ] # Maybe check if image another way
filtered_csv_data = []
+ no_image_rows = []
+ no_data_images = []
+ assigned_images = []
# Now we trim the cleaned_csv_data dict to keep only entries that have at least one image
for item in cleaned_csv_data:
item["SRC_IMG_FILES"] = []
has_image = False
for image in image_list:
- img_name_pattern = r'.*'+re.escape(item[options.get("img_filename_identifier")])+r'[\.\-_].*'
+ img_name_pattern = options.get("filename_regexp_prefix")+re.escape(item[options.get("img_filename_identifier")])+options.get("filename_regexp_suffix")
if re.match(img_name_pattern, image):
item["SRC_IMG_FILES"].append(image)
+ assigned_images.append(image)
has_image = True
if has_image:
filtered_csv_data.append(item)
+ else:
+ # We keep track of the entries that don't have any corresponding image
+ no_image_rows.append(item)
+ # We keep track of the images that don't have any corresponding entry
+ for image in image_list:
+ if image not in assigned_images:
+ no_data_images.append(image)
print("## found " + str(len(filtered_csv_data))+" items with at least one image")
print("# Importing data into Iconolab")
@@ -206,6 +242,7 @@
elif item.get("EPOQ", ""):
item_datation = item["EPOQ"]
item_technics = item["TECH"]
+ item_field = item["DOM"]
item_measurements = item["DIMS"]
item_create_or_usage_location = item["LIEUX"]
item_discovery_context = item["DECV"]
@@ -214,9 +251,9 @@
item_inventory_number = item["INV"]
item_joconde_ref = item["REF"]
if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists():
- print("#### An item with "+item["INV"]+" for inventory number, already exists in databse in the import collection")
+ print("#### An item with "+item["INV"]+" for inventory number, already exists in database in the import collection")
else:
- print("#### Creating item "+item["INV"]+" (inv number) in databse")
+ print("#### Creating item "+item["INV"]+" (inv number) in database")
item_object = Item.objects.create(
collection = collection
)
@@ -229,6 +266,7 @@
technics = item_technics,
measurements = item_measurements,
create_or_usage_location = item_create_or_usage_location,
+ field = item_field,
discovery_context = item_discovery_context,
conservation_location = item_conservation_location,
photo_credits = item_photo_credits,
@@ -238,37 +276,71 @@
print("#### Computing item image(s)")
for image in item["SRC_IMG_FILES"]:
(image_name, ext) = os.path.splitext(image)
- image_path = os.path.join(target_dir, image_name) + ".jpg"
- if os.path.isfile(image_path):
- print("##### A jpeg file already exists in target dir for "+ image)
+ if options.get("no-jpg-conversion") or ext in settings.NO_IMG_CONVERSION_EXTS:
+ print("##### Copying file "+str(image)+" without converting")
+ image_path = os.path.join(target_dir, image)
+ new_image_name = image
+ shutil.copy(os.path.join(source_dir, image), target_dir)
try:
- im = ImagePIL.open(image_path)
+ im = ImagePIL.open(os.path.join(target_dir, image))
im_width, im_height = im.size
except Exception as e:
print(e)
continue
else:
- jpeg_img_path = image_path
- try:
- im = ImagePIL.open(os.path.join(source_dir, image))
- print("##### Generating or copying jpeg for "+image)
- im.thumbnail(im.size)
- im.save(jpeg_img_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY))
- im_width, im_height = im.size
- except Exception as e:
- print(e)
- continue
+ image_path = os.path.join(target_dir, image_name) + ".jpg"
+ new_image_name = image_name+".jpg"
+ if os.path.isfile(image_path):
+ print("##### A jpeg file already exists in target dir for "+ image)
+ try:
+ im = ImagePIL.open(image_path)
+ im_width, im_height = im.size
+ except Exception as e:
+ print(e)
+ continue
+ else:
+ jpeg_img_path = image_path
+ try:
+ im = ImagePIL.open(os.path.join(source_dir, image))
+ print("##### Generating or copying jpeg for "+image)
+ im.thumbnail(im.size)
+ im.save(jpeg_img_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY))
+ im_width, im_height = im.size
+ except Exception as e:
+ print(e)
+ continue
new_image = Image.objects.create(
item = item_object,
- media = "uploads/"+image_name+".jpg",
- name = image_name+".jpg",
+ media = "uploads/"+new_image_name,
+ name = new_image_name,
height = im_height,
width = im_width
)
ImageStats.objects.create(
image = new_image
)
+ print("### Generating thumbnails for item "+item["INV"])
+ for image in item_object.images.all():
+ for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+ print("#### Thumbnail for size "+size)
+ get_thumbnail(image.media, size, crop=False)
+
print("# All done!")
+ print("# Images without data: ")
+ collection_image_file = os.path.split(str(collection.image))[1]
+ if no_data_images and collection_image_file in no_data_images:
+ no_data_images.remove(collection_image_file)
+ if no_data_images:
+ for image in no_data_images:
+ print("## "+image)
+ else:
+ print('## Each image has one corresponding row!')
+ print('# CSV Items without image')
+ if no_image_rows:
+ for item in no_image_rows:
+ print('## Inv number: '+item["INV"])
+ else:
+ print('## Each row found at least one corresponding image!')
except FileNotFoundError:
print("!!! File "+options.get("csv_path")+" does not exist. !!!")
except ValueError as e:
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/iconolab/management/commands/updatecollection.py Mon Nov 28 14:34:07 2016 +0100
@@ -0,0 +1,136 @@
+# -*- coding: UTF-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+from django.conf import settings
+from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
+from sorl.thumbnail import get_thumbnail
+import os, csv, pprint, re, json, shutil
+
+class Command(BaseCommand):
+ help = "import images from a directory into the media folder and creates item and image objects"
+
+ def add_arguments(self, parser):
+ parser.add_argument("csv_path")
+ parser.add_argument(
+ '--encoding',
+ dest='encoding',
+ default='utf-8',
+ help='CSV file encoding'
+
+ )
+ parser.add_argument(
+ '--collection-id',
+ dest='collection_id',
+ default=False,
+ help='insert extracted data into the specified collection instead of trying to load a collection fixture',
+ )
+ parser.add_argument(
+ '--delimiter',
+ dest='csv_delimiter',
+ default=';',
+ help='csv file delimiter'
+ )
+
+ def handle(self, *args, **options):
+ try:
+ # Check we have a collection to store data into:
+ source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
+ print("# Checking collection args")
+ if options.get("collection_id"):
+ print("## Finding collection with id "+options.get("collection_id"))
+ try:
+ collection = Collection.objects.get(pk=options.get("collection_id"))
+ except Collection.DoesNotExist:
+ raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
+ else:
+ raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
+
+ # We read the csv
+ delimiter = options.get('csv_delimiter')
+ if delimiter == "#9":
+ delimiter = chr(9)
+ if delimiter == "#29":
+ delimiter = chr(29)
+ if delimiter == "#30":
+ delimiter = chr(30)
+ if delimiter == "#31":
+ delimiter = chr(31)
+ csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
+ print("# Extracting data from csv file and storing it in standardized format")
+ # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
+ cleaned_csv_data=[]
+ for row in csvreader:
+ cleaned_row_data = {}
+ for key in settings.IMPORT_FIELDS_DICT.keys():
+ cleaned_row_data[key] = ""
+ for row_key in row.keys():
+ if row_key in settings.IMPORT_FIELDS_DICT[key]:
+ # Handling the multiple inventory numbers exports nonsense
+ if key == "INV":
+ inv_number, _, _ = row[row_key].partition(";")
+ cleaned_row_data[key] = inv_number.rstrip()
+ else:
+ cleaned_row_data[key] = row[row_key]
+ break
+ cleaned_csv_data.append(cleaned_row_data)
+ # Listing image files in csv directory
+ filtered_csv_data = []
+ # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the inventory number)
+ for item in cleaned_csv_data:
+ if item.get("INV", "") and ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).exists():
+ filtered_csv_data.append(item)
+ print("## found " + str(len(filtered_csv_data))+" items to update")
+ print("# Updating data from Iconolab")
+ for item in filtered_csv_data:
+ item_metadatas = ItemMetadata.objects.filter(item__collection = collection, inventory_number=item.get("INV")).first()
+ if not item["INV"]:
+ print("#### No INV number, skipping")
+ continue
+ item_authors = item["AUTR"]
+ item_school = item["ECOLE"]
+ item_designation = ""
+ if item.get("TITR", ""):
+ item_designation = item["TITR"]
+ elif item.get("DENO", ""):
+ item_designation = item["DENO"]
+ elif item.get("APPL", ""):
+ item_designation = item["APPL"]
+ item_datation = ""
+ if item.get("PERI", ""):
+ item_datation = item["PERI"]
+ elif item.get("MILL", ""):
+ item_datation = item["MILL"]
+ elif item.get("EPOQ", ""):
+ item_datation = item["EPOQ"]
+ item_technics = item["TECH"]
+ item_field = item["DOM"]
+ item_measurements = item["DIMS"]
+ item_create_or_usage_location = item["LIEUX"]
+ item_discovery_context = item["DECV"]
+ item_conservation_location = item["LOCA"]
+ item_photo_credits = item["PHOT"]
+ item_inventory_number = item["INV"]
+ item_joconde_ref = item["REF"]
+ # Updating metadatas
+ item_metadatas.authors = item_authors
+ item_metadatas.school = item_school
+ item_metadatas.designation = item_designation
+ item_metadatas.datation = item_datation
+ item_metadatas.technics = item_technics
+ item_metadatas.measurements = item_measurements
+ item_metadatas.create_or_usage_location = item_create_or_usage_location
+ item_metadatas.field = item_field
+ item_metadatas.discovery_context = item_discovery_context
+ item_metadatas.conservation_location = item_conservation_location
+ item_metadatas.photo_credits = item_photo_credits
+ item_metadatas.joconde_ref = item_joconde_ref
+ item_metadatas.save()
+ print("### Generating thumbnails for item")
+ for image in item_metadatas.item.images:
+ for size in settings.PREGENERATE_THUMBNAILS_SIZES:
+ get_thumbnail(image.media, size, crop=False)
+ print("# All done!")
+ except FileNotFoundError:
+ print("!!! File "+options.get("csv_path")+" does not exist. !!!")
+ except ValueError as e:
+ print(str(e))
--- a/src/iconolab/models.py Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/models.py Mon Nov 28 14:34:07 2016 +0100
@@ -61,6 +61,7 @@
item = models.OneToOneField('Item', related_name='metadatas')
authors = models.CharField(max_length=255, default="")
school = models.CharField(max_length=255, default="")
+ field = models.CharField(max_length=255, default="")
designation = models.CharField(max_length=255, default="")
datation = models.CharField(max_length=255, default="")
technics = models.CharField(max_length=255, default="")
--- a/src/iconolab/settings/__init__.py Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/settings/__init__.py Mon Nov 28 14:34:07 2016 +0100
@@ -149,6 +149,7 @@
"ECOLE": [],
"TITR": ["Titre"],
"DENO": [],
+ "DOM": ["Domaine"],
"APPL": [],
"PERI": ["Période"],
"MILL": [],
@@ -160,7 +161,7 @@
"DECV": [],
"LOCA": ["Localisation"],
"PHOT": ["Photo"],
- "INV": ["No inventaire",],
+ "INV": ["No inventaire"],
"REF": ["REFERENCE"],
}
@@ -174,3 +175,19 @@
RELEVANT_TAGS_MIN_SCORE = 3
ACCURATE_TAGS_MIN_SCORE = 3
+
+# The different thumbnail sizes that we want to pre-generate when importing or when updating collections using commands
+# This allows to pre-calculate thumbnails for media-heavy pages such as collection_home
+PREGENERATE_THUMBNAILS_SIZES = [#{
+# # Thumbnails that will always be generated without taking image format into account
+# "all": [],
+# # Thumbnails for images in portrait format
+# "portrait": [],
+# # Thumbnails for images in landscape format
+# "landscape": [],
+# }
+
+ # item_images_preview.html
+ "250x250",
+ "100x100",
+]
--- a/src/iconolab/settings/dev.py.tmpl Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/settings/dev.py.tmpl Mon Nov 28 14:34:07 2016 +0100
@@ -239,6 +239,7 @@
"ECOLE": [],
"TITR": ["Titre"],
"DENO": [],
+ "DOM": ["Domaine"],
"APPL": [],
"PERI": ["Période"],
"MILL": [],
--- a/src/iconolab/static/iconolab/css/iconolab.css Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/static/iconolab/css/iconolab.css Mon Nov 28 14:34:07 2016 +0100
@@ -187,7 +187,7 @@
li.image-list-li{
margin-bottom: 5px;
width: 370px;
- height: 350px;
+ height: 400px;
vertical-align:middle;
padding:5px;
}
--- a/src/iconolab/templates/iconolab/detail_item.html Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/templates/iconolab/detail_item.html Mon Nov 28 14:34:07 2016 +0100
@@ -23,6 +23,7 @@
<div class="col-md-6">
{% if item.metadatas.designation %}<h3>Désignation : <small>{{item.metadatas.designation}}</small></h3>{% endif %}
{% if item.metadatas.authors %}<h4>Auteur(s) : <small>{{item.metadatas.designation}}</small></h4>{% endif %}
+ {% if item.metadatas.field %}<h4>Domaine : <small>{{item.metadatas.field}}</small></h4>{% endif %}
{% if item.metadatas.conservation_location %}<h4>Conservé à : <small>{{item.metadatas.conservation_location}}</small></h4>{% endif %}
{% if item.metadatas.datation %}<h4>Datation : <small>{{item.metadatas.datation}}</small></h4>{% endif %}
{% if item.metadatas.technics %}<h5>Techniques : <small>{{item.metadatas.technics}}</small></h5>{% endif %}
--- a/src/iconolab/templates/iconolab/user_home.html Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/templates/iconolab/user_home.html Mon Nov 28 14:34:07 2016 +0100
@@ -120,8 +120,9 @@
{% if profile_user.profile.managed_collections %}
<div class="col-md-12">
<div class="panel panel-default text-center" style="padding: 10px;">
+ <h2><small>Administration de Fonds:</small></h2>
{% for collection in profile_user.profile.managed_collections.all %}
- <h3><small>Administration du Fonds</small> {{collection.verbose_name}}</h3>
+ <h3>{{collection.verbose_name}}</h3>
<div>
<a href="{% url 'user_admin_panel' profile_user.id collection.name %}" class="btn btn-default text-center">Tableau de bord</a>
</div>
--- a/src/iconolab/templates/partials/item_images_preview.html Tue Nov 15 15:48:19 2016 +0100
+++ b/src/iconolab/templates/partials/item_images_preview.html Mon Nov 28 14:34:07 2016 +0100
@@ -5,7 +5,7 @@
{% with item.images_sorted_by_name.first as main_image %}
{% if main_image.wh_ratio < 1.3 %}
<div class="item-links-portrait text-center" style="display:inline-block; margin: 10px 0px;">
- {% thumbnail main_image.media "175x300" crop=False as im %}
+ {% thumbnail main_image.media "250x250" crop=False as im %}
<div class="main-image" style="display:inline-block">
<a href="{% url 'item_detail' item.collection.name item.item_guid %}?show={{main_image.image_guid}}">
<img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
@@ -15,7 +15,7 @@
<div class="secondary-images-container secondary-images-portrait" style="display:inline-block; margin: 10px">
{% for secondary_image in item.images.all %}
{% if secondary_image != main_image and forloop.counter <= 4 %}
- {% thumbnail secondary_image.media "165x90" crop=False as im %}
+ {% thumbnail secondary_image.media "100x100" crop=False as im %}
<div class="secondary-image-portrait">
<a href="{% url 'item_detail' item.collection.name item.item_guid %}?show={{secondary_image.image_guid}}">
<img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
@@ -28,7 +28,7 @@
</div>
{% else %}
<div class="item-links-landscape text-center">
- {% thumbnail main_image.media "350x150" crop=False as im %}
+ {% thumbnail main_image.media "250x250" crop=False as im %}
<div class="main-image" style="display:inline-block">
<a href="{% url 'item_detail' item.collection.name item.item_guid %}?show={{main_image.image_guid}}">
<img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
@@ -38,7 +38,7 @@
<div class="secondary-images-container secondary-images-landscape" style="margin: 10px">
{% for secondary_image in item.images.all %}
{% if secondary_image != main_image and forloop.counter <= 3 %}
- {% thumbnail secondary_image.media "140x140" crop=False as im %}
+ {% thumbnail secondary_image.media "100x100" crop=False as im %}
<div class="secondary-image-landscape pull-left" style="display:inline-block">
<a href="{% url 'item_detail' item.collection.name item.item_guid %}?show={{secondary_image.image_guid}}">
<img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
@@ -54,7 +54,7 @@
{% else %}
{% with item.images.first as main_image %}
<div class="item-links text-center" style="display:inline-block; margin: 10px 0px;">
- {% thumbnail main_image.media "350x210" crop=False as im %}
+ {% thumbnail main_image.media "250x250" crop=False as im %}
<a href="{% url 'item_detail' item.collection.name item.item_guid %}?show={{main_image.image_guid}}">
<img v-el:small-image src="{{ im.url }}" width="{{ im.width }}" height="{{ im.height }}" />
</a>