# HG changeset patch # User durandn # Date 1470930734 -7200 # Node ID ce46bbafb07989d87600d91f6c8f3d9be78b5a20 # Parent 327f6719674e834e11eb321cb21c73761f9ef7fc image import command + proper item metadatas diff -r 327f6719674e -r ce46bbafb079 src/iconolab/management/commands/importimages.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab/management/commands/importimages.py Thu Aug 11 17:52:14 2016 +0200 @@ -0,0 +1,187 @@ +# -*- coding: UTF-8 -*- +from django.core.management.base import BaseCommand, CommandError +from django.core.management import call_command +from django.conf import settings +from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata +from PIL import Image as ImagePIL +import os, csv, pprint, re, json + +class Command(BaseCommand): + help = "import images from a directory into the media folder and creates item and image objects" + + def add_arguments(self, parser): + parser.add_argument("csv_path") + parser.add_argument( + '--encoding', + dest='encoding', + default='utf-8', + help='CSV file encoding' + + ) + parser.add_argument( + '--collection-fixture', + dest='collection_fixture', + default=False, + help='loads the fixture then insert extracted data into the created collection', + ) + parser.add_argument( + '--collection-id', + dest='collection_id', + default=False, + help='insert extracted data into the specified collection instead of trying to load a collection fixture', + ) + + def handle(self, *args, **options): + pp = pprint.PrettyPrinter(indent=4) + try: + # Check we have a collection to store data into: + source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) + print("# Checking collection args") + if options.get("collection_fixture"): + print("## Finding collection json data in "+source_dir) + fixture_path = os.path.join(source_dir, options.get("collection_fixture")) + if not os.path.isfile(fixture_path): + print("### No "+options.get("collection_fixture")+".json file was found in the source directory") + raise ValueError("!!! Fixture file "+fixture_path+" was not found !!!") + try: + with open(fixture_path) as json_fixture_file: + collection_data = json.loads(json_fixture_file.read()) + if len(collection_data) != 1: + raise ValueError("!!! Collection fixture has either 0 or more than one item. It should only provide one and only one collection !!!") + if collection_data[0]["model"] != "iconolab.Collection": + raise ValueError("!!! Collection fixture should provide one iconolab.Collection object and nothing else. !!!") + except ValueError as e: + raise ValueError("!!! JSON Data is invalid. !!!") + elif options.get("collection_id"): + print("## Finding collection with id "+options.get("collection_id")) + try: + collection = Collection.objects.get(pk=options.get("collection_id")) + except Collection.DoesNotExist: + raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!") + else: + raise ValueError("!!! No collection fixture or collection id, aborting because we can't properly generate data. !!!") + # We read the csv + csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=";") + print("# Extracting data from csv file and storing it in standardized format") + # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT + cleaned_csv_data=[] + for row in csvreader: + cleaned_row_data = {} + for key in settings.IMPORT_FIELDS_DICT.keys(): + cleaned_row_data[key] = "" + for row_key in row.keys(): + if row_key in settings.IMPORT_FIELDS_DICT[key]: + cleaned_row_data[key] = row[row_key] + break + cleaned_csv_data.append(cleaned_row_data) + + print("# Finding corresponding images and filtering csv data for found images") + # Listing image files in csv directory + image_list = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f)) and not f.endswith(".csv")] + filtered_csv_data = [] + # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image + for item in cleaned_csv_data: + item["SRC_IMG_FILES"] = [] + has_image = False + for image in image_list: + if image.startswith(item["INV"]): + item["SRC_IMG_FILES"].append(image) + has_image = True + if has_image: + filtered_csv_data.append(item) + print("## found " + str(len(filtered_csv_data))+" items with at least one image") + print("# Importing data into Iconolab") + if options.get("collection_fixture"): + print("## Loading collection fixture") + call_command("loaddata", fixture_path) + collection = Collection.objects.get( + pk = collection_data[0]["pk"] + ) + print("## Converting image and moving it to static dir, creating Image and Item objects") + target_dir = os.path.join(settings.MEDIA_ROOT, "uploads") + print("### Images will be stored in "+target_dir) + for item in filtered_csv_data: + print("#### Computing metadatas for item "+item["INV"]+" (inv number)") + item_authors = item["AUTR"] + item_school = item["ECOLE"] + item_designation = "" + if item.get("TITR", ""): + item_designation = item["TITR"] + elif item.get("DENO", ""): + item_designation = item["DENO"] + elif item.get("APPL", ""): + item_designation = item["APPL"] + item_datation = "" + if item.get("PERI", ""): + item_datation = item["PERI"] + elif item.get("MILL", ""): + item_datation = item["MILL"] + elif item.get("EPOQ", ""): + item_datation = item["EPOQ"] + item_technics = item["TECH"] + item_measurements = item["DIMS"] + item_create_or_usage_location = item["LIEUX"] + item_discovery_context = item["DECV"] + item_conservation_location = item["LOCA"] + item_photo_credits = item["PHOT"] + item_inventory_number = item["INV"] + item_joconde_ref = item["REF"] + if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists(): + print("#### An item with "+item["INV"]+" for inventory number, already exists in databse in the import collection") + else: + print("#### Creating item "+item["INV"]+" (inv number) in databse") + item_object = Item.objects.create( + collection = collection + ) + ItemMetadata.objects.create( + item = item_object, + authors = item_authors, + school = item_school, + designation = item_designation, + datation = item_datation, + technics = item_technics, + measurements = item_measurements, + create_or_usage_location = item_create_or_usage_location, + discovery_context = item_discovery_context, + conservation_location = item_conservation_location, + photo_credits = item_photo_credits, + inventory_number = item_inventory_number, + joconde_ref = item_joconde_ref + ) + print("#### Computing item image(s)") + for image in item["SRC_IMG_FILES"]: + (image_name, ext) = os.path.splitext(image) + image_path = os.path.join(target_dir, image_name) + ".jpg" + if os.path.isfile(image_path): + print("##### A jpeg file already exists in target dir for "+ image) + try: + im = ImagePIL.open(image_path) + im_width, im_height = im.size + except Exception as e: + print(e) + else: + jpeg_img_path = image_path + try: + im = ImagePIL.open(os.path.join(source_dir, image)) + print("##### Generating or copying jpeg for "+image) + im.thumbnail(im.size) + im.save(jpeg_img_path, "JPEG", quality=100) + im_width, im_height = im.size + except Exception as e: + print(e) + new_image = Image.objects.create( + item = item_object, + media = image_path, + name = image_name, + height = im_height, + width = im_width + ) + ImageStats.objects.create( + image = new_image + ) + print("# All done!") + except FileNotFoundError: + print("!!! File "+options.get("csv_path")+" does not exist. !!!") + except ValueError as e: + print(str(e)) + \ No newline at end of file diff -r 327f6719674e -r ce46bbafb079 src/iconolab/migrations/0008_auto_20160811_1050.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/iconolab/migrations/0008_auto_20160811_1050.py Thu Aug 11 17:52:14 2016 +0200 @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.7 on 2016-08-11 10:50 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('iconolab', '0007_auto_20160805_1304'), + ] + + operations = [ + migrations.RemoveField( + model_name='itemmetadata', + name='description', + ), + migrations.RemoveField( + model_name='itemmetadata', + name='domain', + ), + migrations.RemoveField( + model_name='itemmetadata', + name='title', + ), + migrations.AddField( + model_name='itemmetadata', + name='authors', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='conservation_location', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='create_or_usage_location', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='datation', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='designation', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='discovery_context', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='inventory_number', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='measurements', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='photo_credits', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='school', + field=models.CharField(default='', max_length=255), + ), + migrations.AddField( + model_name='itemmetadata', + name='technics', + field=models.CharField(default='', max_length=255), + ), + migrations.AlterField( + model_name='itemmetadata', + name='joconde_ref', + field=models.CharField(default='', max_length=255), + ), + ] diff -r 327f6719674e -r ce46bbafb079 src/iconolab/models.py --- a/src/iconolab/models.py Mon Aug 08 18:24:27 2016 +0200 +++ b/src/iconolab/models.py Thu Aug 11 17:52:14 2016 +0200 @@ -43,10 +43,22 @@ class ItemMetadata(models.Model): item = models.OneToOneField('Item', related_name='metadatas') - joconde_ref = models.CharField(max_length=20) - domain = models.CharField(max_length=255) - title = models.CharField(max_length=255) - description = models.CharField(max_length=255) + authors = models.CharField(max_length=255, default="") + school = models.CharField(max_length=255, default="") + designation = models.CharField(max_length=255, default="") + datation = models.CharField(max_length=255, default="") + technics = models.CharField(max_length=255, default="") + measurements = models.CharField(max_length=255, default="") + create_or_usage_location = models.CharField(max_length=255, default="") + discovery_context = models.CharField(max_length=255, default="") + conservation_location = models.CharField(max_length=255, default="") + photo_credits = models.CharField(max_length=255, default="") + inventory_number = models.CharField(max_length=255, default="") + joconde_ref = models.CharField(max_length=255, default="") + + @property + def get_joconde_url(self): + return self.joconde_ref class ImageStats(models.Model): diff -r 327f6719674e -r ce46bbafb079 src/iconolab/settings/__init__.py --- a/src/iconolab/settings/__init__.py Mon Aug 08 18:24:27 2016 +0200 +++ b/src/iconolab/settings/__init__.py Thu Aug 11 17:52:14 2016 +0200 @@ -147,3 +147,27 @@ USE_L10N = True USE_TZ = True + + +IMPORT_FIELDS_DICT = { + "AUTR": [], + "ECOLE": [], + "TITR": ["Titre"], + "DENO": [], + "APPL": [], + "PERI": ["Période"], + "MILL": [], + "EPOCH": [], + "TECH": [], + "DIMS": ["Dimensions"], + "EPOCH": [], + "LIEUX": [], + "DECV": [], + "LOCA": ["Localisation"], + "PHOT": ["Photo"], + "INV": ["No inventaire",], + "REF": ["REFERENCE"], + +} +NO_IMG_CONVERSION_EXTS = [".jpg"] +IMG_CONVERSION_EXTS = [".tif", ".tiff"] diff -r 327f6719674e -r ce46bbafb079 src/iconolab/templates/iconolab/detail_item.html --- a/src/iconolab/templates/iconolab/detail_item.html Mon Aug 08 18:24:27 2016 +0200 +++ b/src/iconolab/templates/iconolab/detail_item.html Thu Aug 11 17:52:14 2016 +0200 @@ -21,11 +21,18 @@ {% endfor %}
-

{% if item.metadatas.title %}{{item.metadatas.title}}{% else %}Objet sans titre{% endif %}

- {% if item.metadatas.description %} -

Description:

-

{{item.metadatas.description}}


- {% endif %} + {% if item.metadatas.designation %}

Désignation: {{item.metadatas.designation}}

{% endif %} + {% if item.metadatas.authors %}

Auteur(s): {{item.metadatas.designation}}

{% endif %} + {% if item.metadatas.conservation_location %}

Conservé à: {{item.metadatas.conservation_location}}

{% endif %} + {% if item.metadatas.datation %}

Datation: {{item.metadatas.datation}}

{% endif %} + {% if item.metadatas.technics %}
Techniques: {{item.metadatas.technics}}
{% endif %} + {% if item.metadatas.measurements %}
Mesures: {{item.metadatas.measurements}}
{% endif %} + {% if item.metadatas.create_or_usage_location %}
Lieu de création/utilisation: {{item.metadatas.create_or_usage_location}}
{% endif %} + {% if item.metadatas.discovery_context %}
Contexte de découverte: {{item.metadatas.discovery_context}}
{% endif %} + {% if item.metadatas.photo_credits %}
Crédits photographique: {{item.metadatas.photo_credits}}
{% endif %} + {% if item.metadatas.inventory_number %}
Numéro d'inventaire: {{item.metadatas.inventory_number}}
{% endif %} + {% if item.metadatas.joconde_ref %}
Cet objet dans Joconde {{item.metadatas.get_joconde_url}}
{% endif %} +
{% if item.images.all.count > 1 %}

Autres images pour cet objet:

{% for image in item.images.all %} diff -r 327f6719674e -r ce46bbafb079 src/iconolab/urls.py --- a/src/iconolab/urls.py Mon Aug 08 18:24:27 2016 +0200 +++ b/src/iconolab/urls.py Thu Aug 11 17:52:14 2016 +0200 @@ -29,14 +29,14 @@ url(r'^$', views.iconolab.RedirectView.as_view(url=reverse_lazy("home"))), url(r'^admin/', admin.site.urls), url(r'^home$', views.iconolab.GlobalHomepageView.as_view(), name="home"), - url(r'^collections/(?P[a-z]+)$', views.iconolab.CollectionHomepageView.as_view(), name='collection_home'), # Home fond - url(r'^collections/(?P[a-z]+)/items/(?P[^/]+)$', views.iconolab.ShowItemView.as_view(), name='item_detail'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)$', views.iconolab.ShowImageView.as_view(), name='image_detail'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)/annotations/create$', login_required(views.iconolab.CreateAnnotationView.as_view()), name='annotation_create'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/detail$', views.iconolab.ShowAnnotationView.as_view(), name='annotation_detail'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/edit$', login_required(views.iconolab.EditAnnotationView.as_view()), name='annotation_edit'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/revisions/(?P[^/]+)/detail', views.iconolab.ShowRevisionView.as_view(), name='revision_detail'), - url(r'^collections/(?P[a-z]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/revisions/(?P[^/]+)/merge$', login_required(views.iconolab.MergeProposalView.as_view()), name='annotation_merge'), + url(r'^collections/(?P[a-z0-9\-]+)$', views.iconolab.CollectionHomepageView.as_view(), name='collection_home'), # Home fond + url(r'^collections/(?P[a-z0-9\-]+)/items/(?P[^/]+)$', views.iconolab.ShowItemView.as_view(), name='item_detail'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)$', views.iconolab.ShowImageView.as_view(), name='image_detail'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)/annotations/create$', login_required(views.iconolab.CreateAnnotationView.as_view()), name='annotation_create'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/detail$', views.iconolab.ShowAnnotationView.as_view(), name='annotation_detail'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/edit$', login_required(views.iconolab.EditAnnotationView.as_view()), name='annotation_edit'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/revisions/(?P[^/]+)/detail', views.iconolab.ShowRevisionView.as_view(), name='revision_detail'), + url(r'^collections/(?P[a-z0-9\-]+)/images/(?P[^/]+)/annotations/(?P[^/]+)/revisions/(?P[^/]+)/merge$', login_required(views.iconolab.MergeProposalView.as_view()), name='annotation_merge'), url(r'^user/(?P[a-z0-9\-]+)/home/?$', views.iconolab.UserHomeView.as_view(), name="user_home"), url(r'^user/notifications/all/?$', login_required(views.iconolab.UserNotificationsView.as_view()), name="user_notifications"), url(r'^errors/404', views.iconolab.NotFoundErrorView.as_view(), name="404error"),