src/iconolab_mcc/management/commands/updatecollection.py
author ymh <ymh.work@gmail.com>
Tue, 26 Jun 2018 16:22:54 +0200
changeset 21 631f70f55fed
parent 5 cfd40849d24c
permissions -rw-r--r--
Use the new generic "get_natural_key" method

# -*- coding: UTF-8 -*-
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.conf import settings
from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
from sorl.thumbnail import get_thumbnail
import os, csv, pprint, re, json, shutil

class Command(BaseCommand):
    help = "import images from a directory into the media folder and creates item and image objects"

    def add_arguments(self, parser):
        parser.add_argument("csv_path")
        parser.add_argument(
            '--encoding',
            dest='encoding',
            default='utf-8',
            help='CSV file encoding'

        )
        parser.add_argument(
            '--collection-id',
            dest='collection_id',
            default=False,
            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
        )
        parser.add_argument(
            '--delimiter',
            dest='csv_delimiter',
            default=';',
            help='csv file delimiter'
        )

    def handle(self, *args, **options):
        try:
            # Check we have a collection to store data into:
            source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
            print("# Checking collection args")
            if options.get("collection_id"):
                print("## Finding collection with id "+options.get("collection_id"))
                try:
                    collection = Collection.objects.get(pk=options.get("collection_id"))
                except Collection.DoesNotExist:
                    raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
            else:
                raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")

            # We read the csv
            delimiter = options.get('csv_delimiter')
            if delimiter == "#9":
                delimiter = chr(9)
            if delimiter == "#29":
                delimiter = chr(29)
            if delimiter == "#30":
                delimiter = chr(30)
            if delimiter == "#31":
                delimiter = chr(31)
            csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
            print("# Extracting data from csv file and storing it in standardized format")
            # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
            cleaned_csv_data=[]
            for row in csvreader:
                cleaned_row_data = {}
                for key in settings.IMPORT_FIELDS_DICT.keys():
                    cleaned_row_data[key] = ""
                    for row_key in row.keys():
                        if row_key in settings.IMPORT_FIELDS_DICT[key]:
                            # Handling the multiple natural keys exports nonsense
                            if key == "REF":
                                natural_key_number, _, _ = row[row_key].partition(";")
                                cleaned_row_data[key] = natural_key_number.rstrip()
                            else:
                                cleaned_row_data[key] = row[row_key]
                            break
                cleaned_csv_data.append(cleaned_row_data)
            # Listing image files in csv directory
            filtered_csv_data = []
            # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key)
            for item in cleaned_csv_data:
                if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists():
                    filtered_csv_data.append(item)
            print("## found " + str(len(filtered_csv_data))+" items to update")
            print("# Updating data from Iconolab")
            for item in filtered_csv_data:
                item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first()
                if not item["REF"]:
                    print("#### No Natural key, skipping")
                    continue
                item_authors = item["AUTR"]
                item_school = item["ECOLE"]
                item_designation = ""
                if item.get("TITR", ""):
                    item_designation = item["TITR"]
                elif item.get("DENO", ""):
                    item_designation = item["DENO"]
                elif item.get("APPL", ""):
                    item_designation = item["APPL"]
                item_datation = ""
                if item.get("PERI", ""):
                    item_datation = item["PERI"]
                elif item.get("MILL", ""):
                    item_datation = item["MILL"]
                elif item.get("EPOQ", ""):
                    item_datation = item["EPOQ"]
                item_technics = item["TECH"]
                item_field = item["DOM"]
                item_measurements = item["DIMS"]
                item_create_or_usage_location = item["LIEUX"]
                item_discovery_context = item["DECV"]
                item_conservation_location = item["LOCA"]
                item_photo_credits = item["PHOT"]
                item_inventory_number = item["INV"]
                item_joconde_ref = item["REF"]
                # Updating metadatas

                new_metadata = {
                    "authors" : item_authors,
                    "school" : item_school,
                    "designation" : item_designation,
                    "field" : item_field,
                    "datation" : item_datation,
                    "technics" : item_technics,
                    "measurements" : item_measurements,
                    "create_or_usage_location" : item_create_or_usage_location,
                    "discovery_context" : item_discovery_context,
                    "conservation_location" : item_conservation_location,
                    "photo_credits" : item_photo_credits,
                    "inventory_number" : item_inventory_number,
                    "joconde_ref" : item_joconde_ref
                }

                item_metadatas.metadata = json.dumps(new_metadata)
                item_metadatas.natural_key = item_joconde_ref
                item_metadatas.save()
                print('### Generating thumbnails for item '+item['REF'])
                for image in item_metadatas.item.images.all():
                    for size in settings.PREGENERATE_THUMBNAILS_SIZES:
                        print('#### Thumbnail for size '+size)
                        get_thumbnail(image.media, size, crop=False)
            print("# All done!")
        except FileNotFoundError:
            print("!!! File "+options.get("csv_path")+" does not exist. !!!")
        except ValueError as e:
            print(str(e))