# -*- coding: UTF-8 -*-
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.conf import settings
from iconolab.models import Collection, Item, ItemMetadata, MetaCategory
from sorl.thumbnail import get_thumbnail
import os, csv, pprint, re, json, shutil
class Command(BaseCommand):
help = "import images from a directory into the media folder and creates item and image objects"
def add_arguments(self, parser):
parser.add_argument("csv_path")
parser.add_argument(
'--encoding',
dest='encoding',
default='utf-8',
help='CSV file encoding'
)
parser.add_argument(
'--collection-id',
dest='collection_id',
default=False,
help='insert extracted data into the specified collection instead of trying to load a collection fixture',
)
parser.add_argument(
'--delimiter',
dest='csv_delimiter',
default=';',
help='csv file delimiter'
)
def handle(self, *args, **options):
try:
# Check we have a collection to store data into:
source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
print("# Checking collection args")
if options.get("collection_id"):
print("## Finding collection with id "+options.get("collection_id"))
try:
collection = Collection.objects.get(pk=options.get("collection_id"))
except Collection.DoesNotExist:
raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!")
else:
raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!")
# We read the csv
delimiter = options.get('csv_delimiter')
if delimiter == "#9":
delimiter = chr(9)
if delimiter == "#29":
delimiter = chr(29)
if delimiter == "#30":
delimiter = chr(30)
if delimiter == "#31":
delimiter = chr(31)
csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
print("# Extracting data from csv file and storing it in standardized format")
# We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
cleaned_csv_data=[]
for row in csvreader:
cleaned_row_data = {}
for key in settings.IMPORT_FIELDS_DICT.keys():
cleaned_row_data[key] = ""
for row_key in row.keys():
if row_key in settings.IMPORT_FIELDS_DICT[key]:
# Handling the multiple natural keys exports nonsense
if key == "REF":
natural_key_number, _, _ = row[row_key].partition(";")
cleaned_row_data[key] = natural_key_number.rstrip()
else:
cleaned_row_data[key] = row[row_key]
break
cleaned_csv_data.append(cleaned_row_data)
# Listing image files in csv directory
filtered_csv_data = []
# Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key)
for item in cleaned_csv_data:
if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists():
filtered_csv_data.append(item)
print("## found " + str(len(filtered_csv_data))+" items to update")
print("# Updating data from Iconolab")
for item in filtered_csv_data:
item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first()
if not item["REF"]:
print("#### No Natural key, skipping")
continue
item_authors = item["AUTR"]
item_school = item["ECOLE"]
item_designation = ""
if item.get("TITR", ""):
item_designation = item["TITR"]
elif item.get("DENO", ""):
item_designation = item["DENO"]
elif item.get("APPL", ""):
item_designation = item["APPL"]
item_datation = ""
if item.get("PERI", ""):
item_datation = item["PERI"]
elif item.get("MILL", ""):
item_datation = item["MILL"]
elif item.get("EPOQ", ""):
item_datation = item["EPOQ"]
item_technics = item["TECH"]
item_field = item["DOM"]
item_measurements = item["DIMS"]
item_create_or_usage_location = item["LIEUX"]
item_discovery_context = item["DECV"]
item_conservation_location = item["LOCA"]
item_photo_credits = item["PHOT"]
item_inventory_number = item["INV"]
item_joconde_ref = item["REF"]
# Updating metadatas
new_metadata = {
"authors" : item_authors,
"school" : item_school,
"designation" : item_designation,
"field" : item_field,
"datation" : item_datation,
"technics" : item_technics,
"measurements" : item_measurements,
"create_or_usage_location" : item_create_or_usage_location,
"discovery_context" : item_discovery_context,
"conservation_location" : item_conservation_location,
"photo_credits" : item_photo_credits,
"inventory_number" : item_inventory_number,
"joconde_ref" : item_joconde_ref
}
item_metadatas.metadata = json.dumps(new_metadata)
item_metadatas.natural_key = item_joconde_ref
item_metadatas.save()
print('### Generating thumbnails for item '+item['REF'])
for image in item_metadatas.item.images.all():
for size in settings.PREGENERATE_THUMBNAILS_SIZES:
print('#### Thumbnail for size '+size)
get_thumbnail(image.media, size, crop=False)
print("# All done!")
except FileNotFoundError:
print("!!! File "+options.get("csv_path")+" does not exist. !!!")
except ValueError as e:
print(str(e))