|
1 # -*- coding: UTF-8 -*- |
|
2 from django.core.management.base import BaseCommand, CommandError |
|
3 from django.core.management import call_command |
|
4 from django.conf import settings |
|
5 from iconolab.models import Collection, Item, ItemMetadata, MetaCategory |
|
6 from sorl.thumbnail import get_thumbnail |
|
7 import os, csv, pprint, re, json, shutil |
|
8 |
|
9 class Command(BaseCommand): |
|
10 help = "import images from a directory into the media folder and creates item and image objects" |
|
11 |
|
12 def add_arguments(self, parser): |
|
13 parser.add_argument("csv_path") |
|
14 parser.add_argument( |
|
15 '--encoding', |
|
16 dest='encoding', |
|
17 default='utf-8', |
|
18 help='CSV file encoding' |
|
19 |
|
20 ) |
|
21 parser.add_argument( |
|
22 '--collection-id', |
|
23 dest='collection_id', |
|
24 default=False, |
|
25 help='insert extracted data into the specified collection instead of trying to load a collection fixture', |
|
26 ) |
|
27 parser.add_argument( |
|
28 '--delimiter', |
|
29 dest='csv_delimiter', |
|
30 default=';', |
|
31 help='csv file delimiter' |
|
32 ) |
|
33 |
|
34 def handle(self, *args, **options): |
|
35 try: |
|
36 # Check we have a collection to store data into: |
|
37 source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) |
|
38 print("# Checking collection args") |
|
39 if options.get("collection_id"): |
|
40 print("## Finding collection with id "+options.get("collection_id")) |
|
41 try: |
|
42 collection = Collection.objects.get(pk=options.get("collection_id")) |
|
43 except Collection.DoesNotExist: |
|
44 raise ValueError("!!! Collection with primary key "+options.get("collection_id")+" was not found, aborting !!!") |
|
45 else: |
|
46 raise ValueError("!!! No collection id, aborting because we don't know which collection to update. !!!") |
|
47 |
|
48 # We read the csv |
|
49 delimiter = options.get('csv_delimiter') |
|
50 if delimiter == "#9": |
|
51 delimiter = chr(9) |
|
52 if delimiter == "#29": |
|
53 delimiter = chr(29) |
|
54 if delimiter == "#30": |
|
55 delimiter = chr(30) |
|
56 if delimiter == "#31": |
|
57 delimiter = chr(31) |
|
58 csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter) |
|
59 print("# Extracting data from csv file and storing it in standardized format") |
|
60 # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT |
|
61 cleaned_csv_data=[] |
|
62 for row in csvreader: |
|
63 cleaned_row_data = {} |
|
64 for key in settings.IMPORT_FIELDS_DICT.keys(): |
|
65 cleaned_row_data[key] = "" |
|
66 for row_key in row.keys(): |
|
67 if row_key in settings.IMPORT_FIELDS_DICT[key]: |
|
68 # Handling the multiple natural keys exports nonsense |
|
69 if key == "REF": |
|
70 natural_key_number, _, _ = row[row_key].partition(";") |
|
71 cleaned_row_data[key] = natural_key_number.rstrip() |
|
72 else: |
|
73 cleaned_row_data[key] = row[row_key] |
|
74 break |
|
75 cleaned_csv_data.append(cleaned_row_data) |
|
76 # Listing image files in csv directory |
|
77 filtered_csv_data = [] |
|
78 # Now we trim the cleaned_csv_data dict to keep only entries that already exist in the database in Item form (using the natural key) |
|
79 for item in cleaned_csv_data: |
|
80 if item.get("REF", "") and ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).exists(): |
|
81 filtered_csv_data.append(item) |
|
82 print("## found " + str(len(filtered_csv_data))+" items to update") |
|
83 print("# Updating data from Iconolab") |
|
84 for item in filtered_csv_data: |
|
85 item_metadatas = ItemMetadata.objects.filter(item__collection = collection, natural_key=item.get("REF")).first() |
|
86 if not item["REF"]: |
|
87 print("#### No Natural key, skipping") |
|
88 continue |
|
89 item_authors = item["AUTR"] |
|
90 item_school = item["ECOLE"] |
|
91 item_designation = "" |
|
92 if item.get("TITR", ""): |
|
93 item_designation = item["TITR"] |
|
94 elif item.get("DENO", ""): |
|
95 item_designation = item["DENO"] |
|
96 elif item.get("APPL", ""): |
|
97 item_designation = item["APPL"] |
|
98 item_datation = "" |
|
99 if item.get("PERI", ""): |
|
100 item_datation = item["PERI"] |
|
101 elif item.get("MILL", ""): |
|
102 item_datation = item["MILL"] |
|
103 elif item.get("EPOQ", ""): |
|
104 item_datation = item["EPOQ"] |
|
105 item_technics = item["TECH"] |
|
106 item_field = item["DOM"] |
|
107 item_measurements = item["DIMS"] |
|
108 item_create_or_usage_location = item["LIEUX"] |
|
109 item_discovery_context = item["DECV"] |
|
110 item_conservation_location = item["LOCA"] |
|
111 item_photo_credits = item["PHOT"] |
|
112 item_inventory_number = item["INV"] |
|
113 item_joconde_ref = item["REF"] |
|
114 # Updating metadatas |
|
115 |
|
116 new_metadata = { |
|
117 "authors" : item_authors, |
|
118 "school" : item_school, |
|
119 "designation" : item_designation, |
|
120 "field" : item_field, |
|
121 "datation" : item_datation, |
|
122 "technics" : item_technics, |
|
123 "measurements" : item_measurements, |
|
124 "create_or_usage_location" : item_create_or_usage_location, |
|
125 "discovery_context" : item_discovery_context, |
|
126 "conservation_location" : item_conservation_location, |
|
127 "photo_credits" : item_photo_credits, |
|
128 "inventory_number" : item_inventory_number, |
|
129 "joconde_ref" : item_joconde_ref |
|
130 } |
|
131 |
|
132 item_metadatas.metadata = json.dumps(new_metadata) |
|
133 item_metadatas.natural_key = item_joconde_ref |
|
134 item_metadatas.save() |
|
135 print('### Generating thumbnails for item '+item['REF']) |
|
136 for image in item_metadatas.item.images.all(): |
|
137 for size in settings.PREGENERATE_THUMBNAILS_SIZES: |
|
138 print('#### Thumbnail for size '+size) |
|
139 get_thumbnail(image.media, size, crop=False) |
|
140 print("# All done!") |
|
141 except FileNotFoundError: |
|
142 print("!!! File "+options.get("csv_path")+" does not exist. !!!") |
|
143 except ValueError as e: |
|
144 print(str(e)) |