--- a/src/iconolab/management/commands/importimages.py Mon Sep 12 16:43:56 2016 +0200
+++ b/src/iconolab/management/commands/importimages.py Thu Sep 15 15:27:11 2016 +0200
@@ -2,7 +2,7 @@
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.conf import settings
-from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata
+from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory
from PIL import Image as ImagePIL
import os, csv, pprint, re, json
@@ -26,10 +26,15 @@
)
parser.add_argument(
- '--collection-fixture',
- dest='collection_fixture',
+ '--collection-json',
+ dest='collection_json',
default=False,
- help='loads the fixture then insert extracted data into the created collection',
+ help='creates a new collection from a json file, must be an object with fields : '+ \
+ '"name" (identifier), '+ \
+ '"verbose_name" (proper title name), '+ \
+ '"description" (description on homepage, html is supported), '+ \
+ '"image" (image on homepages, must be "uploads/<imgname>"), '+ \
+ '"height" and "width" (height and width of the image)',
)
parser.add_argument(
'--collection-id',
@@ -38,10 +43,10 @@
help='insert extracted data into the specified collection instead of trying to load a collection fixture',
)
parser.add_argument(
- '--metacategories-fixture',
- dest='metacategories_fixture',
+ '--metacategories-json',
+ dest='metacategories_json',
default=False,
- help='add metacategories to the created collection from a fixture file',
+ help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',
)
parser.add_argument(
'--delimiter',
@@ -49,6 +54,12 @@
default=';',
help='csv file delimiter'
)
+ parser.add_argument(
+ '--img-filename-identifier',
+ dest='img_filename_identifier',
+ default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
+ help='codename of the csv field we\'ll try to match to find the related image to a given object'
+ )
def handle(self, *args, **options):
pp = pprint.PrettyPrinter(indent=4)
@@ -56,19 +67,28 @@
# Check we have a collection to store data into:
source_dir = os.path.dirname(os.path.realpath(options.get("csv_path")))
print("# Checking collection args")
- if options.get("collection_fixture"):
+ if options.get("collection_json"):
print("## Finding collection json data in "+source_dir)
- fixture_path = os.path.join(source_dir, options.get("collection_fixture"))
- if not os.path.isfile(fixture_path):
- print("### No "+options.get("collection_fixture")+".json file was found in the source directory")
- raise ValueError("!!! Fixture file "+fixture_path+" was not found !!!")
+ collection_json_path = os.path.join(source_dir, options.get("collection_json"))
+ if not os.path.isfile(collection_json_path):
+ print("### No "+options.get("collection_json")+".json file was found in the source directory")
+ raise ValueError("!!! Json file "+collection_json_path+" was not found !!!")
try:
- with open(fixture_path) as json_fixture_file:
+ with open(collection_json_path) as json_fixture_file:
collection_data = json.loads(json_fixture_file.read())
- if len(collection_data) != 1:
- raise ValueError("!!! Collection fixture has either 0 or more than one item. It should only provide one and only one collection !!!")
- if collection_data[0]["model"] != "iconolab.Collection":
- raise ValueError("!!! Collection fixture should provide one iconolab.Collection object and nothing else. !!!")
+ for key in ["name", "verbose_name", "description", "image", "height", "width"]:
+ if not key in collection_data.keys():
+ print("!!! Json file "+collection_json_path+" has no "+key+" field !!!")
+ raise ValueError()
+ if not collection_data.get("name", ""):
+ print("!!! Collection data key 'name' is empty")
+ raise ValueError()
+ if Collection.objects.filter(name=collection_data.get("name")).exists():
+ print("!!! A Collection with the provided name already exists!")
+ raise ValueError()
+ if collection_data.get("image", "") and not (collection_data.get("width", 0) and collection_data.get("height", 0)):
+ print("!!! Collection data has an image but no height and width")
+ raise ValueError()
except ValueError as e:
raise ValueError("!!! JSON Data is invalid. !!!")
elif options.get("collection_id"):
@@ -80,23 +100,29 @@
else:
raise ValueError("!!! No collection fixture or collection id, aborting because we can't properly generate data. !!!")
- if options.get("metacategories_fixture"):
+ if options.get("metacategories_json"):
print("## Finding metacategories fixture json data in "+source_dir)
- metacategories_fixture_path = os.path.join(source_dir, options.get("metacategories_fixture"))
- if not os.path.isfile(metacategories_fixture_path):
- print("### No "+options.get("metacategories_fixture")+".json file was found in the source directory")
- raise ValueError("!!! Fixture file "+metacategories_fixture_path+" was not found !!!")
- with open(metacategories_fixture_path) as metacategories_fixture_file:
- metacategories_data = json.loads(metacategories_fixture_file.read())
+ metacategories_json_path = os.path.join(source_dir, options.get("metacategories_json"))
+ if not os.path.isfile(metacategories_json_path):
+ print("### No "+options.get("metacategories_json")+".json file was found in the source directory")
+ raise ValueError("!!! Fixture file "+metacategories_json_path+" was not found !!!")
+ with open(metacategories_json_path) as metacategories_json_file:
+ metacategories_data = json.loads(metacategories_json_file.read())
for metacategory in metacategories_data:
- if options.get("collection_fixture") and metacategory["fields"].get("collection", False) != collection_data[0].get("pk"):
- print(metacategory["fields"].get("collection", False))
- raise ValueError("!!! The fixture should only contain metacategories for the imported collection !!!")
- elif options.get("collection_id") and metacategory["fields"].get("collection", False) != collection.id:
- raise ValueError("!!! The fixture should only contain metacategories for the imported collection !!!")
+ if mecategory.get("label", None) is None:
+ raise ValueError("!!! Metacategory without label !!!")
# We read the csv
- csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=options.get('csv_delimiter', ';'))
+ delimiter = options.get('csv_delimiter')
+ if delimiter == "#9":
+ delimiter = chr(9)
+ if delimiter == "#29":
+ delimiter = chr(29)
+ if delimiter == "#30":
+ delimiter = chr(30)
+ if delimiter == "#31":
+ delimiter = chr(31)
+ csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter)
print("# Extracting data from csv file and storing it in standardized format")
# We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
cleaned_csv_data=[]
@@ -109,8 +135,7 @@
cleaned_row_data[key] = row[row_key]
break
cleaned_csv_data.append(cleaned_row_data)
-
- print("# Finding corresponding images and filtering csv data for found images")
+ print(cleaned_csv_data)
# Listing image files in csv directory
image_list = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f)) and not f.endswith(".csv")]
filtered_csv_data = []
@@ -118,19 +143,26 @@
for item in cleaned_csv_data:
item["SRC_IMG_FILES"] = []
has_image = False
- for image in image_list:
- if image.startswith(item["INV"]):
+ for image in image_list:
+ img_name_pattern = r'.*'+re.escape(item[options.get("img_filename_identifier")])+r'[\.\-_].*'
+ print(img_name_pattern+" versus "+image)
+ if re.match(img_name_pattern, image):
item["SRC_IMG_FILES"].append(image)
has_image = True
if has_image:
filtered_csv_data.append(item)
+
print("## found " + str(len(filtered_csv_data))+" items with at least one image")
print("# Importing data into Iconolab")
- if options.get("collection_fixture"):
- print("## Loading collection fixture")
- call_command("loaddata", fixture_path)
- collection = Collection.objects.get(
- pk = collection_data[0]["pk"]
+ if options.get("collection_json"):
+ print("## Loading collection json")
+ collection = Collection.objects.create(
+ name = collection_data.get("name"),
+ verbose_name = collection_data.get("verbose_name", ""),
+ description = collection_data.get("description", ""),
+ image = collection_data.get("image", ""),
+ height = collection_data.get("height", 0),
+ width = collection_data.get("width", 0),
)
if collection.image:
collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))
@@ -144,8 +176,13 @@
col_im.save(collection_image_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY))
except Exception as e:
print(e)
- if options.get("metacategories_fixture"):
- call_command("loaddata", metacategories_fixture_path)
+ if options.get("metacategories_json"):
+ for metacategory in metacategories_data:
+ MetaCategory.objects.create(
+ collection = collection,
+ label = metacategory.get("label"),
+ triggers_notifications = metacategory.get("triggers_notifications", 0)
+ )
print("## Converting image and moving it to static dir, creating Image and Item objects")
target_dir = os.path.join(settings.MEDIA_ROOT, "uploads")
print("### Images will be stored in "+target_dir)