# HG changeset patch # User durandn # Date 1473946031 -7200 # Node ID 7581a9f3b2c90e9084be0a5f7f8cd00e9f6a8341 # Parent c0ce1738aa881bfc89fe264725348f1bed903294 reworked import command (no startswith, create object from human-readable json files instead of fixtures) diff -r c0ce1738aa88 -r 7581a9f3b2c9 src/iconolab/management/commands/importimages.py --- a/src/iconolab/management/commands/importimages.py Mon Sep 12 16:43:56 2016 +0200 +++ b/src/iconolab/management/commands/importimages.py Thu Sep 15 15:27:11 2016 +0200 @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.core.management import call_command from django.conf import settings -from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata +from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory from PIL import Image as ImagePIL import os, csv, pprint, re, json @@ -26,10 +26,15 @@ ) parser.add_argument( - '--collection-fixture', - dest='collection_fixture', + '--collection-json', + dest='collection_json', default=False, - help='loads the fixture then insert extracted data into the created collection', + help='creates a new collection from a json file, must be an object with fields : '+ \ + '"name" (identifier), '+ \ + '"verbose_name" (proper title name), '+ \ + '"description" (description on homepage, html is supported), '+ \ + '"image" (image on homepages, must be "uploads/"), '+ \ + '"height" and "width" (height and width of the image)', ) parser.add_argument( '--collection-id', @@ -38,10 +43,10 @@ help='insert extracted data into the specified collection instead of trying to load a collection fixture', ) parser.add_argument( - '--metacategories-fixture', - dest='metacategories_fixture', + '--metacategories-json', + dest='metacategories_json', default=False, - help='add metacategories to the created collection from a fixture file', + help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)', ) parser.add_argument( '--delimiter', @@ -49,6 +54,12 @@ default=';', help='csv file delimiter' ) + parser.add_argument( + '--img-filename-identifier', + dest='img_filename_identifier', + default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER, + help='codename of the csv field we\'ll try to match to find the related image to a given object' + ) def handle(self, *args, **options): pp = pprint.PrettyPrinter(indent=4) @@ -56,19 +67,28 @@ # Check we have a collection to store data into: source_dir = os.path.dirname(os.path.realpath(options.get("csv_path"))) print("# Checking collection args") - if options.get("collection_fixture"): + if options.get("collection_json"): print("## Finding collection json data in "+source_dir) - fixture_path = os.path.join(source_dir, options.get("collection_fixture")) - if not os.path.isfile(fixture_path): - print("### No "+options.get("collection_fixture")+".json file was found in the source directory") - raise ValueError("!!! Fixture file "+fixture_path+" was not found !!!") + collection_json_path = os.path.join(source_dir, options.get("collection_json")) + if not os.path.isfile(collection_json_path): + print("### No "+options.get("collection_json")+".json file was found in the source directory") + raise ValueError("!!! Json file "+collection_json_path+" was not found !!!") try: - with open(fixture_path) as json_fixture_file: + with open(collection_json_path) as json_fixture_file: collection_data = json.loads(json_fixture_file.read()) - if len(collection_data) != 1: - raise ValueError("!!! Collection fixture has either 0 or more than one item. It should only provide one and only one collection !!!") - if collection_data[0]["model"] != "iconolab.Collection": - raise ValueError("!!! Collection fixture should provide one iconolab.Collection object and nothing else. !!!") + for key in ["name", "verbose_name", "description", "image", "height", "width"]: + if not key in collection_data.keys(): + print("!!! Json file "+collection_json_path+" has no "+key+" field !!!") + raise ValueError() + if not collection_data.get("name", ""): + print("!!! Collection data key 'name' is empty") + raise ValueError() + if Collection.objects.filter(name=collection_data.get("name")).exists(): + print("!!! A Collection with the provided name already exists!") + raise ValueError() + if collection_data.get("image", "") and not (collection_data.get("width", 0) and collection_data.get("height", 0)): + print("!!! Collection data has an image but no height and width") + raise ValueError() except ValueError as e: raise ValueError("!!! JSON Data is invalid. !!!") elif options.get("collection_id"): @@ -80,23 +100,29 @@ else: raise ValueError("!!! No collection fixture or collection id, aborting because we can't properly generate data. !!!") - if options.get("metacategories_fixture"): + if options.get("metacategories_json"): print("## Finding metacategories fixture json data in "+source_dir) - metacategories_fixture_path = os.path.join(source_dir, options.get("metacategories_fixture")) - if not os.path.isfile(metacategories_fixture_path): - print("### No "+options.get("metacategories_fixture")+".json file was found in the source directory") - raise ValueError("!!! Fixture file "+metacategories_fixture_path+" was not found !!!") - with open(metacategories_fixture_path) as metacategories_fixture_file: - metacategories_data = json.loads(metacategories_fixture_file.read()) + metacategories_json_path = os.path.join(source_dir, options.get("metacategories_json")) + if not os.path.isfile(metacategories_json_path): + print("### No "+options.get("metacategories_json")+".json file was found in the source directory") + raise ValueError("!!! Fixture file "+metacategories_json_path+" was not found !!!") + with open(metacategories_json_path) as metacategories_json_file: + metacategories_data = json.loads(metacategories_json_file.read()) for metacategory in metacategories_data: - if options.get("collection_fixture") and metacategory["fields"].get("collection", False) != collection_data[0].get("pk"): - print(metacategory["fields"].get("collection", False)) - raise ValueError("!!! The fixture should only contain metacategories for the imported collection !!!") - elif options.get("collection_id") and metacategory["fields"].get("collection", False) != collection.id: - raise ValueError("!!! The fixture should only contain metacategories for the imported collection !!!") + if mecategory.get("label", None) is None: + raise ValueError("!!! Metacategory without label !!!") # We read the csv - csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=options.get('csv_delimiter', ';')) + delimiter = options.get('csv_delimiter') + if delimiter == "#9": + delimiter = chr(9) + if delimiter == "#29": + delimiter = chr(29) + if delimiter == "#30": + delimiter = chr(30) + if delimiter == "#31": + delimiter = chr(31) + csvreader = csv.DictReader(open(options.get("csv_path"), encoding=options.get("encoding")), delimiter=delimiter) print("# Extracting data from csv file and storing it in standardized format") # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT cleaned_csv_data=[] @@ -109,8 +135,7 @@ cleaned_row_data[key] = row[row_key] break cleaned_csv_data.append(cleaned_row_data) - - print("# Finding corresponding images and filtering csv data for found images") + print(cleaned_csv_data) # Listing image files in csv directory image_list = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f)) and not f.endswith(".csv")] filtered_csv_data = [] @@ -118,19 +143,26 @@ for item in cleaned_csv_data: item["SRC_IMG_FILES"] = [] has_image = False - for image in image_list: - if image.startswith(item["INV"]): + for image in image_list: + img_name_pattern = r'.*'+re.escape(item[options.get("img_filename_identifier")])+r'[\.\-_].*' + print(img_name_pattern+" versus "+image) + if re.match(img_name_pattern, image): item["SRC_IMG_FILES"].append(image) has_image = True if has_image: filtered_csv_data.append(item) + print("## found " + str(len(filtered_csv_data))+" items with at least one image") print("# Importing data into Iconolab") - if options.get("collection_fixture"): - print("## Loading collection fixture") - call_command("loaddata", fixture_path) - collection = Collection.objects.get( - pk = collection_data[0]["pk"] + if options.get("collection_json"): + print("## Loading collection json") + collection = Collection.objects.create( + name = collection_data.get("name"), + verbose_name = collection_data.get("verbose_name", ""), + description = collection_data.get("description", ""), + image = collection_data.get("image", ""), + height = collection_data.get("height", 0), + width = collection_data.get("width", 0), ) if collection.image: collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image)) @@ -144,8 +176,13 @@ col_im.save(collection_image_path, "JPEG", quality=options.get("jpeg_quality", settings.IMG_JPG_DEFAULT_QUALITY)) except Exception as e: print(e) - if options.get("metacategories_fixture"): - call_command("loaddata", metacategories_fixture_path) + if options.get("metacategories_json"): + for metacategory in metacategories_data: + MetaCategory.objects.create( + collection = collection, + label = metacategory.get("label"), + triggers_notifications = metacategory.get("triggers_notifications", 0) + ) print("## Converting image and moving it to static dir, creating Image and Item objects") target_dir = os.path.join(settings.MEDIA_ROOT, "uploads") print("### Images will be stored in "+target_dir) diff -r c0ce1738aa88 -r 7581a9f3b2c9 src/iconolab/settings/__init__.py --- a/src/iconolab/settings/__init__.py Mon Sep 12 16:43:56 2016 +0200 +++ b/src/iconolab/settings/__init__.py Thu Sep 15 15:27:11 2016 +0200 @@ -217,6 +217,7 @@ "REF": ["REFERENCE"], } +IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER = "INV" NO_IMG_CONVERSION_EXTS = [".jpg"] IMG_CONVERSION_EXTS = [".tif", ".tiff"]