# -*- coding: UTF-8 -*-

from django.core.management.base import BaseCommand, CommandError

from django.core.management import call_command

from django.conf import settings

from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder

from PIL import Image as ImagePIL

from sorl.thumbnail import get_thumbnail

import os, csv, pprint, re, json, shutil, logging

if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):

    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)

else:

    logger = logging.getLogger(__name__)

class Command(BaseCommand):

    help = 'import images from a directory into the media folder and creates item and image objects'

    def add_arguments(self, parser):

        parser.add_argument('csv_path')

        parser.add_argument(

            '--jpeg-quality',

            dest='jpeg_quality',

            default=settings.IMG_JPG_DEFAULT_QUALITY,

            help='Jpeg default quality'

        )

        parser.add_argument(

            '--encoding',

            dest='encoding',

            default='utf-8',

            help='CSV file encoding'

        )

        parser.add_argument(

            '--collection-json',

            dest='collection_json',

            default=False,

            help='creates a new collection from a json file, must be an object with fields : '+ \

                 '"name" (identifier), '+ \

                 '"verbose_name" (proper title name), '+ \

                 '"description" (description on homepage, html is supported), '+ \

                 '"image" (image on homepages, must be "uploads/<imgname>"), '+ \

                 '"height" and "width" (height and width of the image)',

        )

        parser.add_argument(

            '--collection-id',

            dest='collection_id',

            default=False,

            help='insert extracted data into the specified collection instead of trying to load a collection fixture',

        )

        parser.add_argument(

            '--metacategories-json',

            dest='metacategories_json',

            default=False,

            help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',

        )

        parser.add_argument(

            '--delimiter',

            dest='csv_delimiter',

            default=';',

            help='csv file delimiter'

        )

        parser.add_argument(

            '--no-jpg-conversion',

            dest='no-jpg-conversion',

            default=False,

            help='use this option if you only want the image copied and not converted'

        )

        parser.add_argument(

            '--img-filename-identifier',

            dest='img_filename_identifier',

            default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,

            help='codename of the csv field we\'ll try to match to find the related image to a given object'

        )

        parser.add_argument(

            '--filename-regexp-prefix',

            dest='filename_regexp_prefix',

            default=r'.*',

            help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'

        )

        parser.add_argument(

            '--filename-regexp-suffix',

            dest='filename_regexp_suffix',

            default=r'[\.\-_].*',

            help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'

        )

        parser.add_argument(

            '--folders',

            dest='import_folders',

            default=False,

            action='store_const',

            const=True,

            help='option to create folders'

        )

        parser.add_argument(

            '--folders-regexp',

            dest='folders_regexp',

            default=False,

            help='regexp used to extract the folder name/number'

        )

        parser.add_argument(

            '--folders-metadata',

            dest='folders_metadata',

            default='REF',

            help='metadata from which to extract the folder name/number'

        )

    def handle(self, *args, **options):

        """

            Step-by-step for import:

            1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files

            1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic:

                * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas

                to which we associate a list of column header that will identified as that metadata

                * The cleaned_row_data list will associate the identifier with the actual value for its related column

            2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated

            to each row for the filtered data.

            3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary

            4) For each item:

                We create the object in the database

                * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT

                We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image

            Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command.

        """

        try:

            print('# Logging with logger '+logger.name)

            logger.debug('# Initializing command with args: %r', options)

            # Check we have a collection to store data into:

            source_dir = os.path.dirname(os.path.realpath(options.get('csv_path')))

            print('# Checking collection args')

            if options.get('collection_json'):

                print('## Finding collection json data in '+source_dir)

                collection_json_path = os.path.join(source_dir, options.get('collection_json'))

                if not os.path.isfile(collection_json_path):

                    print('### No '+options.get('collection_json')+'.json file was found in the source directory')

                    raise ValueError('!!! Json file '+collection_json_path+' was not found !!!')

                try:

                    with open(collection_json_path) as json_fixture_file:

                        collection_data = json.loads(json_fixture_file.read())

                        for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:

                            if not key in collection_data.keys():

                                print('!!! Json file '+collection_json_path+' has no '+key+' field !!!')

                                raise ValueError()

                        if not collection_data.get('name', ''):

                            print('!!! Collection data key "name" is empty')

                            raise ValueError()

                        if Collection.objects.filter(name=collection_data.get('name')).exists():

                            print('!!! A Collection with the provided name already exists!')

                            raise ValueError()

                        if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):

                            print('!!! Collection data has an image but no height and width')

                            raise ValueError()

                except ValueError as e:

                    raise ValueError('!!! JSON Data is invalid. !!!')

            elif options.get('collection_id'):

                print('## Finding collection with id '+options.get('collection_id'))

                try:

                    collection = Collection.objects.get(pk=options.get('collection_id'))

                except Collection.DoesNotExist:

                    raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!')

            else:

                raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')

            if options.get('metacategories_json'):

                print('## Finding metacategories fixture json data in '+source_dir)

                metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json'))

                if not os.path.isfile(metacategories_json_path):

                    print('### No '+options.get('metacategories_json')+'.json file was found in the source directory')

                    raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!')

                with open(metacategories_json_path) as metacategories_json_file:

                    metacategories_data = json.loads(metacategories_json_file.read())

                    for metacategory in metacategories_data:

                        if metacategory.get('label', None) is None:

                            raise ValueError('!!! Metacategory without label !!!')

            if options['import_folders'] and not options['folders_regexp']:

                raise ValueError('!!! No regexp specified to extract folder name !!!')

            # We read the csv

            delimiter = options.get('csv_delimiter')

            if delimiter == '#9':

                delimiter = chr(9)

            if delimiter == '#29':

                delimiter = chr(29)

            if delimiter == '#30':

                delimiter = chr(30)

            if delimiter == '#31':

                delimiter = chr(31)

            csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter)

            print('# Extracting data from csv file and storing it in standardized format')

            # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT

            cleaned_csv_data=[]

            duplicate_rows=[]

            for row in csvreader:

                cleaned_row_data = {}

                for key in settings.IMPORT_FIELDS_DICT.keys():

                    cleaned_row_data[key] = ''

                    for row_key in row.keys():

                        if row_key in settings.IMPORT_FIELDS_DICT[key]:

                            if key == 'REF':

                                ref_number, _, _ = row[row_key].partition(';')

                                cleaned_row_data[key] = ref_number.rstrip()

                            else:

                                cleaned_row_data[key] = row[row_key]

                            break

                if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]:

                    print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line")

                    duplicate_rows.append(cleaned_row_data)

                else:

                    cleaned_csv_data.append(cleaned_row_data)

            # Listing image files in csv directory

            image_list = [

                f for f in os.listdir(source_dir)

                if os.path.isfile(os.path.join(source_dir, f))

                and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png'))

            ] # Maybe check if image another way

            filtered_csv_data = []

            no_image_rows = []

            no_data_images = []

            assigned_images = []

            # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image

            for item in cleaned_csv_data:

                item['SRC_IMG_FILES'] = []

                has_image = False

                for image in image_list:

                    img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix')

                    if re.match(img_name_pattern, image):

                        item['SRC_IMG_FILES'].append(image)

                        assigned_images.append(image)

                        has_image = True

                if has_image:

                    filtered_csv_data.append(item)

                else:

                    # We keep track of the entries that don't have any corresponding image

                    no_image_rows.append(item)

            # We keep track of the images that don't have any corresponding entry

            for image in image_list:

                if image not in assigned_images:

                    no_data_images.append(image)

            print('## found ' + str(len(filtered_csv_data))+' items with at least one image')

            print('# Importing data into Iconolab')

            if options.get('collection_json'):

                print('## Loading collection json')

                collection = Collection.objects.create(

                    name = collection_data.get('name'),

                    verbose_name = collection_data.get('verbose_name', ''),

                    description = collection_data.get('description', ''),

                    image = collection_data.get('image', ''),

                    height = collection_data.get('height', 0),

                    width = collection_data.get('width', 0),

                )

                if collection.image:

                    collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))

                    if not os.path.isfile(collection_image_path):

                        print('### Moving collection image')

                        _ , collection_image_name = os.path.split(collection_image_path)

                        try:

                            col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name))

                            print('##### Generating or copying jpeg for '+collection_image_name)

                            col_im.thumbnail(col_im.size)

                            col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))

                        except Exception as e:

                            print(e)

            if options.get('metacategories_json'):

                for metacategory in metacategories_data:

                    MetaCategory.objects.create(

                        collection = collection,

                        label = metacategory.get('label'),

                        triggers_notifications = metacategory.get('triggers_notifications', 0)

                    )

            print('## Converting image and moving it to static dir, creating Image and Item objects')

            target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads')

            print('### Images will be stored in '+target_dir)

            for item in filtered_csv_data:

                print('#### Computing metadatas for item '+item['REF']+' (natural key)')

                if not item['REF']:

                    print('#### No Natural key, skipping')

                    continue

                item_authors = item['AUTR']

                item_school = item['ECOLE']

                item_designation = ''

                if item.get('TITR', ''):

                    item_designation = item['TITR']

                elif item.get('DENO', ''):

                    item_designation = item['DENO']

                elif item.get('APPL', ''):

                    item_designation = item['APPL']

                item_datation = ''

                if item.get('PERI', ''):

                    item_datation = item['PERI']

                elif item.get('MILL', ''):

                    item_datation = item['MILL']

                elif item.get('EPOQ', ''):

                    item_datation = item['EPOQ']

                item_technics = item['TECH']

                item_field = item['DOM']

                item_measurements = item['DIMS']

                item_create_or_usage_location = item['LIEUX']

                item_discovery_context = item['DECV']

                item_conservation_location = item['LOCA']

                item_photo_credits = item['PHOT']

                item_inventory_number = item['INV']

                item_joconde_ref = item['REF']

                if ItemMetadata.objects.filter(item__collection = collection, natural_key = item_joconde_ref).exists():

                    print('#### An item with '+item['REF']+' for natural key, already exists in database in the import collection')

                    if options['import_folders']:

                        # Extract folder name from natural key

                        m = re.search(options['folders_regexp'], item[options['folders_metadata']])

                        folder_id = m.group(1)

                        if not Folder.objects.filter(original_id=folder_id).exists():

                            print('#### Creating folder "'+folder_id+'"')

                            folder = Folder.objects.create(

                                collection = collection,

                                name = 'Dossier '+folder_id,

                                original_id = folder_id

                            )

                        else:

                            print('#### Folder "'+folder_id+'" already exists')

                            folder = Folder.objects.get(original_id=folder_id)

                        item_metadata = ItemMetadata.objects.get(item__collection = collection, natural_key = item_joconde_ref)

                        item = item_metadata.item

                        item.folders.add(folder)

                else:

                    print('#### Creating item '+item['REF']+' (natural key) in database')

                    item_object = Item.objects.create(

                        collection = collection

                    )

                    new_metadata = {

                        "authors" : item_authors,

                        "school" : item_school,

                        "designation" : item_designation,

                        "field" : item_field,

                        "datation" : item_datation,

                        "technics" : item_technics,

                        "measurements" : item_measurements,

                        "create_or_usage_location" : item_create_or_usage_location,

                        "discovery_context" : item_discovery_context,

                        "conservation_location" : item_conservation_location,

                        "photo_credits" : item_photo_credits,

                        "inventory_number" : item_inventory_number,

                        "joconde_ref" : item_joconde_ref

                    }

                    ItemMetadata.objects.create(

                        item = item_object,

                        metadata = json.dumps(new_metadata),

                        natural_key = item_joconde_ref

                        )

                    print('#### Computing item image(s)')

                    for image in item['SRC_IMG_FILES']:

                        (image_name, ext) = os.path.splitext(image)

                        if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS:

                            print('##### Copying file '+str(image)+' without converting')

                            image_path = os.path.join(target_dir, image)

                            new_image_name = image

                            shutil.copy(os.path.join(source_dir, image), target_dir)

                            try:

                                im = ImagePIL.open(os.path.join(target_dir, image))

                                im_width, im_height = im.size

                            except Exception as e:

                                print(e)

                                continue

                        else:

                            image_path = os.path.join(target_dir, image_name) + '.jpg'

                            new_image_name = image_name+'.jpg'

                            if os.path.isfile(image_path):

                                print('##### A jpeg file already exists in target dir for '+ image)

                                try:

                                    im = ImagePIL.open(image_path)

                                    im_width, im_height = im.size

                                except Exception as e:

                                    print(e)

                                    continue

                            else:

                                jpeg_img_path = image_path

                                try: