# -*- coding: UTF-8 -*-
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.conf import settings
from iconolab.models import Collection, Image, ImageStats, Item, ItemMetadata, MetaCategory, Folder
from PIL import Image as ImagePIL
from sorl.thumbnail import get_thumbnail
import os, csv, pprint, re, json, shutil, logging

if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
else:
    logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = 'import images from a directory into the media folder and creates item and image objects'

    def add_arguments(self, parser):
        parser.add_argument('csv_path')
        parser.add_argument(
            '--jpeg-quality',
            dest='jpeg_quality',
            default=settings.IMG_JPG_DEFAULT_QUALITY,
            help='Jpeg default quality'

        )
        parser.add_argument(
            '--encoding',
            dest='encoding',
            default='utf-8',
            help='CSV file encoding'

        )
        parser.add_argument(
            '--collection-json',
            dest='collection_json',
            default=False,
            help='creates a new collection from a json file, must be an object with fields : '+ \
                 '"name" (identifier), '+ \
                 '"verbose_name" (proper title name), '+ \
                 '"description" (description on homepage, html is supported), '+ \
                 '"image" (image on homepages, must be "uploads/<imgname>"), '+ \
                 '"height" and "width" (height and width of the image)',
        )
        parser.add_argument(
            '--collection-id',
            dest='collection_id',
            default=False,
            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
        )
        parser.add_argument(
            '--metacategories-json',
            dest='metacategories_json',
            default=False,
            help='add metacategories to the collection from a json file (json must be a list of object with "label" and "triggers_notifications" fields)',
        )
        parser.add_argument(
            '--delimiter',
            dest='csv_delimiter',
            default=';',
            help='csv file delimiter'
        )
        parser.add_argument(
            '--no-jpg-conversion',
            dest='no-jpg-conversion',
            default=False,
            help='use this option if you only want the image copied and not converted'
        )
        parser.add_argument(
            '--img-filename-identifier',
            dest='img_filename_identifier',
            default=settings.IMPORT_DEFAULT_FIELD_TO_FILENAME_IDENTIFIER,
            help='codename of the csv field we\'ll try to match to find the related image to a given object'
        )
        parser.add_argument(
            '--filename-regexp-prefix',
            dest='filename_regexp_prefix',
            default=r'.*',
            help='regexp prefix to properly parse image names with info from csv. The pattern should describe the part before the filename identifier string, default is .*'
        )
        parser.add_argument(
            '--filename-regexp-suffix',
            dest='filename_regexp_suffix',
            default=r'[\.\-_].*',
            help='regexp suffix to properly parse image names with info from csv. The pattern should describe the part after the filename identifier string, default is [\.\-_].*'
        )
        parser.add_argument(
            '--folders',
            dest='import_folders',
            default=False,
            action='store_const',
            const=True,
            help='option to create folders'
        )
        parser.add_argument(
            '--folders-regexp',
            dest='folders_regexp',
            default=False,
            help='regexp used to extract the folder name/number'
        )
        parser.add_argument(
            '--folders-metadata',
            dest='folders_metadata',
            default='INV',
            help='metadata from which to extract the folder name/number'
        )
    def handle(self, *args, **options):
        """
            Step-by-step for import:

            1) Argument checks for file existence and database state to check that everything can proceed without issue before reading the files
            1) We import data from csv in a 'pivot' list of dicts 'cleaned_row_data' with the following logic:
                * in the settings, there is value "IMPORT_FIELDS_DICT" that is a dict where each key is an identifier for the metadatas
                to which we associate a list of column header that will identified as that metadata
                * The cleaned_row_data list will associate the identifier with the actual value for its related column
            2) Once we have cleaned_row_data, we filter out rows that don't have any associated image into a 'filtered_row_data' list, and add a key "SRC_IMG_FILES" that contains the list of images associated
            to each row for the filtered data.
            3) At this point we have a list of all the items that will be created into the database and the related images to import, so we create the collection object if necessary
            4) For each item:
                We create the object in the database
                * Metadatas are extracted from the filtered_csv_data using the pivot identifiers from settings.IMPORT_FIELD_DICT
                We copy/convert the image into the MEDIA_ROOT/uploads/ dir: thumbnails size listed in settings.PREGENERATE_THUMBNAIL_SIZES are pre-generated for each image

            Note: each unused row and each unused image in the import folder is kept track of in no_data_images, no_image_rows and duplicate_rows lists and logged at the end of the command.
        """
        try:
            print('# Logging with logger '+logger.name)
            logger.debug('# Initializing command with args: %r', options)
            # Check we have a collection to store data into:
            source_dir = os.path.dirname(os.path.realpath(options.get('csv_path')))
            print('# Checking collection args')
            if options.get('collection_json'):
                print('## Finding collection json data in '+source_dir)
                collection_json_path = os.path.join(source_dir, options.get('collection_json'))
                if not os.path.isfile(collection_json_path):
                    print('### No '+options.get('collection_json')+'.json file was found in the source directory')
                    raise ValueError('!!! Json file '+collection_json_path+' was not found !!!')
                try:
                    with open(collection_json_path) as json_fixture_file:
                        collection_data = json.loads(json_fixture_file.read())
                        for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:
                            if not key in collection_data.keys():
                                print('!!! Json file '+collection_json_path+' has no '+key+' field !!!')
                                raise ValueError()
                        if not collection_data.get('name', ''):
                            print('!!! Collection data key "name" is empty')
                            raise ValueError()
                        if Collection.objects.filter(name=collection_data.get('name')).exists():
                            print('!!! A Collection with the provided name already exists!')
                            raise ValueError()
                        if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):
                            print('!!! Collection data has an image but no height and width')
                            raise ValueError()
                except ValueError as e:
                    raise ValueError('!!! JSON Data is invalid. !!!')
            elif options.get('collection_id'):
                print('## Finding collection with id '+options.get('collection_id'))
                try:
                    collection = Collection.objects.get(pk=options.get('collection_id'))
                except Collection.DoesNotExist:
                    raise ValueError('!!! Collection with primary key '+options.get('collection_id')+' was not found, aborting !!!')
            else:
                raise ValueError('!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')

            if options.get('metacategories_json'):
                print('## Finding metacategories fixture json data in '+source_dir)
                metacategories_json_path = os.path.join(source_dir, options.get('metacategories_json'))
                if not os.path.isfile(metacategories_json_path):
                    print('### No '+options.get('metacategories_json')+'.json file was found in the source directory')
                    raise ValueError('!!! Fixture file '+metacategories_json_path+' was not found !!!')
                with open(metacategories_json_path) as metacategories_json_file:
                    metacategories_data = json.loads(metacategories_json_file.read())
                    for metacategory in metacategories_data:
                        if metacategory.get('label', None) is None:
                            raise ValueError('!!! Metacategory without label !!!')

            if options['import_folders'] and not options['folders_regexp']:
                raise ValueError('!!! No regexp specified to extract folder name !!!')

            # We read the csv
            delimiter = options.get('csv_delimiter')
            if delimiter == '#9':
                delimiter = chr(9)
            if delimiter == '#29':
                delimiter = chr(29)
            if delimiter == '#30':
                delimiter = chr(30)
            if delimiter == '#31':
                delimiter = chr(31)
            csvreader = csv.DictReader(open(options.get('csv_path'), encoding=options.get('encoding')), delimiter=delimiter)
            print('# Extracting data from csv file and storing it in standardized format')
            # We store data using the Jocondelab keys, as defined in settings.IMPORT_FIELDS_DICT
            cleaned_csv_data=[]
            duplicate_rows=[]
            for row in csvreader:
                cleaned_row_data = {}
                for key in settings.IMPORT_FIELDS_DICT.keys():
                    cleaned_row_data[key] = ''
                    for row_key in row.keys():
                        if row_key in settings.IMPORT_FIELDS_DICT[key]:
                            if key == 'INV':
                                inv_number, _, _ = row[row_key].partition(';')
                                cleaned_row_data[key] = inv_number.rstrip()
                            else:
                                cleaned_row_data[key] = row[row_key]
                            break
                if cleaned_row_data[options.get('img_filename_identifier')] in [row[options.get('img_filename_identifier')] for row in cleaned_csv_data]:
                    print("## We already have "+options.get('img_filename_identifier')+" value "+cleaned_row_data[options.get('img_filename_identifier')]+" in the data to import, ignoring duplicate line")
                    duplicate_rows.append(cleaned_row_data)
                else:
                    cleaned_csv_data.append(cleaned_row_data)
            # Listing image files in csv directory
            image_list = [
                f for f in os.listdir(source_dir)
                if os.path.isfile(os.path.join(source_dir, f))
                and (f.endswith('.jpg') or f.endswith('.tif') or f.endswith('.bmp') or f.endswith('.png'))
            ] # Maybe check if image another way
            filtered_csv_data = []
            no_image_rows = []
            no_data_images = []
            assigned_images = []
            # Now we trim the cleaned_csv_data dict to keep only entries that have at least one image
            for item in cleaned_csv_data:
                item['SRC_IMG_FILES'] = []
                has_image = False
                for image in image_list:
                    img_name_pattern = options.get('filename_regexp_prefix')+re.escape(item[options.get('img_filename_identifier')])+options.get('filename_regexp_suffix')
                    if re.match(img_name_pattern, image):
                        item['SRC_IMG_FILES'].append(image)
                        assigned_images.append(image)
                        has_image = True
                if has_image:
                    filtered_csv_data.append(item)
                else:
                    # We keep track of the entries that don't have any corresponding image
                    no_image_rows.append(item)
            # We keep track of the images that don't have any corresponding entry
            for image in image_list:
                if image not in assigned_images:
                    no_data_images.append(image)

            print('## found ' + str(len(filtered_csv_data))+' items with at least one image')
            print('# Importing data into Iconolab')
            if options.get('collection_json'):
                print('## Loading collection json')
                collection = Collection.objects.create(
                    name = collection_data.get('name'),
                    verbose_name = collection_data.get('verbose_name', ''),
                    description = collection_data.get('description', ''),
                    image = collection_data.get('image', ''),
                    height = collection_data.get('height', 0),
                    width = collection_data.get('width', 0),
                )
                if collection.image:
                    collection_image_path = os.path.join(settings.MEDIA_ROOT, str(collection.image))
                    if not os.path.isfile(collection_image_path):
                        print('### Moving collection image')
                        _ , collection_image_name = os.path.split(collection_image_path)
                        try:
                            col_im = ImagePIL.open(os.path.join(source_dir, collection_image_name))
                            print('##### Generating or copying jpeg for '+collection_image_name)
                            col_im.thumbnail(col_im.size)
                            col_im.save(collection_image_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
                        except Exception as e:
                            print(e)
            if options.get('metacategories_json'):
                for metacategory in metacategories_data:
                    MetaCategory.objects.create(
                        collection = collection,
                        label = metacategory.get('label'),
                        triggers_notifications = metacategory.get('triggers_notifications', 0)
                    )
            print('## Converting image and moving it to static dir, creating Image and Item objects')
            target_dir = os.path.join(settings.MEDIA_ROOT, 'uploads')
            print('### Images will be stored in '+target_dir)
            for item in filtered_csv_data:
                print('#### Computing metadatas for item '+item['INV']+' (inv number)')
                if not item['INV']:
                    print('#### No INV number, skipping')
                    continue
                item_authors = item['AUTR']
                item_school = item['ECOLE']
                item_designation = ''
                if item.get('TITR', ''):
                    item_designation = item['TITR']
                elif item.get('DENO', ''):
                    item_designation = item['DENO']
                elif item.get('APPL', ''):
                    item_designation = item['APPL']
                item_datation = ''
                if item.get('PERI', ''):
                    item_datation = item['PERI']
                elif item.get('MILL', ''):
                    item_datation = item['MILL']
                elif item.get('EPOQ', ''):
                    item_datation = item['EPOQ']
                item_technics = item['TECH']
                item_field = item['DOM']
                item_measurements = item['DIMS']
                item_create_or_usage_location = item['LIEUX']
                item_discovery_context = item['DECV']
                item_conservation_location = item['LOCA']
                item_photo_credits = item['PHOT']
                item_inventory_number = item['INV']
                item_joconde_ref = item['REF']
                if ItemMetadata.objects.filter(item__collection = collection, inventory_number = item_inventory_number).exists():
                    print('#### An item with '+item['INV']+' for inventory number, already exists in database in the import collection')

                    if options['import_folders']:

                        # Extract folder name from inventory number
                        m = re.search(options['folders_regexp'], item[options['folders_metadata']])
                        folder_id = m.group(1)

                        if not Folder.objects.filter(original_id=folder_id).exists():
                            print('#### Creating folder "'+folder_id+'"')
                            folder = Folder.objects.create(
                                collection = collection,
                                name = 'Dossier '+folder_id,
                                original_id = folder_id
                            )
                        else:
                            print('#### Folder "'+folder_id+'" already exists')
                            folder = Folder.objects.get(original_id=folder_id)

                        item_metadata = ItemMetadata.objects.get(item__collection = collection, inventory_number = item_inventory_number)
                        item = item_metadata.item

                        item.folders.add(folder)

                else:
                    print('#### Creating item '+item['INV']+' (inv number) in database')
                    item_object = Item.objects.create(
                        collection = collection
                    )
                    ItemMetadata.objects.create(
                        item = item_object,
                        authors = item_authors,
                        school = item_school,
                        designation = item_designation,
                        datation = item_datation,
                        technics = item_technics,
                        measurements = item_measurements,
                        create_or_usage_location = item_create_or_usage_location,
                        field = item_field,
                        discovery_context = item_discovery_context,
                        conservation_location = item_conservation_location,
                        photo_credits = item_photo_credits,
                        inventory_number = item_inventory_number,
                        joconde_ref = item_joconde_ref
                    )
                    print('#### Computing item image(s)')
                    for image in item['SRC_IMG_FILES']:
                        (image_name, ext) = os.path.splitext(image)
                        if options.get('no-jpg-conversion') or ext in settings.NO_IMG_CONVERSION_EXTS:
                            print('##### Copying file '+str(image)+' without converting')
                            image_path = os.path.join(target_dir, image)
                            new_image_name = image
                            shutil.copy(os.path.join(source_dir, image), target_dir)
                            try:
                                im = ImagePIL.open(os.path.join(target_dir, image))
                                im_width, im_height = im.size
                            except Exception as e:
                                print(e)
                                continue
                        else:
                            image_path = os.path.join(target_dir, image_name) + '.jpg'
                            new_image_name = image_name+'.jpg'
                            if os.path.isfile(image_path):
                                print('##### A jpeg file already exists in target dir for '+ image)
                                try:
                                    im = ImagePIL.open(image_path)
                                    im_width, im_height = im.size
                                except Exception as e:
                                    print(e)
                                    continue
                            else:
                                jpeg_img_path = image_path
                                try:
                                    im = ImagePIL.open(os.path.join(source_dir, image))
                                    print('##### Generating or copying jpeg for '+image)
                                    im.thumbnail(im.size)
                                    im.save(jpeg_img_path, 'JPEG', quality=options.get('jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
                                    im_width, im_height = im.size
                                except Exception as e:
                                    print(e)
                                    continue
                        new_image = Image.objects.create(
                            item = item_object,
                            media = 'uploads/'+new_image_name,
                            name = new_image_name,
                            height = im_height,
                            width = im_width
                        )
                        ImageStats.objects.create(
                            image = new_image
                        )
                    print('### Generating thumbnails for item '+item['INV'])
                    for image in item_object.images.all():
                        for size in settings.PREGENERATE_THUMBNAILS_SIZES:
                            print('#### Thumbnail for size '+size)
                            get_thumbnail(image.media, size, crop=False)

            print('# All done!')

            logger.debug('# Recap for import command: ')
            print('# Images without data: ')
            logger.debug('## Checking images left without data')
            collection_image_file = os.path.split(str(collection.image))[1]
            if no_data_images and collection_image_file in no_data_images:
                no_data_images.remove(collection_image_file)

            if no_data_images:
                for image in no_data_images:
                    logger.debug('### %r', image)
                    print('## '+image)
            else:
                print('## Each image has one corresponding row!')
                logger.debug('### Each image has one corresponding row!')
            print('# CSV Items without image')
            logger.debug('## Checking csv rows left without image')
            if no_image_rows:
                for item in no_image_rows:
                    logger.debug('### %r', item['INV'])
                    print('## Inv number: '+item['INV'])
            else:
                print('## Each row found at least one corresponding image!')
                logger.debug('### Each row found at least one corresponding image!')
            print('# Duplicate rows in csv')
            logger.debug('## Checking duplicate rows in csv')
            if duplicate_rows:
                for item in no_image_rows:
                    logger.debug('### %r: %r', options.get('img_filename_identifier'), item[options.get('img_filename_identifier')])
                    print('## '+options.get('img_filename_identifier')+': '+item[options.get('img_filename_identifier')])
            else:
                print('## Each row found at least one corresponding image!')
                logger.debug('### Each row found at least one corresponding image!')
        except FileNotFoundError:
            print('!!! File '+options.get('csv_path')+' does not exist. !!!')
        except ValueError as e:
            print(str(e))
