src/iconolab_episteme/management/commands/importcollection.py
author Riwad Salim
Tue, 26 Jun 2018 15:55:08 +0200
changeset 0 df27f9610c82
permissions -rw-r--r--
Creating iconolab episteme project
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     1
# -*- coding: UTF-8 -*-
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     2
import json
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     3
import logging
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     4
import os
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     5
import pprint
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     6
import re
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     7
import shutil
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     8
import imghdr
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     9
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    10
from django.conf import settings
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    11
from django.core.management.base import BaseCommand, CommandError
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    12
from PIL import Image as ImagePIL
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    13
from sorl.thumbnail import get_thumbnail
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    14
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    15
from iconolab.management.commands.importimages import BaseImportImagesCommand
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    16
from iconolab.models import (Collection, Folder, Image, ImageStats, Item,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    17
                             ItemMetadata, MetaCategory)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    18
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    19
if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    20
    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    21
else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    22
    logger = logging.getLogger(__name__)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    23
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    24
class Command(BaseImportImagesCommand):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    25
    help = 'import collection image and file from a directory'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    26
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    27
    def add_arguments(self, parser):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    28
        parser.add_argument('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    29
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    30
            '--encoding',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    31
            dest='encoding',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    32
            default='utf-8',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    33
            help='JSON file encoding'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    34
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    35
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    36
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    37
            '--collection-json',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    38
            dest='collection_json',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    39
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    40
            help='creates a new collection from a json file, must be an object with fields : '+ 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    41
                 '"name" (identifier), '+ 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    42
                 '"verbose_name" (proper title name), '+ 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    43
                 '"description" (description on homepage, html is supported), '+ 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    44
                 '"image" (image on homepages, must be "uploads/<imgname>"), '+ 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    45
                 '"height" and "width" (height and width of the image)',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    46
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    47
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    48
            '--collection-id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    49
            dest='collection_id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    50
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    51
            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    52
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    53
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    54
            '--no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    55
            dest='no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    56
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    57
            help='use this option if you only want the image copied and not converted'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    58
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    59
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    60
    def handle(self, *args, **options):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    61
            
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    62
        print('# Logging with logger '+logger.name)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    63
        logger.debug('# Initializing command with args: %r', options)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    64
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    65
        '''Check we have a collection to store data into'''
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    66
        self.source_dir = options.get('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    67
        print('# Checking collection args')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    68
        if options.get('collection_json'):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    69
            print('## Finding collection json data in '+self.source_dir)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    70
            collection_json_path = os.path.join(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    71
                self.source_dir, options.get('collection_json'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    72
            if not os.path.isfile(collection_json_path):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    73
                print('### No '+options.get('collection_json') +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    74
                        '.json file was found in the source directory')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    75
                raise ValueError(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    76
                    '!!! Json file '+collection_json_path+' was not found !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    77
            try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    78
                with open(collection_json_path) as json_fixture_file:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    79
                    collection_data = json.loads(json_fixture_file.read())
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    80
                    for key in ['name', 'verbose_name', 'description', 'image', 'height', 'width']:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    81
                        if not key in collection_data.keys():
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    82
                            print('!!! Json file '+collection_json_path +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    83
                                    ' has no '+key+' field !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    84
                            raise ValueError()
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    85
                    if not collection_data.get('name', ''):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    86
                        print('!!! Collection data key "name" is empty')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    87
                        raise ValueError()
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    88
                    if Collection.objects.filter(name=collection_data.get('name')).exists():
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    89
                        print(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    90
                            '!!! A Collection with the provided name already exists!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    91
                        raise ValueError()
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    92
                    if collection_data.get('image', '') and not (collection_data.get('width', 0) and collection_data.get('height', 0)):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    93
                        print(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    94
                            '!!! Collection data has an image but no height and width')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    95
                        raise ValueError()
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    96
            except ValueError as e:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    97
                raise ValueError('!!! JSON Data is invalid. !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    98
        elif options.get('collection_id'):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    99
            print('## Finding collection with id ' + 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   100
                    options.get('collection_id'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   101
            try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   102
                collection = Collection.objects.get(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   103
                    pk=options.get('collection_id'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   104
            except Collection.DoesNotExist:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   105
                raise ValueError('!!! Collection with primary key ' +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   106
                                    options.get('collection_id')+' was not found, aborting !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   107
        else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   108
            raise ValueError(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   109
                '!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   110
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   111
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   112
        '''Import image collection in target directory'''
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   113
           
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   114
        if options.get('collection_json'):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   115
            print('## Loading collection json')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   116
            collection = Collection.objects.create(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   117
                name=collection_data.get('name'),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   118
                verbose_name=collection_data.get('verbose_name', ''),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   119
                description=collection_data.get('description', ''),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   120
                image=collection_data.get('image', ''),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   121
                height=collection_data.get('height', 0),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   122
                width=collection_data.get('width', 0),
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   123
            )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   124
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   125
            if collection.image:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   126
                collection_image_path = os.path.join(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   127
                    settings.MEDIA_ROOT, str(collection.image))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   128
                if not os.path.isfile(collection_image_path):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   129
                    print('### Moving collection image')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   130
                    _, collection_image_name = os.path.split(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   131
                        collection_image_path)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   132
                    try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   133
                        col_im = ImagePIL.open(os.path.join(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   134
                            self.source_dir, collection_image_name))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   135
                        print('##### Generating or copying jpeg for ' +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   136
                                collection_image_name)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   137
                        col_im.thumbnail(col_im.size)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   138
                        col_im.save(collection_image_path, 'JPEG', quality=options.get(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   139
                            'jpeg_quality', settings.IMG_JPG_DEFAULT_QUALITY))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   140
                    except Exception as e:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   141
                        print(e)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   142
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   143