src/iconolab_episteme/management/commands/importimages.py
author ymh <ymh.work@gmail.com>
Wed, 27 Jun 2018 16:00:29 +0200
changeset 1 3b0a8a6e685e
parent 0 df27f9610c82
child 3 16fb4f5efa69
permissions -rw-r--r--
* Move importcollection and importmetacategories commands to the generic project * Add site synchronisation script * Add some test data * remove dev settings * Add setup file to build the application * update .hgignore
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     1
# -*- coding: UTF-8 -*-
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     2
import json
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     3
import logging
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     4
import os
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     5
import pprint
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     6
import re
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     7
import shutil
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     8
import imghdr
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     9
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    10
from django.conf import settings
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    11
from django.core.management.base import BaseCommand, CommandError
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    12
from PIL import Image as ImagePIL
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    13
from sorl.thumbnail import get_thumbnail
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    14
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    15
from iconolab.management.commands.importimages import BaseImportImagesCommand
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    16
from iconolab.models import (Collection, Folder, Image, ImageStats, Item,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    17
                             ItemMetadata, MetaCategory)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    18
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    19
if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    20
    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    21
else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    22
    logger = logging.getLogger(__name__)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    23
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    24
class Command(BaseImportImagesCommand):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    25
    help = 'import images from a directory into the media folder and creates item and image objects'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    26
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    27
    def add_arguments(self, parser):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    28
        parser.add_argument('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    29
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    30
            '--encoding',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    31
            dest='encoding',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    32
            default='utf-8',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    33
            help='JSON file encoding'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    34
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    35
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    36
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    37
            '--collection-id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    38
            dest='collection_id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    39
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    40
            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    41
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    42
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    43
            '--no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    44
            dest='no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    45
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    46
            help='use this option if you only want the image copied and not converted'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    47
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    48
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    49
            '--folders',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    50
            dest='import_folders',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    51
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    52
            action='store_const',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    53
            const=True,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    54
            help='option to create folders'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    55
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    56
        # parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    57
        #     '--folders-regexp',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    58
        #     dest='folders_regexp',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    59
        #     default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    60
        #     help='regexp used to extract the folder name/number'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    61
        # )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    62
        # parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    63
        #     '--folders-metadata',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    64
        #     dest='folders_metadata',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    65
        #     default='REF',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    66
        #     help='metadata from which to extract the folder name/number'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    67
        # )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    68
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    69
    def handle(self, *args, **options):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    70
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    71
        print('# Logging with logger '+logger.name)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    72
        logger.debug('# Initializing command with args: %r', options)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    73
      
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    74
        self.source_dir = options.get('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    75
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    76
        if options.get('collection_id'):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    77
            print('## Finding collection with id ' + 
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    78
                    options.get('collection_id'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    79
            try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    80
                collection = Collection.objects.get(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    81
                    pk=options.get('collection_id'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    82
            except Collection.DoesNotExist:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    83
                raise ValueError('!!! Collection with primary key ' +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    84
                                    options.get('collection_id')+' was not found, aborting !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    85
        else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    86
            raise ValueError(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    87
                '!!! No collection fixture or collection id, aborting because we can\'t properly generate data. !!!')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    88
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    89
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    90
        '''Listing image files in target directory'''
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    91
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    92
        print(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    93
            '## Converting image and moving it to static dir, creating Image and Item objects')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    94
        print('### Images will be stored in ' + os.path.join(settings.MEDIA_ROOT,'uploads'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    95
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    96
        for dirname, dirs, files in os.walk(self.source_dir):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    97
            for filename in files:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    98
                filename_without_extension, extension = os.path.splitext(filename)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    99
                if imghdr.what(os.path.join(dirname, filename)) is None:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   100
                    continue
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   101
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   102
                json_path = os.path.join(dirname, filename_without_extension + ".json")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   103
                if not os.path.isfile(json_path):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   104
                    continue
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   105
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   106
                with open(json_path) as json_data:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   107
                    eso_data = json.load(json_data)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   108
                    eso_object = eso_data['object']
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   109
                    eso_image = eso_data['image']
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   110
                
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   111
                    path_images = os.path.join(filename_without_extension, filename)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   112
                    image_list = [path_images]
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   113
                    image_dir = filename_without_extension
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   114
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   115
                    natural_key = ItemMetadata.get_natural_key(collection, eso_image['id'])
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   116
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   117
                    if ItemMetadata.objects.filter(item__collection=collection, natural_key=natural_key).exists():
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   118
                        print('#### An item with ' +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   119
                            natural_key +' for natural key, already exists in database in the import collection')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   120
                    else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   121
                        try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   122
                            os.mkdir(os.path.join(settings.MEDIA_ROOT,'uploads', image_dir))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   123
                            print(image_dir, "directory created")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   124
                        except FileExistsError:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   125
                            print(image_dir, "directory already exists")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   126
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   127
                        self.create_item_and_metadata(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   128
                            natural_key, collection, eso_data, image_list, options, self.source_dir)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   129
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   130
        print('# All done!')