src/iconolab_episteme/management/commands/importimages.py
author ymh <ymh.work@gmail.com>
Thu, 28 Jun 2018 18:50:20 +0200
changeset 7 a40fd3990850
parent 3 16fb4f5efa69
permissions -rw-r--r--
skip already processed images
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     1
# -*- coding: UTF-8 -*-
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     2
import json
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     3
import logging
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     4
import os
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     5
import pprint
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     6
import re
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     7
import shutil
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     8
import imghdr
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
     9
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    10
from django.conf import settings
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    11
from django.core.management.base import BaseCommand, CommandError
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    12
from PIL import Image as ImagePIL
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    13
from sorl.thumbnail import get_thumbnail
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    14
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    15
from iconolab.management.commands.importimages import BaseImportImagesCommand
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    16
from iconolab.models import (Collection, Folder, Image, ImageStats, Item,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    17
                             ItemMetadata, MetaCategory)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    18
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    19
if settings.IMPORT_LOGGER_NAME and settings.LOGGING['loggers'].get(settings.IMPORT_LOGGER_NAME, ''):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    20
    logger = logging.getLogger(settings.IMPORT_LOGGER_NAME)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    21
else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    22
    logger = logging.getLogger(__name__)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    23
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    24
class Command(BaseImportImagesCommand):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    25
    help = 'import images from a directory into the media folder and creates item and image objects'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    26
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    27
    def add_arguments(self, parser):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    28
        parser.add_argument('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    29
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    30
            '--collection-id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    31
            dest='collection_id',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    32
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    33
            help='insert extracted data into the specified collection instead of trying to load a collection fixture',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    34
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    35
        parser.add_argument(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    36
            '--no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    37
            dest='no-jpg-conversion',
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    38
            default=False,
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    39
            help='use this option if you only want the image copied and not converted'
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    40
        )
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    41
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    42
    def handle(self, *args, **options):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    43
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    44
        #Set no image size limit to PIL to be able to process big images.
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    45
        ImagePIL.MAX_IMAGE_PIXELS = None
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    46
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    47
        print('# Logging with logger '+logger.name)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    48
        logger.debug('# Initializing command with args: %r', options)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    49
      
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    50
        self.source_dir = options.get('source_dir')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    51
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    52
        collection_id = options.get('collection_id')
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    53
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    54
        if not collection_id:
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    55
            raise CommandError("No collection id, aborting")
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    56
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    57
        print('## Finding collection with id %s' % collection_id) 
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    58
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    59
        try:
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    60
            try:
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    61
                collection = Collection.objects.get(pk=int(collection_id))
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    62
            except ValueError:
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    63
                collection = Collection.objects.get(name=collection_id)
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    64
        except Collection.DoesNotExist:
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    65
            raise CommandError('!!! Collection with id ' + collection_id
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    66
                                +' was not found, aborting !!!')
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    67
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    68
        print(
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    69
            '## Converting image and moving it to static dir, creating Image and Item objects')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    70
        print('### Images will be stored in ' + os.path.join(settings.MEDIA_ROOT,'uploads'))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    71
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    72
        for dirname, dirs, files in os.walk(self.source_dir):
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    73
            for filename in files:
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    74
                print("::Examining %s" % filename)
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    75
                filename_without_extension, extension = os.path.splitext(filename)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    76
                if imghdr.what(os.path.join(dirname, filename)) is None:
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    77
                    print("-> This is not an image: continue")
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    78
                    continue
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    79
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    80
                json_path = os.path.join(dirname, filename_without_extension + ".json")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    81
                if not os.path.isfile(json_path):
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    82
                    print("-> has not a matching json: continue")
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    83
                    continue
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    84
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    85
                print("-> Processing %s" %json_path)
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    86
                with open(json_path) as json_data:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    87
                    eso_data = json.load(json_data)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    88
                    eso_object = eso_data['object']
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    89
                    eso_image = eso_data['image']
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    90
                
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    91
                    path_images = os.path.join(filename_without_extension, filename)
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    92
                    image_list = [path_images]
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    93
                    image_dir = filename_without_extension
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    94
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    95
                    natural_key = ItemMetadata.get_natural_key(collection, eso_image['id'])
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    96
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    97
                    if ItemMetadata.objects.filter(item__collection=collection, natural_key=natural_key).exists():
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    98
                        print('#### An item with ' +
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
    99
                            natural_key +' for natural key, already exists in database in the import collection')
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   100
                    else:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   101
                        try:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   102
                            os.mkdir(os.path.join(settings.MEDIA_ROOT,'uploads', image_dir))
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   103
                            print(image_dir, "directory created")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   104
                        except FileExistsError:
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   105
                            print(image_dir, "directory already exists")
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   106
3
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   107
                        try:
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   108
                            self.create_item_and_metadata(
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   109
                                natural_key, collection, eso_data, image_list, options, self.source_dir)
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   110
                        except Exception as e:
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   111
                            print("!!! Exception processing %s : %s" % (json_path, e))
16fb4f5efa69 * add scraping script
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   112
                            continue
0
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   113
df27f9610c82 Creating iconolab episteme project
Riwad Salim
parents:
diff changeset
   114
        print('# All done!')