src/ldtplatform/management/commands/replacedelete.py
author bellierp
Tue, 07 Mar 2017 16:49:31 +0100
changeset 324 dffa18e6cba4
parent 323 bdffbceb0730
child 327 c3998a528fd9
permissions -rw-r--r--
add list of files without contents in mediaInformations.csv
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
'''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
List flv and f4v medias, replace them with mp4 urls and update the content and projects.
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
'''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
import csv
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
import logging
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
from itertools import chain
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
import requests
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
from django.conf import settings
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.contrib.sites.models import Site
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from django.core.management.base import BaseCommand
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
from lxml import etree
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
from ldt.ldt_utils import models
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
#this function replace bad suffixs and prefixs of some media URL
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
#by a new one, beginning with "http" and ending with ".mp4"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
def tohttps(source, vidpath, tomp4=1):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
    to https
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4':
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
        tomp4 = 0
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    if tomp4 == 1:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
        source = source[0:len(source)-3]+"mp4"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    if source[0:4] == "http" or source[0:4] == "sftp":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
        return "https"+source[4:len(source)]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
    elif source[0:7] == "/video/":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
        return "https://media.iri.centrepompidou.fr"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
    elif source[0:6] == "video/" or source[0:6] == "audio/":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
        return "https://media.iri.centrepompidou.fr/"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
    elif source[0:4] == "mp4:":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
        return "hue"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
    elif vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/video/regardssignes/' or \
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
        vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/regardssignes/':
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
        return "https://media.iri.centrepompidou.fr/video/regardssignes/"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
    else:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
        return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
def numberofcontents(source):    #this counts the number of contents linked to a media
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
    numberofcontents
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
    return len(models.Content.objects.filter(media_obj_id=source.id))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
def numberofproject(source):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
    numberofproject
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
    if numberofcontents(source) > 0:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
        return len(models.Project.objects.filter\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
                (content=models.Content.objects.filter(media_obj_id=source.id)[0]))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
    else:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
        return 0
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
class Command(BaseCommand):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
    Command class
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
    help = 'delete medias without contents, replace media\'s source by a new URL'
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
    def add_arguments(self, parser):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
        add arguments
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        parser.add_argument('-f', action='store_true')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
    def handle(self, *args, **options):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
        handle
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
        forcsv = [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
            "Source link",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
            "Tried link",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
            "Change ?",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
            "Request status",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
            "Informations",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
            "Number of contents",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
            "Number of projects"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
            ]]
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    81
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    82
        j=0
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    83
        files1 = models.Media.objects.all() #this list contains every media
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    84
        for elem1 in files1:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    85
            if numberofcontents(elem1) == 0:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    86
                if options['f']:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    87
                    elem1.delete()  #if there is no content
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    88
                    #linked to the media, the media is removed for the database
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    89
                    self.stdout.write(" No content found, media has been removed")
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    90
                else:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    91
                    forcsv += [[
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    92
                    elem1.src,
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    93
                    "XXX",
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    94
                    "XXX",
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    95
                    "XXX",
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    96
                    "NO CONTENT : media will be erased",
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    97
                    "0",
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    98
                    "0"
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
    99
                    ]]
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   100
                j += 1
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   101
        if options['f']:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   102
            self.stdout.write("%s files deleted"%j)
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   103
            infos = open('results.txt', 'w')
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   104
            parser = etree.XMLParser(encoding='utf-8')
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   105
        else:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   106
            self.stdout.write("%s files to delete"%j)
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   107
            infos = open('projectsToChange.txt', 'w')
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
        i = 0
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        files = list(chain(
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
            models.Media.objects.filter(src__iregex=r".*.flv$"),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
            models.Media.objects.filter(src__iregex=r".*.f4v$"),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
            models.Media.objects.filter(src__iregex=r".*.mp4$").exclude(src__iregex=r"^https://.*")
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
        ))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
        logger = logging.getLogger(__name__)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
        for elem in files:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
            self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
            i += 1
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   118
            if numberofcontents(elem) == 0:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   119
                self.stdout.write(" no content",ending='')
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   120
                continue
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
            mysrc = elem.src
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
            newsource = tohttps(elem.src, elem.videopath)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
            try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
                res = requests.head(newsource, timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
            except requests.ConnectionError:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
                self.stdout.write(" connection error", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
                logger.error("CONNECTION ERROR FOR %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
                try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
                    res = requests.head(elem, timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
                except requests.ConnectionError:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
                    res = "connection error"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
                        res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
                        ": website doesn't exist anymore",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
                        ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
                except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
                        "XXX",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
                        "missing schema on base source!",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
                        ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
                except requests.exceptions.Timeout:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   154
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
                        "XXX",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
                        "TIMEOUT!",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
                        ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
                else:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
                        res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   170
                        "use source link : website doesn't work with https",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   171
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   172
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
                        ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
            except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
                self.stdout.write(" Missing schema !", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
                logger.warning("MISSING SCHEMA FOR %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
                forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
                    mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
                    newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   181
                    0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   182
                    "XXX",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   183
                    "missing schema!",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   184
                    numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   185
                    numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   186
                ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   187
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   188
            except requests.exceptions.Timeout:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   189
                self.stdout.write(" Timeout !", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   190
                logger.warning("Timeout FOR %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   191
                forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   192
                    mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   193
                    newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   194
                    0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   195
                    "XXX",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   196
                    "TIMEOUT!",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   197
                    numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   198
                    numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   199
                ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   200
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   201
            if res > 400:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   202
                try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   203
                    ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   204
                    timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   205
                except (requests.exceptions.Timeout, requests.ConnectionError):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   206
                    self.stdout.write(" can't access source/new files", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   207
                    logger.warning("can't access %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   208
                    res = "connection error"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   209
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   210
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   211
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   212
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   213
                        res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   214
                        "website doesn't exist anymore",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   215
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   216
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   217
                    ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   218
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   219
                if ressrc == 404:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   220
                    self.stdout.write(" can't access source/new files", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   221
                    logger.warning("can't access %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   222
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   223
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   224
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   225
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   226
                        res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   227
                        "can't access source/new files",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   228
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   229
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   230
                    ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   231
                elif ressrc == 200:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   232
                    self.stdout.write(
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   233
                        " file not transcoded yet :"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   234
                        "keep source extension or wait transcoding to be done",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   235
                        ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   236
                    logger.warning("%s not transcoded yet", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   237
                    forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   238
                        mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   239
                        newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   240
                        0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   241
                        res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   242
                        "file not transcoded yet : keep source extension",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   243
                        numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   244
                        numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   245
                    ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   246
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   247
            self.stdout.write(" It works", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   248
            if options['f']:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   249
                elem.src = newsource
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   250
            alreadyin = False
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   251
            for everyelem in models.Media.objects.all():
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   252
                if newsource == everyelem.src:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   253
                    alreadyin = True
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   254
                    break
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   255
            if alreadyin:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   256
                self.stdout.write(" element already in table", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   257
                logger.warning("%s already in table", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   258
                forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   259
                    mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   260
                    newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   261
                    0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   262
                    res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   263
                    "element already in table",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   264
                    numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   265
                    numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   266
                ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   267
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   268
            if numberofcontents(elem) == 0:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   269
                self.stdout.write(" source has no content", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   270
                forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   271
                    mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   272
                    newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   273
                    0,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   274
                    res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   275
                    "source has no content",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   276
                    numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   277
                    numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   278
                ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   279
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   280
            if options['f']:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   281
                elem.videopath = ''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   282
                elem.save()
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   283
            ldtproj = models.Project.objects.filter\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   284
            (content=models.Content.objects.filter(media_obj_id=elem.id)[0])
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   285
            if numberofproject(elem) == 0:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   286
                self.stdout.write(" no project", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   287
                logger.warning("NO PROJECT FOR %s", elem.title)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   288
                forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   289
                    mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   290
                    newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   291
                    1,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   292
                    res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   293
                    "It works but there is no project with this media",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   294
                    numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   295
                    numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   296
                ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   297
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   298
            infos.write("\nProjects : \n")
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   299
            for numproject in xrange(numberofproject(elem)):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   300
                base_url = Site.objects.get_current().domain + settings.BASE_URL
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   301
                ldt_id = ldtproj[numproject].ldt_id
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   302
                embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   303
                           "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   304
                           "player_id=player_project_{ldt_id}&" \
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   305
                           "ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   306
                infos.write("%s \n"%(embedurl))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   307
            infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   308
            % (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   309
            if options['f']:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   310
                for numproject in xrange(len(ldtproj)):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   311
                    root = etree.XML(ldtproj[numproject].ldt.encode('utf-8'), parser)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   312
                    if len(root.xpath('medias/media')) == 0:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   313
                        self.stdout.write(" le .ldt ne contient pas de media", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   314
                        infos.write("le .ldt ne contient pas de media")
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   315
                        continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   316
                    root.xpath('medias/media')[0].set("video", '')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   317
                    ldtproj[numproject].ldt = etree.tostring(root)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   318
                    ldtproj[numproject].save()
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   319
                    infos.write("\nSuccessful !\n")
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   320
                    logger.info("%s DONE\n", embedurl)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   321
            self.stdout.write(" done", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   322
            forcsv += [[
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   323
                mysrc,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   324
                newsource,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   325
                1,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   326
                res,
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   327
                "It works",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   328
                numberofcontents(elem),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   329
                numberofproject(elem)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   330
            ]]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   331
        if not options['f']:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   332
            with open('mediaInformations.csv', 'wb') as csvfile:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   333
                mycsvfile = csv.writer(csvfile)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   334
                for mycsv in forcsv:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   335
                    mycsvfile.writerow(mycsv)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   336
        infos.close()