src/ldtplatform/management/commands/replacedelete.py
author bellierp
Fri, 24 Mar 2017 15:03:50 +0100
changeset 332 324717f075f9
parent 331 509b57af708d
child 333 77b56a7aaa7e
permissions -rw-r--r--
better informations about objects changed
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
'''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
List flv and f4v medias, replace them with mp4 urls and update the content and projects.
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
'''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
import csv
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
import logging
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
     6
import re
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
from itertools import chain
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
import requests
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.conf import settings
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from django.contrib.sites.models import Site
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
from django.core.management.base import BaseCommand
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    13
from django.core import management
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
from lxml import etree
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
from ldt.ldt_utils import models
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
#this function replace bad suffixs and prefixs of some media URL
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
#by a new one, beginning with "http" and ending with ".mp4"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
def tohttps(source, vidpath, tomp4=1):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    to https
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
    '''
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    25
    if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4'\
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    26
        or not re.match(r".*\..{3}$", source):
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
        tomp4 = 0
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
    if tomp4 == 1:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
        source = source[0:len(source)-3]+"mp4"
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    30
    if source[0:5] == "https":
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    31
        return source
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    32
    elif source[0:4] == "http" or source[0:4] == "sftp":
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
        return "https"+source[4:len(source)]
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
    elif source[0:7] == "/video/":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
        return "https://media.iri.centrepompidou.fr"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
    elif source[0:6] == "video/" or source[0:6] == "audio/":
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
        return "https://media.iri.centrepompidou.fr/"+source
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
    elif vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/video/regardssignes/' or \
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
        vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/regardssignes/':
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
        return "https://media.iri.centrepompidou.fr/video/regardssignes/"+source
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    41
    elif source[0:4] == "mp4:":
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    42
        if vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/':
330
61a9b0c4dac4 modify sources of mp4: files without extension
bellierp
parents: 327
diff changeset
    43
            if re.match(r".*\..{3}$", source):
61a9b0c4dac4 modify sources of mp4: files without extension
bellierp
parents: 327
diff changeset
    44
                return "https://media.iri.centrepompidou.fr/" + source[4:]
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    45
            return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4"
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    46
    return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
330
61a9b0c4dac4 modify sources of mp4: files without extension
bellierp
parents: 327
diff changeset
    48
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
def numberofcontents(source):    #this counts the number of contents linked to a media
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
    numberofcontents
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
    return len(models.Content.objects.filter(media_obj_id=source.id))
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
def numberofproject(source):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
    numberofproject
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
    if numberofcontents(source) > 0:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
        return len(models.Project.objects.filter\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
                (content=models.Content.objects.filter(media_obj_id=source.id)[0]))
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
    62
    return 0
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    64
def constructytembed(source):
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    65
    '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    66
    construct youtube video oembed link
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    67
    '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    68
    if re.match(r".*feature=player_embedded.+", source) != None:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    69
        return "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    70
        + source[len(source)-11:] +"&format=json"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    71
    return "http://www.youtube.com/oembed?url=" + source + "&format=json"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    72
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
class Command(BaseCommand):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
    Command class
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
    '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
    help = 'delete medias without contents, replace media\'s source by a new URL'
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    78
    base_url = Site.objects.get_current().domain + settings.BASE_URL
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    79
    parser = etree.XMLParser(encoding='utf-8')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    80
    logger = logging.getLogger(__name__)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    81
    csvfile = open('mediaInformations.csv', 'wb')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    82
    mycsvfile = csv.writer(csvfile)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    83
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    84
    def constructldtembed(self, ldtid):
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    85
        '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    86
        construct ldt embed
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    87
        '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    88
        return "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    89
                   "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    90
                   "player_id=player_project_{ldt_id}&" \
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    91
                   "ldt_id={ldt_id}".format(base_url=Command.base_url, ldt_id=ldtid)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    92
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    93
    def cleanmediaproject(self, element, force, newsrc=None):
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    94
        '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    95
        change media objects' videopath and source if necessary
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    96
        change project .ldt
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    97
        '''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    98
        basesrc = element.src
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
    99
        if force:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   100
            element.videopath = ''
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   101
            element.save()
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   102
        if newsrc != None:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   103
            if force:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   104
                element.src = newsrc
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   105
                element.save()
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   106
            Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   107
                "Media",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   108
                basesrc,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   109
                "Yes",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   110
                "changing source/videopath",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   111
                newsrc, "\'\'"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   112
                ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   113
        if numberofproject(element) == 0:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   114
            Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   115
                "Project",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   116
                element.src,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   117
                "Yes",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   118
                "initializing object(no project)"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   119
                ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   120
            if force:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   121
                mycontentid = models.Content.objects.filter(media_obj_id=element.id)[0].iri_id
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   122
                try:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   123
                    management.call_command('initfrontproject', mycontentid)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   124
                except Exception:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   125
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   126
                    "Project",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   127
                    element.src,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   128
                    "No",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   129
                    "socket error"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   130
                    ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   131
                    return
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   132
                self.stdout.write(" Initializing project", ending='')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   133
            else:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   134
                self.stdout.write(" Project has to be initialized ", ending='')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   135
                return
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   136
        ldtproj = models.Project.objects.filter\
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   137
        (content=models.Content.objects.filter(media_obj_id=element.id)[0])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   138
        for singleproject in ldtproj:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   139
            root = etree.XML(singleproject.ldt.encode('utf-8'), Command.parser)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   140
            if root.xpath('medias/media') == []:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   141
                self.stdout.write(" le .ldt ne contient pas de media", ending='')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   142
                continue
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   143
            if root.xpath('medias/media')[0].get("video") != '':
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   144
                embedurl = self.constructldtembed(singleproject.ldt_id)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   145
                if force:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   146
                    root.xpath('medias/media')[0].set("video", '')
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   147
                self.stdout.write(" changing videopath arg in .ldt ")
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   148
                Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   149
                    "Project",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   150
                    embedurl,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   151
                    "Yes",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   152
                    "changing .ldt /medias/media/video",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   153
                    "\'\'"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   154
                    ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   155
                singleproject.ldt = etree.tostring(root)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   156
                singleproject.save()
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   157
                Command.logger.info("%s DONE\n", embedurl)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   158
        element.save()
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   159
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
    def add_arguments(self, parser):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        add arguments
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        parser.add_argument('-f', action='store_true')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
    def handle(self, *args, **options):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
        '''
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
        handle
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
        '''
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   170
        Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   171
            "Object type",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   172
            "which object",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
            "Change ?",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   174
            "What(if Y)/Why (if N)",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   175
            "How"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   176
            ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   177
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   178
        force = bool(options['f'])
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   179
        j = 0
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   180
        files1 = models.Media.objects.all() #this list contains every media
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   181
        for elem1 in files1:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   182
            if numberofcontents(elem1) == 0:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   183
                if force:
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   184
                    elem1.delete()  #if there is no content
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   185
                    #linked to the media, the media is removed for the database
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   186
                    self.stdout.write(" No content found, media has been removed")
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   187
                else:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   188
                    self.stdout.write(" No content found, media will be removed")
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   189
                Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   190
                    "Media",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   191
                    elem1.src,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   192
                    "Yes",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   193
                    "deleting object (no content)"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   194
                    ])
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   195
                j += 1
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   196
                continue
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   197
            if elem1.src.lower() == tohttps(elem1.src, elem1.videopath).lower():
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   198
                self.cleanmediaproject(elem1, force)
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   199
            if re.match(r".*\.youtube\.com.*", elem1.src) != None\
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   200
            or re.match(r".*youtu\.be.+", elem1.src) != None:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   201
                myembed = constructytembed(elem1.src)
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   202
                if requests.get(myembed).status_code == 404:
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   203
                    self.stdout.write("%s : Video doesn't exists"% elem1.src)
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   204
                    if numberofproject(elem1) > 0:
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   205
                        ldtproj = models.Project.objects.get(id=models.Content.objects.filter\
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   206
                        (media_obj_id=elem1.id)[0].front_project_id).ldt
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   207
                        root = etree.XML(ldtproj.encode('utf-8'), Command.parser)
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   208
                        if root.xpath('annotations/content/ensemble/decoupage/elements/element')\
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   209
                        == []:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   210
                            if force:
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   211
                                elem1.delete()
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   212
                                self.stdout.write("video doesn't exist anymore : media deleted")
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   213
                            Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   214
                                "Media/Content/Project",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   215
                                elem1.src,
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   216
                                "Yes",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   217
                                "deleting(Video doesn't exist anymore + empty projects)"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   218
                                ])
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   219
                            j += 1
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   220
                else:
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   221
                    self.cleanmediaproject(elem1,force)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   222
        if force:
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   223
            self.stdout.write("%s files deleted"%j)
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   224
        else:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   225
            self.stdout.write("%s files to delete"%j)
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   226
        i = 0
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   227
        files = list(chain(
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   228
            models.Media.objects.filter(src__iregex=r".*.flv$"),
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   229
            models.Media.objects.filter(src__iregex=r".*.f4v$"),
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   230
            models.Media.objects.filter(src__iregex=r".*.m4v$"),
330
61a9b0c4dac4 modify sources of mp4: files without extension
bellierp
parents: 327
diff changeset
   231
            models.Media.objects.filter(src__iregex=r".*.mp4$").exclude(src__iregex=r"^https://.*"),
61a9b0c4dac4 modify sources of mp4: files without extension
bellierp
parents: 327
diff changeset
   232
            models.Media.objects.filter(src__iregex=r"^mp4:.*").exclude(src__iregex=r".*\..{3}$")
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   233
        ))
327
c3998a528fd9 delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents: 324
diff changeset
   234
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   235
        for elem in files:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   236
            self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   237
            i += 1
324
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   238
            if numberofcontents(elem) == 0:
dffa18e6cba4 add list of files without contents in mediaInformations.csv
bellierp
parents: 323
diff changeset
   239
                continue
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   240
            mysrc = elem.src
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   241
            newsource = tohttps(elem.src, elem.videopath)
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   242
            try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   243
                res = requests.head(newsource, timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   244
            except requests.ConnectionError:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   245
                self.stdout.write(" connection error", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   246
                Command.logger.error("CONNECTION ERROR FOR %s", elem.title)
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   247
                try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   248
                    res = requests.head(elem, timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   249
                except requests.ConnectionError:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   250
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   251
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   252
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   253
                        "No",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   254
                        "connection error",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   255
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   256
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   257
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   258
                except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   259
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   260
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   261
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   262
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   263
                        "missing schema on base source!",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   264
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   265
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   266
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   267
                except requests.exceptions.Timeout:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   268
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   269
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   270
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   271
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   272
                        "TIMEOUT!",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   273
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   274
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   275
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   276
                else:
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   277
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   278
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   279
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   280
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   281
                        "use source link : website doesn't work with https",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   282
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   283
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   284
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   285
            except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   286
                self.stdout.write(" Missing schema !", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   287
                Command.logger.warning("MISSING SCHEMA FOR %s", elem.title)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   288
                Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   289
                    "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   290
                    mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   291
                    "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   292
                    "missing schema!",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   293
                    newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   294
                    ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   295
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   296
            except requests.exceptions.Timeout:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   297
                self.stdout.write(" Timeout !", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   298
                Command.logger.warning("Timeout FOR %s", elem.title)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   299
                Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   300
                    "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   301
                    mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   302
                    "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   303
                    "TIMEOUT!",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   304
                    newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   305
                    ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   306
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   307
            if res > 400:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   308
                try:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   309
                    ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   310
                    timeout=10).status_code
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   311
                except (requests.exceptions.Timeout, requests.ConnectionError):
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   312
                    self.stdout.write(" can't access source/new files", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   313
                    Command.logger.warning("can't access %s", elem.title)
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   314
                    res = "connection error"
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   315
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   316
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   317
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   318
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   319
                        "website doesn't exist anymore",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   320
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   321
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   322
                    continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   323
                if ressrc == 404:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   324
                    self.stdout.write(" can't access source/new files", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   325
                    Command.logger.warning("can't access %s", elem.title)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   326
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   327
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   328
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   329
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   330
                        "can't access source/new files",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   331
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   332
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   333
                elif ressrc == 200:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   334
                    self.stdout.write(
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   335
                        " file not transcoded yet :"
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   336
                        "keep source extension or wait transcoding to be done",
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   337
                        ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   338
                    Command.logger.warning("%s not transcoded yet", elem.title)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   339
                    Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   340
                        "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   341
                        mysrc,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   342
                        "No",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   343
                        "file not transcoded yet : keep source extension",
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   344
                        newsource
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   345
                        ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   346
                continue
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   347
            self.stdout.write(" It works", ending='')
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   348
            alreadyin = False
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   349
            for everyelem in models.Media.objects.all():
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   350
                if newsource == everyelem.src:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   351
                    alreadyin = True
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   352
                    break
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   353
            if alreadyin:
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   354
                self.stdout.write(" element already in table", ending='')
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   355
                Command.logger.warning("%s already in table", elem.title)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   356
                Command.mycsvfile.writerow([
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   357
                    "Media",
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   358
                    newsource,
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   359
                    "No",
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   360
                    "new source already in table"
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   361
                    ])
323
bdffbceb0730 add media migration command
ymh <ymh.work@gmail.com>
parents:
diff changeset
   362
                continue
332
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   363
            self.cleanmediaproject(elem, force, newsource)
324717f075f9 better informations about objects changed
bellierp
parents: 331
diff changeset
   364
        Command.csvfile.close()