author | bellierp |
Thu, 30 Mar 2017 16:17:24 +0200 | |
changeset 333 | 77b56a7aaa7e |
parent 332 | 324717f075f9 |
child 334 | 0ddcaaf893e9 |
permissions | -rw-r--r-- |
323 | 1 |
''' |
2 |
List flv and f4v medias, replace them with mp4 urls and update the content and projects. |
|
3 |
''' |
|
4 |
import csv |
|
5 |
import logging |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
6 |
import re |
323 | 7 |
from itertools import chain |
8 |
||
9 |
import requests |
|
10 |
from django.conf import settings |
|
11 |
from django.contrib.sites.models import Site |
|
12 |
from django.core.management.base import BaseCommand |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
13 |
from django.core import management |
323 | 14 |
from lxml import etree |
15 |
||
16 |
from ldt.ldt_utils import models |
|
17 |
||
18 |
||
19 |
#this function replace bad suffixs and prefixs of some media URL |
|
20 |
#by a new one, beginning with "http" and ending with ".mp4" |
|
333 | 21 |
def to_https(source, vidpath, tomp4=1): |
323 | 22 |
''' |
23 |
to https |
|
24 |
''' |
|
332 | 25 |
if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4'\ |
26 |
or not re.match(r".*\..{3}$", source): |
|
323 | 27 |
tomp4 = 0 |
28 |
if tomp4 == 1: |
|
29 |
source = source[0:len(source)-3]+"mp4" |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
30 |
if source[0:5] == "https": |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
31 |
return source |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
32 |
elif source[0:4] == "http" or source[0:4] == "sftp": |
323 | 33 |
return "https"+source[4:len(source)] |
34 |
elif source[0:7] == "/video/": |
|
35 |
return "https://media.iri.centrepompidou.fr"+source |
|
36 |
elif source[0:6] == "video/" or source[0:6] == "audio/": |
|
37 |
return "https://media.iri.centrepompidou.fr/"+source |
|
38 |
elif vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/video/regardssignes/' or \ |
|
39 |
vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/regardssignes/': |
|
40 |
return "https://media.iri.centrepompidou.fr/video/regardssignes/"+source |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
41 |
elif source[0:4] == "mp4:": |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
42 |
if vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/': |
330 | 43 |
if re.match(r".*\..{3}$", source): |
44 |
return "https://media.iri.centrepompidou.fr/" + source[4:] |
|
332 | 45 |
return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4" |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
46 |
return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source |
323 | 47 |
|
330 | 48 |
|
333 | 49 |
def number_of_contents(source): #this counts the number of contents linked to a media |
323 | 50 |
''' |
333 | 51 |
number_of_contents |
323 | 52 |
''' |
53 |
return len(models.Content.objects.filter(media_obj_id=source.id)) |
|
54 |
||
333 | 55 |
def number_of_projects(source): |
323 | 56 |
''' |
333 | 57 |
number_of_projects |
323 | 58 |
''' |
333 | 59 |
if number_of_contents(source) > 0: |
323 | 60 |
return len(models.Project.objects.filter\ |
61 |
(content=models.Content.objects.filter(media_obj_id=source.id)[0])) |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
62 |
return 0 |
323 | 63 |
|
333 | 64 |
def construct_youtube_embed(source): |
332 | 65 |
''' |
66 |
construct youtube video oembed link |
|
67 |
''' |
|
68 |
if re.match(r".*feature=player_embedded.+", source) != None: |
|
69 |
return "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\ |
|
70 |
+ source[len(source)-11:] +"&format=json" |
|
71 |
return "http://www.youtube.com/oembed?url=" + source + "&format=json" |
|
72 |
||
323 | 73 |
class Command(BaseCommand): |
74 |
''' |
|
75 |
Command class |
|
76 |
''' |
|
77 |
help = 'delete medias without contents, replace media\'s source by a new URL' |
|
332 | 78 |
|
333 | 79 |
def add_arguments(self, parser): |
80 |
''' |
|
81 |
add arguments |
|
82 |
''' |
|
83 |
parser.add_argument('-f', |
|
84 |
'--force', |
|
85 |
dest='force', |
|
86 |
action='store_true' |
|
87 |
) |
|
88 |
parser.add_argument('-p', |
|
89 |
'--path', |
|
90 |
dest='path', |
|
91 |
default=None |
|
92 |
) |
|
93 |
||
94 |
||
95 |
def construct_ldt_embed(self, ldtid): |
|
332 | 96 |
''' |
97 |
construct ldt embed |
|
98 |
''' |
|
99 |
return "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ |
|
100 |
"http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ |
|
101 |
"player_id=player_project_{ldt_id}&" \ |
|
333 | 102 |
"ldt_id={ldt_id}".format(base_url=self.base_url, ldt_id=ldtid) |
332 | 103 |
|
333 | 104 |
def clean_media_project(self, element, force, newsrc=None): |
332 | 105 |
''' |
106 |
change media objects' videopath and source if necessary |
|
107 |
change project .ldt |
|
108 |
''' |
|
109 |
basesrc = element.src |
|
110 |
if force: |
|
111 |
element.videopath = '' |
|
112 |
element.save() |
|
113 |
if newsrc != None: |
|
114 |
if force: |
|
115 |
element.src = newsrc |
|
116 |
element.save() |
|
333 | 117 |
self.mycsvfile.writerow([ |
332 | 118 |
"Media", |
119 |
basesrc, |
|
120 |
"Yes", |
|
121 |
"changing source/videopath", |
|
122 |
newsrc, "\'\'" |
|
123 |
]) |
|
333 | 124 |
if number_of_projects(element) == 0: |
125 |
self.mycsvfile.writerow([ |
|
332 | 126 |
"Project", |
127 |
element.src, |
|
128 |
"Yes", |
|
129 |
"initializing object(no project)" |
|
130 |
]) |
|
131 |
if force: |
|
132 |
mycontentid = models.Content.objects.filter(media_obj_id=element.id)[0].iri_id |
|
133 |
try: |
|
134 |
management.call_command('initfrontproject', mycontentid) |
|
135 |
except Exception: |
|
333 | 136 |
self.mycsvfile.writerow([ |
137 |
"Project", |
|
138 |
element.src, |
|
139 |
"No", |
|
140 |
"socket error" |
|
141 |
]) |
|
332 | 142 |
return |
143 |
self.stdout.write(" Initializing project", ending='') |
|
144 |
else: |
|
145 |
self.stdout.write(" Project has to be initialized ", ending='') |
|
146 |
return |
|
147 |
ldtproj = models.Project.objects.filter\ |
|
148 |
(content=models.Content.objects.filter(media_obj_id=element.id)[0]) |
|
149 |
for singleproject in ldtproj: |
|
333 | 150 |
root = etree.XML(singleproject.ldt.encode('utf-8'), self.parser) |
332 | 151 |
if root.xpath('medias/media') == []: |
152 |
self.stdout.write(" le .ldt ne contient pas de media", ending='') |
|
153 |
continue |
|
154 |
if root.xpath('medias/media')[0].get("video") != '': |
|
333 | 155 |
embedurl = self.construct_ldt_embed(singleproject.ldt_id) |
332 | 156 |
if force: |
157 |
root.xpath('medias/media')[0].set("video", '') |
|
333 | 158 |
self.stdout.write(" changing videopath arg in .ldt ", ending='') |
159 |
self.mycsvfile.writerow([ |
|
332 | 160 |
"Project", |
161 |
embedurl, |
|
162 |
"Yes", |
|
163 |
"changing .ldt /medias/media/video", |
|
164 |
"\'\'" |
|
165 |
]) |
|
166 |
singleproject.ldt = etree.tostring(root) |
|
167 |
singleproject.save() |
|
333 | 168 |
self.logger.info("%s DONE\n", embedurl) |
332 | 169 |
element.save() |
170 |
||
333 | 171 |
|
323 | 172 |
|
173 |
def handle(self, *args, **options): |
|
174 |
''' |
|
175 |
handle |
|
176 |
''' |
|
333 | 177 |
self.base_url = Site.objects.get_current().domain + settings.BASE_URL |
178 |
self.parser = etree.XMLParser(encoding='utf-8') |
|
179 |
self.logger = logging.getLogger(__name__) |
|
180 |
path = options['path'] |
|
181 |
force = options['force'] |
|
182 |
if not path: |
|
183 |
path = 'mediaInformations.csv' |
|
184 |
try: |
|
185 |
csvfile = open(path, 'wb') |
|
186 |
except IOError: |
|
187 |
self.stdout.write('file can\'t be opened') |
|
188 |
self.logger.error('cant open file') |
|
189 |
return |
|
190 |
self.mycsvfile = csv.writer(csvfile) |
|
191 |
self.mycsvfile.writerow([ |
|
332 | 192 |
"Object type", |
193 |
"which object", |
|
323 | 194 |
"Change ?", |
332 | 195 |
"What(if Y)/Why (if N)", |
196 |
"How" |
|
197 |
]) |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
198 |
j = 0 |
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
199 |
files1 = models.Media.objects.all() #this list contains every media |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
200 |
for elem1 in files1: |
333 | 201 |
if number_of_contents(elem1) == 0: |
332 | 202 |
if force: |
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
203 |
elem1.delete() #if there is no content |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
204 |
#linked to the media, the media is removed for the database |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
205 |
self.stdout.write(" No content found, media has been removed") |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
206 |
else: |
332 | 207 |
self.stdout.write(" No content found, media will be removed") |
333 | 208 |
self.mycsvfile.writerow([ |
332 | 209 |
"Media", |
210 |
elem1.src, |
|
211 |
"Yes", |
|
212 |
"deleting object (no content)" |
|
213 |
]) |
|
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
214 |
j += 1 |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
215 |
continue |
333 | 216 |
if elem1.src.lower() == to_https(elem1.src, elem1.videopath).lower(): |
217 |
self.clean_media_project(elem1, force) |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
218 |
if re.match(r".*\.youtube\.com.*", elem1.src) != None\ |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
219 |
or re.match(r".*youtu\.be.+", elem1.src) != None: |
333 | 220 |
myembed = construct_youtube_embed(elem1.src) |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
221 |
if requests.get(myembed).status_code == 404: |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
222 |
self.stdout.write("%s : Video doesn't exists"% elem1.src) |
333 | 223 |
if number_of_projects(elem1) > 0: |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
224 |
ldtproj = models.Project.objects.get(id=models.Content.objects.filter\ |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
225 |
(media_obj_id=elem1.id)[0].front_project_id).ldt |
333 | 226 |
root = etree.XML(ldtproj.encode('utf-8'), self.parser) |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
227 |
if root.xpath('annotations/content/ensemble/decoupage/elements/element')\ |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
228 |
== []: |
332 | 229 |
if force: |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
230 |
elem1.delete() |
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
231 |
self.stdout.write("video doesn't exist anymore : media deleted") |
333 | 232 |
self.mycsvfile.writerow([ |
332 | 233 |
"Media/Content/Project", |
234 |
elem1.src, |
|
235 |
"Yes", |
|
236 |
"deleting(Video doesn't exist anymore + empty projects)" |
|
237 |
]) |
|
238 |
j += 1 |
|
239 |
else: |
|
333 | 240 |
self.clean_media_project(elem1, force) |
332 | 241 |
if force: |
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
242 |
self.stdout.write("%s files deleted"%j) |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
243 |
else: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
244 |
self.stdout.write("%s files to delete"%j) |
323 | 245 |
i = 0 |
246 |
files = list(chain( |
|
247 |
models.Media.objects.filter(src__iregex=r".*.flv$"), |
|
248 |
models.Media.objects.filter(src__iregex=r".*.f4v$"), |
|
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
249 |
models.Media.objects.filter(src__iregex=r".*.m4v$"), |
330 | 250 |
models.Media.objects.filter(src__iregex=r".*.mp4$").exclude(src__iregex=r"^https://.*"), |
251 |
models.Media.objects.filter(src__iregex=r"^mp4:.*").exclude(src__iregex=r".*\..{3}$") |
|
323 | 252 |
)) |
327
c3998a528fd9
delete well formed source links videopaths, check youtube videos, change m4v source links
bellierp
parents:
324
diff
changeset
|
253 |
|
323 | 254 |
for elem in files: |
255 |
self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') |
|
256 |
i += 1 |
|
333 | 257 |
if number_of_contents(elem) == 0: |
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
258 |
continue |
323 | 259 |
mysrc = elem.src |
333 | 260 |
newsource = to_https(elem.src, elem.videopath) |
323 | 261 |
try: |
262 |
res = requests.head(newsource, timeout=10).status_code |
|
263 |
except requests.ConnectionError: |
|
264 |
self.stdout.write(" connection error", ending='') |
|
333 | 265 |
self.logger.error("CONNECTION ERROR FOR %s", elem.title) |
323 | 266 |
try: |
267 |
res = requests.head(elem, timeout=10).status_code |
|
268 |
except requests.ConnectionError: |
|
333 | 269 |
self.mycsvfile.writerow([ |
332 | 270 |
"Media", |
323 | 271 |
mysrc, |
332 | 272 |
"No", |
273 |
"connection error", |
|
274 |
newsource |
|
275 |
]) |
|
323 | 276 |
continue |
277 |
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
333 | 278 |
self.mycsvfile.writerow([ |
332 | 279 |
"Media", |
323 | 280 |
mysrc, |
332 | 281 |
"No", |
323 | 282 |
"missing schema on base source!", |
332 | 283 |
newsource |
284 |
]) |
|
323 | 285 |
continue |
286 |
except requests.exceptions.Timeout: |
|
333 | 287 |
self.mycsvfile.writerow([ |
332 | 288 |
"Media", |
323 | 289 |
mysrc, |
332 | 290 |
"No", |
323 | 291 |
"TIMEOUT!", |
332 | 292 |
newsource |
293 |
]) |
|
323 | 294 |
continue |
295 |
else: |
|
333 | 296 |
self.mycsvfile.writerow([ |
332 | 297 |
"Media", |
323 | 298 |
mysrc, |
332 | 299 |
"No", |
323 | 300 |
"use source link : website doesn't work with https", |
332 | 301 |
newsource |
302 |
]) |
|
323 | 303 |
continue |
304 |
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
305 |
self.stdout.write(" Missing schema !", ending='') |
|
333 | 306 |
self.logger.warning("MISSING SCHEMA FOR %s", elem.title) |
307 |
self.mycsvfile.writerow([ |
|
332 | 308 |
"Media", |
323 | 309 |
mysrc, |
332 | 310 |
"No", |
323 | 311 |
"missing schema!", |
332 | 312 |
newsource |
313 |
]) |
|
323 | 314 |
continue |
315 |
except requests.exceptions.Timeout: |
|
316 |
self.stdout.write(" Timeout !", ending='') |
|
333 | 317 |
self.logger.warning("Timeout FOR %s", elem.title) |
318 |
self.mycsvfile.writerow([ |
|
332 | 319 |
"Media", |
323 | 320 |
mysrc, |
332 | 321 |
"No", |
323 | 322 |
"TIMEOUT!", |
332 | 323 |
newsource |
324 |
]) |
|
323 | 325 |
continue |
326 |
if res > 400: |
|
327 |
try: |
|
333 | 328 |
ressrc = requests.head(to_https(elem.src, elem.videopath, 0),\ |
323 | 329 |
timeout=10).status_code |
330 |
except (requests.exceptions.Timeout, requests.ConnectionError): |
|
331 |
self.stdout.write(" can't access source/new files", ending='') |
|
333 | 332 |
self.logger.warning("can't access %s", elem.title) |
323 | 333 |
res = "connection error" |
333 | 334 |
self.mycsvfile.writerow([ |
332 | 335 |
"Media", |
323 | 336 |
mysrc, |
332 | 337 |
"No", |
323 | 338 |
"website doesn't exist anymore", |
332 | 339 |
newsource |
340 |
]) |
|
323 | 341 |
continue |
342 |
if ressrc == 404: |
|
343 |
self.stdout.write(" can't access source/new files", ending='') |
|
333 | 344 |
self.logger.warning("can't access %s", elem.title) |
345 |
self.mycsvfile.writerow([ |
|
332 | 346 |
"Media", |
323 | 347 |
mysrc, |
332 | 348 |
"No", |
323 | 349 |
"can't access source/new files", |
332 | 350 |
newsource |
351 |
]) |
|
323 | 352 |
elif ressrc == 200: |
353 |
self.stdout.write( |
|
354 |
" file not transcoded yet :" |
|
333 | 355 |
"keep source extension or wait transcoding to be done",\ |
323 | 356 |
ending='') |
333 | 357 |
self.logger.warning("%s not transcoded yet", elem.title) |
358 |
self.mycsvfile.writerow([ |
|
332 | 359 |
"Media", |
323 | 360 |
mysrc, |
332 | 361 |
"No", |
323 | 362 |
"file not transcoded yet : keep source extension", |
332 | 363 |
newsource |
364 |
]) |
|
323 | 365 |
continue |
366 |
self.stdout.write(" It works", ending='') |
|
367 |
alreadyin = False |
|
368 |
for everyelem in models.Media.objects.all(): |
|
369 |
if newsource == everyelem.src: |
|
370 |
alreadyin = True |
|
371 |
break |
|
372 |
if alreadyin: |
|
373 |
self.stdout.write(" element already in table", ending='') |
|
333 | 374 |
self.logger.warning("%s already in table", elem.title) |
375 |
self.mycsvfile.writerow([ |
|
332 | 376 |
"Media", |
323 | 377 |
newsource, |
332 | 378 |
"No", |
379 |
"new source already in table" |
|
380 |
]) |
|
323 | 381 |
continue |
333 | 382 |
self.clean_media_project(elem, force, newsource) |
383 |
csvfile.close() |