author | bellierp |
Tue, 07 Mar 2017 16:49:31 +0100 | |
changeset 324 | dffa18e6cba4 |
parent 323 | bdffbceb0730 |
child 327 | c3998a528fd9 |
permissions | -rw-r--r-- |
323 | 1 |
''' |
2 |
List flv and f4v medias, replace them with mp4 urls and update the content and projects. |
|
3 |
''' |
|
4 |
import csv |
|
5 |
import logging |
|
6 |
from itertools import chain |
|
7 |
||
8 |
import requests |
|
9 |
from django.conf import settings |
|
10 |
from django.contrib.sites.models import Site |
|
11 |
from django.core.management.base import BaseCommand |
|
12 |
from lxml import etree |
|
13 |
||
14 |
from ldt.ldt_utils import models |
|
15 |
||
16 |
||
17 |
#this function replace bad suffixs and prefixs of some media URL |
|
18 |
#by a new one, beginning with "http" and ending with ".mp4" |
|
19 |
def tohttps(source, vidpath, tomp4=1): |
|
20 |
''' |
|
21 |
to https |
|
22 |
''' |
|
23 |
if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4': |
|
24 |
tomp4 = 0 |
|
25 |
if tomp4 == 1: |
|
26 |
source = source[0:len(source)-3]+"mp4" |
|
27 |
if source[0:4] == "http" or source[0:4] == "sftp": |
|
28 |
return "https"+source[4:len(source)] |
|
29 |
elif source[0:7] == "/video/": |
|
30 |
return "https://media.iri.centrepompidou.fr"+source |
|
31 |
elif source[0:6] == "video/" or source[0:6] == "audio/": |
|
32 |
return "https://media.iri.centrepompidou.fr/"+source |
|
33 |
elif source[0:4] == "mp4:": |
|
34 |
return "hue" |
|
35 |
elif vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/video/regardssignes/' or \ |
|
36 |
vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/regardssignes/': |
|
37 |
return "https://media.iri.centrepompidou.fr/video/regardssignes/"+source |
|
38 |
else: |
|
39 |
return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source |
|
40 |
||
41 |
def numberofcontents(source): #this counts the number of contents linked to a media |
|
42 |
''' |
|
43 |
numberofcontents |
|
44 |
''' |
|
45 |
return len(models.Content.objects.filter(media_obj_id=source.id)) |
|
46 |
||
47 |
def numberofproject(source): |
|
48 |
''' |
|
49 |
numberofproject |
|
50 |
''' |
|
51 |
if numberofcontents(source) > 0: |
|
52 |
return len(models.Project.objects.filter\ |
|
53 |
(content=models.Content.objects.filter(media_obj_id=source.id)[0])) |
|
54 |
else: |
|
55 |
return 0 |
|
56 |
||
57 |
class Command(BaseCommand): |
|
58 |
''' |
|
59 |
Command class |
|
60 |
''' |
|
61 |
help = 'delete medias without contents, replace media\'s source by a new URL' |
|
62 |
def add_arguments(self, parser): |
|
63 |
''' |
|
64 |
add arguments |
|
65 |
''' |
|
66 |
parser.add_argument('-f', action='store_true') |
|
67 |
||
68 |
def handle(self, *args, **options): |
|
69 |
''' |
|
70 |
handle |
|
71 |
''' |
|
72 |
forcsv = [[ |
|
73 |
"Source link", |
|
74 |
"Tried link", |
|
75 |
"Change ?", |
|
76 |
"Request status", |
|
77 |
"Informations", |
|
78 |
"Number of contents", |
|
79 |
"Number of projects" |
|
80 |
]] |
|
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
81 |
|
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
82 |
j=0 |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
83 |
files1 = models.Media.objects.all() #this list contains every media |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
84 |
for elem1 in files1: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
85 |
if numberofcontents(elem1) == 0: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
86 |
if options['f']: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
87 |
elem1.delete() #if there is no content |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
88 |
#linked to the media, the media is removed for the database |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
89 |
self.stdout.write(" No content found, media has been removed") |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
90 |
else: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
91 |
forcsv += [[ |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
92 |
elem1.src, |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
93 |
"XXX", |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
94 |
"XXX", |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
95 |
"XXX", |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
96 |
"NO CONTENT : media will be erased", |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
97 |
"0", |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
98 |
"0" |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
99 |
]] |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
100 |
j += 1 |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
101 |
if options['f']: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
102 |
self.stdout.write("%s files deleted"%j) |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
103 |
infos = open('results.txt', 'w') |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
104 |
parser = etree.XMLParser(encoding='utf-8') |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
105 |
else: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
106 |
self.stdout.write("%s files to delete"%j) |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
107 |
infos = open('projectsToChange.txt', 'w') |
323 | 108 |
i = 0 |
109 |
files = list(chain( |
|
110 |
models.Media.objects.filter(src__iregex=r".*.flv$"), |
|
111 |
models.Media.objects.filter(src__iregex=r".*.f4v$"), |
|
112 |
models.Media.objects.filter(src__iregex=r".*.mp4$").exclude(src__iregex=r"^https://.*") |
|
113 |
)) |
|
114 |
logger = logging.getLogger(__name__) |
|
115 |
for elem in files: |
|
116 |
self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') |
|
117 |
i += 1 |
|
324
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
118 |
if numberofcontents(elem) == 0: |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
119 |
self.stdout.write(" no content",ending='') |
dffa18e6cba4
add list of files without contents in mediaInformations.csv
bellierp
parents:
323
diff
changeset
|
120 |
continue |
323 | 121 |
mysrc = elem.src |
122 |
newsource = tohttps(elem.src, elem.videopath) |
|
123 |
try: |
|
124 |
res = requests.head(newsource, timeout=10).status_code |
|
125 |
except requests.ConnectionError: |
|
126 |
self.stdout.write(" connection error", ending='') |
|
127 |
logger.error("CONNECTION ERROR FOR %s", elem.title) |
|
128 |
try: |
|
129 |
res = requests.head(elem, timeout=10).status_code |
|
130 |
except requests.ConnectionError: |
|
131 |
res = "connection error" |
|
132 |
forcsv += [[ |
|
133 |
mysrc, |
|
134 |
newsource, |
|
135 |
0, |
|
136 |
res, |
|
137 |
": website doesn't exist anymore", |
|
138 |
numberofcontents(elem), |
|
139 |
numberofproject(elem) |
|
140 |
]] |
|
141 |
continue |
|
142 |
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
143 |
forcsv += [[ |
|
144 |
mysrc, |
|
145 |
newsource, |
|
146 |
0, |
|
147 |
"XXX", |
|
148 |
"missing schema on base source!", |
|
149 |
numberofcontents(elem), |
|
150 |
numberofproject(elem) |
|
151 |
]] |
|
152 |
continue |
|
153 |
except requests.exceptions.Timeout: |
|
154 |
forcsv += [[ |
|
155 |
mysrc, |
|
156 |
newsource, |
|
157 |
0, |
|
158 |
"XXX", |
|
159 |
"TIMEOUT!", |
|
160 |
numberofcontents(elem), |
|
161 |
numberofproject(elem) |
|
162 |
]] |
|
163 |
continue |
|
164 |
else: |
|
165 |
forcsv += [[ |
|
166 |
mysrc, |
|
167 |
newsource, |
|
168 |
0, |
|
169 |
res, |
|
170 |
"use source link : website doesn't work with https", |
|
171 |
numberofcontents(elem), |
|
172 |
numberofproject(elem) |
|
173 |
]] |
|
174 |
continue |
|
175 |
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
176 |
self.stdout.write(" Missing schema !", ending='') |
|
177 |
logger.warning("MISSING SCHEMA FOR %s", elem.title) |
|
178 |
forcsv += [[ |
|
179 |
mysrc, |
|
180 |
newsource, |
|
181 |
0, |
|
182 |
"XXX", |
|
183 |
"missing schema!", |
|
184 |
numberofcontents(elem), |
|
185 |
numberofproject(elem) |
|
186 |
]] |
|
187 |
continue |
|
188 |
except requests.exceptions.Timeout: |
|
189 |
self.stdout.write(" Timeout !", ending='') |
|
190 |
logger.warning("Timeout FOR %s", elem.title) |
|
191 |
forcsv += [[ |
|
192 |
mysrc, |
|
193 |
newsource, |
|
194 |
0, |
|
195 |
"XXX", |
|
196 |
"TIMEOUT!", |
|
197 |
numberofcontents(elem), |
|
198 |
numberofproject(elem) |
|
199 |
]] |
|
200 |
continue |
|
201 |
if res > 400: |
|
202 |
try: |
|
203 |
ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\ |
|
204 |
timeout=10).status_code |
|
205 |
except (requests.exceptions.Timeout, requests.ConnectionError): |
|
206 |
self.stdout.write(" can't access source/new files", ending='') |
|
207 |
logger.warning("can't access %s", elem.title) |
|
208 |
res = "connection error" |
|
209 |
forcsv += [[ |
|
210 |
mysrc, |
|
211 |
newsource, |
|
212 |
0, |
|
213 |
res, |
|
214 |
"website doesn't exist anymore", |
|
215 |
numberofcontents(elem), |
|
216 |
numberofproject(elem) |
|
217 |
]] |
|
218 |
continue |
|
219 |
if ressrc == 404: |
|
220 |
self.stdout.write(" can't access source/new files", ending='') |
|
221 |
logger.warning("can't access %s", elem.title) |
|
222 |
forcsv += [[ |
|
223 |
mysrc, |
|
224 |
newsource, |
|
225 |
0, |
|
226 |
res, |
|
227 |
"can't access source/new files", |
|
228 |
numberofcontents(elem), |
|
229 |
numberofproject(elem) |
|
230 |
]] |
|
231 |
elif ressrc == 200: |
|
232 |
self.stdout.write( |
|
233 |
" file not transcoded yet :" |
|
234 |
"keep source extension or wait transcoding to be done", |
|
235 |
ending='') |
|
236 |
logger.warning("%s not transcoded yet", elem.title) |
|
237 |
forcsv += [[ |
|
238 |
mysrc, |
|
239 |
newsource, |
|
240 |
0, |
|
241 |
res, |
|
242 |
"file not transcoded yet : keep source extension", |
|
243 |
numberofcontents(elem), |
|
244 |
numberofproject(elem) |
|
245 |
]] |
|
246 |
continue |
|
247 |
self.stdout.write(" It works", ending='') |
|
248 |
if options['f']: |
|
249 |
elem.src = newsource |
|
250 |
alreadyin = False |
|
251 |
for everyelem in models.Media.objects.all(): |
|
252 |
if newsource == everyelem.src: |
|
253 |
alreadyin = True |
|
254 |
break |
|
255 |
if alreadyin: |
|
256 |
self.stdout.write(" element already in table", ending='') |
|
257 |
logger.warning("%s already in table", elem.title) |
|
258 |
forcsv += [[ |
|
259 |
mysrc, |
|
260 |
newsource, |
|
261 |
0, |
|
262 |
res, |
|
263 |
"element already in table", |
|
264 |
numberofcontents(elem), |
|
265 |
numberofproject(elem) |
|
266 |
]] |
|
267 |
continue |
|
268 |
if numberofcontents(elem) == 0: |
|
269 |
self.stdout.write(" source has no content", ending='') |
|
270 |
forcsv += [[ |
|
271 |
mysrc, |
|
272 |
newsource, |
|
273 |
0, |
|
274 |
res, |
|
275 |
"source has no content", |
|
276 |
numberofcontents(elem), |
|
277 |
numberofproject(elem) |
|
278 |
]] |
|
279 |
continue |
|
280 |
if options['f']: |
|
281 |
elem.videopath = '' |
|
282 |
elem.save() |
|
283 |
ldtproj = models.Project.objects.filter\ |
|
284 |
(content=models.Content.objects.filter(media_obj_id=elem.id)[0]) |
|
285 |
if numberofproject(elem) == 0: |
|
286 |
self.stdout.write(" no project", ending='') |
|
287 |
logger.warning("NO PROJECT FOR %s", elem.title) |
|
288 |
forcsv += [[ |
|
289 |
mysrc, |
|
290 |
newsource, |
|
291 |
1, |
|
292 |
res, |
|
293 |
"It works but there is no project with this media", |
|
294 |
numberofcontents(elem), |
|
295 |
numberofproject(elem) |
|
296 |
]] |
|
297 |
continue |
|
298 |
infos.write("\nProjects : \n") |
|
299 |
for numproject in xrange(numberofproject(elem)): |
|
300 |
base_url = Site.objects.get_current().domain + settings.BASE_URL |
|
301 |
ldt_id = ldtproj[numproject].ldt_id |
|
302 |
embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ |
|
303 |
"http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ |
|
304 |
"player_id=player_project_{ldt_id}&" \ |
|
305 |
"ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id) |
|
306 |
infos.write("%s \n"%(embedurl)) |
|
307 |
infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\ |
|
308 |
% (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id)) |
|
309 |
if options['f']: |
|
310 |
for numproject in xrange(len(ldtproj)): |
|
311 |
root = etree.XML(ldtproj[numproject].ldt.encode('utf-8'), parser) |
|
312 |
if len(root.xpath('medias/media')) == 0: |
|
313 |
self.stdout.write(" le .ldt ne contient pas de media", ending='') |
|
314 |
infos.write("le .ldt ne contient pas de media") |
|
315 |
continue |
|
316 |
root.xpath('medias/media')[0].set("video", '') |
|
317 |
ldtproj[numproject].ldt = etree.tostring(root) |
|
318 |
ldtproj[numproject].save() |
|
319 |
infos.write("\nSuccessful !\n") |
|
320 |
logger.info("%s DONE\n", embedurl) |
|
321 |
self.stdout.write(" done", ending='') |
|
322 |
forcsv += [[ |
|
323 |
mysrc, |
|
324 |
newsource, |
|
325 |
1, |
|
326 |
res, |
|
327 |
"It works", |
|
328 |
numberofcontents(elem), |
|
329 |
numberofproject(elem) |
|
330 |
]] |
|
331 |
if not options['f']: |
|
332 |
with open('mediaInformations.csv', 'wb') as csvfile: |
|
333 |
mycsvfile = csv.writer(csvfile) |
|
334 |
for mycsv in forcsv: |
|
335 |
mycsvfile.writerow(mycsv) |
|
336 |
infos.close() |