|
1 ''' |
|
2 List flv and f4v medias, replace them with mp4 urls and update the content and projects. |
|
3 ''' |
|
4 import csv |
|
5 import logging |
|
6 from itertools import chain |
|
7 |
|
8 import requests |
|
9 from django.conf import settings |
|
10 from django.contrib.sites.models import Site |
|
11 from django.core.management.base import BaseCommand |
|
12 from lxml import etree |
|
13 |
|
14 from ldt.ldt_utils import models |
|
15 |
|
16 |
|
17 #this function replace bad suffixs and prefixs of some media URL |
|
18 #by a new one, beginning with "http" and ending with ".mp4" |
|
19 def tohttps(source, vidpath, tomp4=1): |
|
20 ''' |
|
21 to https |
|
22 ''' |
|
23 if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4': |
|
24 tomp4 = 0 |
|
25 if tomp4 == 1: |
|
26 source = source[0:len(source)-3]+"mp4" |
|
27 if source[0:4] == "http" or source[0:4] == "sftp": |
|
28 return "https"+source[4:len(source)] |
|
29 elif source[0:7] == "/video/": |
|
30 return "https://media.iri.centrepompidou.fr"+source |
|
31 elif source[0:6] == "video/" or source[0:6] == "audio/": |
|
32 return "https://media.iri.centrepompidou.fr/"+source |
|
33 elif source[0:4] == "mp4:": |
|
34 return "hue" |
|
35 elif vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/video/regardssignes/' or \ |
|
36 vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/regardssignes/': |
|
37 return "https://media.iri.centrepompidou.fr/video/regardssignes/"+source |
|
38 else: |
|
39 return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source |
|
40 |
|
41 def numberofcontents(source): #this counts the number of contents linked to a media |
|
42 ''' |
|
43 numberofcontents |
|
44 ''' |
|
45 return len(models.Content.objects.filter(media_obj_id=source.id)) |
|
46 |
|
47 def numberofproject(source): |
|
48 ''' |
|
49 numberofproject |
|
50 ''' |
|
51 if numberofcontents(source) > 0: |
|
52 return len(models.Project.objects.filter\ |
|
53 (content=models.Content.objects.filter(media_obj_id=source.id)[0])) |
|
54 else: |
|
55 return 0 |
|
56 |
|
57 class Command(BaseCommand): |
|
58 ''' |
|
59 Command class |
|
60 ''' |
|
61 help = 'delete medias without contents, replace media\'s source by a new URL' |
|
62 def add_arguments(self, parser): |
|
63 ''' |
|
64 add arguments |
|
65 ''' |
|
66 parser.add_argument('-f', action='store_true') |
|
67 |
|
68 def handle(self, *args, **options): |
|
69 ''' |
|
70 handle |
|
71 ''' |
|
72 if options['f']: |
|
73 infos = open('results.txt', 'w') |
|
74 parser = etree.XMLParser(encoding='utf-8') |
|
75 files1 = models.Media.objects.all() #this list contains every media |
|
76 for elem1 in files1: |
|
77 if numberofcontents(elem1) == 0: |
|
78 elem1.delete() #if there is no content |
|
79 #linked to the media, the media is removed for the database |
|
80 self.stdout.write(" No content found, media has been removed") |
|
81 else: |
|
82 infos = open('projectsToChange.txt', 'w') |
|
83 forcsv = [[ |
|
84 "Source link", |
|
85 "Tried link", |
|
86 "Change ?", |
|
87 "Request status", |
|
88 "Informations", |
|
89 "Number of contents", |
|
90 "Number of projects" |
|
91 ]] |
|
92 i = 0 |
|
93 files = list(chain( |
|
94 models.Media.objects.filter(src__iregex=r".*.flv$"), |
|
95 models.Media.objects.filter(src__iregex=r".*.f4v$"), |
|
96 models.Media.objects.filter(src__iregex=r".*.mp4$").exclude(src__iregex=r"^https://.*") |
|
97 )) |
|
98 logger = logging.getLogger(__name__) |
|
99 for elem in files: |
|
100 self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') |
|
101 i += 1 |
|
102 mysrc = elem.src |
|
103 newsource = tohttps(elem.src, elem.videopath) |
|
104 try: |
|
105 res = requests.head(newsource, timeout=10).status_code |
|
106 except requests.ConnectionError: |
|
107 self.stdout.write(" connection error", ending='') |
|
108 logger.error("CONNECTION ERROR FOR %s", elem.title) |
|
109 try: |
|
110 res = requests.head(elem, timeout=10).status_code |
|
111 except requests.ConnectionError: |
|
112 res = "connection error" |
|
113 forcsv += [[ |
|
114 mysrc, |
|
115 newsource, |
|
116 0, |
|
117 res, |
|
118 ": website doesn't exist anymore", |
|
119 numberofcontents(elem), |
|
120 numberofproject(elem) |
|
121 ]] |
|
122 continue |
|
123 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
124 forcsv += [[ |
|
125 mysrc, |
|
126 newsource, |
|
127 0, |
|
128 "XXX", |
|
129 "missing schema on base source!", |
|
130 numberofcontents(elem), |
|
131 numberofproject(elem) |
|
132 ]] |
|
133 continue |
|
134 except requests.exceptions.Timeout: |
|
135 forcsv += [[ |
|
136 mysrc, |
|
137 newsource, |
|
138 0, |
|
139 "XXX", |
|
140 "TIMEOUT!", |
|
141 numberofcontents(elem), |
|
142 numberofproject(elem) |
|
143 ]] |
|
144 continue |
|
145 else: |
|
146 forcsv += [[ |
|
147 mysrc, |
|
148 newsource, |
|
149 0, |
|
150 res, |
|
151 "use source link : website doesn't work with https", |
|
152 numberofcontents(elem), |
|
153 numberofproject(elem) |
|
154 ]] |
|
155 continue |
|
156 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
|
157 self.stdout.write(" Missing schema !", ending='') |
|
158 logger.warning("MISSING SCHEMA FOR %s", elem.title) |
|
159 forcsv += [[ |
|
160 mysrc, |
|
161 newsource, |
|
162 0, |
|
163 "XXX", |
|
164 "missing schema!", |
|
165 numberofcontents(elem), |
|
166 numberofproject(elem) |
|
167 ]] |
|
168 continue |
|
169 except requests.exceptions.Timeout: |
|
170 self.stdout.write(" Timeout !", ending='') |
|
171 logger.warning("Timeout FOR %s", elem.title) |
|
172 forcsv += [[ |
|
173 mysrc, |
|
174 newsource, |
|
175 0, |
|
176 "XXX", |
|
177 "TIMEOUT!", |
|
178 numberofcontents(elem), |
|
179 numberofproject(elem) |
|
180 ]] |
|
181 continue |
|
182 if res > 400: |
|
183 try: |
|
184 ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\ |
|
185 timeout=10).status_code |
|
186 except (requests.exceptions.Timeout, requests.ConnectionError): |
|
187 self.stdout.write(" can't access source/new files", ending='') |
|
188 logger.warning("can't access %s", elem.title) |
|
189 res = "connection error" |
|
190 forcsv += [[ |
|
191 mysrc, |
|
192 newsource, |
|
193 0, |
|
194 res, |
|
195 "website doesn't exist anymore", |
|
196 numberofcontents(elem), |
|
197 numberofproject(elem) |
|
198 ]] |
|
199 continue |
|
200 if ressrc == 404: |
|
201 self.stdout.write(" can't access source/new files", ending='') |
|
202 logger.warning("can't access %s", elem.title) |
|
203 forcsv += [[ |
|
204 mysrc, |
|
205 newsource, |
|
206 0, |
|
207 res, |
|
208 "can't access source/new files", |
|
209 numberofcontents(elem), |
|
210 numberofproject(elem) |
|
211 ]] |
|
212 elif ressrc == 200: |
|
213 self.stdout.write( |
|
214 " file not transcoded yet :" |
|
215 "keep source extension or wait transcoding to be done", |
|
216 ending='') |
|
217 logger.warning("%s not transcoded yet", elem.title) |
|
218 forcsv += [[ |
|
219 mysrc, |
|
220 newsource, |
|
221 0, |
|
222 res, |
|
223 "file not transcoded yet : keep source extension", |
|
224 numberofcontents(elem), |
|
225 numberofproject(elem) |
|
226 ]] |
|
227 continue |
|
228 self.stdout.write(" It works", ending='') |
|
229 if options['f']: |
|
230 elem.src = newsource |
|
231 alreadyin = False |
|
232 for everyelem in models.Media.objects.all(): |
|
233 if newsource == everyelem.src: |
|
234 alreadyin = True |
|
235 break |
|
236 if alreadyin: |
|
237 self.stdout.write(" element already in table", ending='') |
|
238 logger.warning("%s already in table", elem.title) |
|
239 forcsv += [[ |
|
240 mysrc, |
|
241 newsource, |
|
242 0, |
|
243 res, |
|
244 "element already in table", |
|
245 numberofcontents(elem), |
|
246 numberofproject(elem) |
|
247 ]] |
|
248 continue |
|
249 if numberofcontents(elem) == 0: |
|
250 self.stdout.write(" source has no content", ending='') |
|
251 forcsv += [[ |
|
252 mysrc, |
|
253 newsource, |
|
254 0, |
|
255 res, |
|
256 "source has no content", |
|
257 numberofcontents(elem), |
|
258 numberofproject(elem) |
|
259 ]] |
|
260 continue |
|
261 if options['f']: |
|
262 elem.videopath = '' |
|
263 elem.save() |
|
264 ldtproj = models.Project.objects.filter\ |
|
265 (content=models.Content.objects.filter(media_obj_id=elem.id)[0]) |
|
266 if numberofproject(elem) == 0: |
|
267 self.stdout.write(" no project", ending='') |
|
268 logger.warning("NO PROJECT FOR %s", elem.title) |
|
269 forcsv += [[ |
|
270 mysrc, |
|
271 newsource, |
|
272 1, |
|
273 res, |
|
274 "It works but there is no project with this media", |
|
275 numberofcontents(elem), |
|
276 numberofproject(elem) |
|
277 ]] |
|
278 continue |
|
279 infos.write("\nProjects : \n") |
|
280 for numproject in xrange(numberofproject(elem)): |
|
281 base_url = Site.objects.get_current().domain + settings.BASE_URL |
|
282 ldt_id = ldtproj[numproject].ldt_id |
|
283 embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ |
|
284 "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ |
|
285 "player_id=player_project_{ldt_id}&" \ |
|
286 "ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id) |
|
287 infos.write("%s \n"%(embedurl)) |
|
288 infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\ |
|
289 % (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id)) |
|
290 if options['f']: |
|
291 for numproject in xrange(len(ldtproj)): |
|
292 root = etree.XML(ldtproj[numproject].ldt.encode('utf-8'), parser) |
|
293 if len(root.xpath('medias/media')) == 0: |
|
294 self.stdout.write(" le .ldt ne contient pas de media", ending='') |
|
295 infos.write("le .ldt ne contient pas de media") |
|
296 continue |
|
297 root.xpath('medias/media')[0].set("video", '') |
|
298 ldtproj[numproject].ldt = etree.tostring(root) |
|
299 ldtproj[numproject].save() |
|
300 infos.write("\nSuccessful !\n") |
|
301 logger.info("%s DONE\n", embedurl) |
|
302 self.stdout.write(" done", ending='') |
|
303 forcsv += [[ |
|
304 mysrc, |
|
305 newsource, |
|
306 1, |
|
307 res, |
|
308 "It works", |
|
309 numberofcontents(elem), |
|
310 numberofproject(elem) |
|
311 ]] |
|
312 if not options['f']: |
|
313 with open('mediaInformations.csv', 'wb') as csvfile: |
|
314 mycsvfile = csv.writer(csvfile) |
|
315 for mycsv in forcsv: |
|
316 mycsvfile.writerow(mycsv) |
|
317 infos.close() |