59 if numberofcontents(source) > 0: |
59 if numberofcontents(source) > 0: |
60 return len(models.Project.objects.filter\ |
60 return len(models.Project.objects.filter\ |
61 (content=models.Content.objects.filter(media_obj_id=source.id)[0])) |
61 (content=models.Content.objects.filter(media_obj_id=source.id)[0])) |
62 return 0 |
62 return 0 |
63 |
63 |
|
64 def constructytembed(source): |
|
65 ''' |
|
66 construct youtube video oembed link |
|
67 ''' |
|
68 if re.match(r".*feature=player_embedded.+", source) != None: |
|
69 return "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\ |
|
70 + source[len(source)-11:] +"&format=json" |
|
71 return "http://www.youtube.com/oembed?url=" + source + "&format=json" |
|
72 |
64 class Command(BaseCommand): |
73 class Command(BaseCommand): |
65 ''' |
74 ''' |
66 Command class |
75 Command class |
67 ''' |
76 ''' |
68 help = 'delete medias without contents, replace media\'s source by a new URL' |
77 help = 'delete medias without contents, replace media\'s source by a new URL' |
|
78 base_url = Site.objects.get_current().domain + settings.BASE_URL |
|
79 parser = etree.XMLParser(encoding='utf-8') |
|
80 logger = logging.getLogger(__name__) |
|
81 csvfile = open('mediaInformations.csv', 'wb') |
|
82 mycsvfile = csv.writer(csvfile) |
|
83 |
|
84 def constructldtembed(self, ldtid): |
|
85 ''' |
|
86 construct ldt embed |
|
87 ''' |
|
88 return "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ |
|
89 "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ |
|
90 "player_id=player_project_{ldt_id}&" \ |
|
91 "ldt_id={ldt_id}".format(base_url=Command.base_url, ldt_id=ldtid) |
|
92 |
|
93 def cleanmediaproject(self, element, force, newsrc=None): |
|
94 ''' |
|
95 change media objects' videopath and source if necessary |
|
96 change project .ldt |
|
97 ''' |
|
98 basesrc = element.src |
|
99 if force: |
|
100 element.videopath = '' |
|
101 element.save() |
|
102 if newsrc != None: |
|
103 if force: |
|
104 element.src = newsrc |
|
105 element.save() |
|
106 Command.mycsvfile.writerow([ |
|
107 "Media", |
|
108 basesrc, |
|
109 "Yes", |
|
110 "changing source/videopath", |
|
111 newsrc, "\'\'" |
|
112 ]) |
|
113 if numberofproject(element) == 0: |
|
114 Command.mycsvfile.writerow([ |
|
115 "Project", |
|
116 element.src, |
|
117 "Yes", |
|
118 "initializing object(no project)" |
|
119 ]) |
|
120 if force: |
|
121 mycontentid = models.Content.objects.filter(media_obj_id=element.id)[0].iri_id |
|
122 try: |
|
123 management.call_command('initfrontproject', mycontentid) |
|
124 except Exception: |
|
125 Command.mycsvfile.writerow([ |
|
126 "Project", |
|
127 element.src, |
|
128 "No", |
|
129 "socket error" |
|
130 ]) |
|
131 return |
|
132 self.stdout.write(" Initializing project", ending='') |
|
133 else: |
|
134 self.stdout.write(" Project has to be initialized ", ending='') |
|
135 return |
|
136 ldtproj = models.Project.objects.filter\ |
|
137 (content=models.Content.objects.filter(media_obj_id=element.id)[0]) |
|
138 for singleproject in ldtproj: |
|
139 root = etree.XML(singleproject.ldt.encode('utf-8'), Command.parser) |
|
140 if root.xpath('medias/media') == []: |
|
141 self.stdout.write(" le .ldt ne contient pas de media", ending='') |
|
142 continue |
|
143 if root.xpath('medias/media')[0].get("video") != '': |
|
144 embedurl = self.constructldtembed(singleproject.ldt_id) |
|
145 if force: |
|
146 root.xpath('medias/media')[0].set("video", '') |
|
147 self.stdout.write(" changing videopath arg in .ldt ") |
|
148 Command.mycsvfile.writerow([ |
|
149 "Project", |
|
150 embedurl, |
|
151 "Yes", |
|
152 "changing .ldt /medias/media/video", |
|
153 "\'\'" |
|
154 ]) |
|
155 singleproject.ldt = etree.tostring(root) |
|
156 singleproject.save() |
|
157 Command.logger.info("%s DONE\n", embedurl) |
|
158 element.save() |
|
159 |
69 def add_arguments(self, parser): |
160 def add_arguments(self, parser): |
70 ''' |
161 ''' |
71 add arguments |
162 add arguments |
72 ''' |
163 ''' |
73 parser.add_argument('-f', action='store_true') |
164 parser.add_argument('-f', action='store_true') |
74 |
165 |
75 def handle(self, *args, **options): |
166 def handle(self, *args, **options): |
76 ''' |
167 ''' |
77 handle |
168 handle |
78 ''' |
169 ''' |
79 forcsv = [[ |
170 Command.mycsvfile.writerow([ |
80 "Source link", |
171 "Object type", |
81 "Tried link", |
172 "which object", |
82 "Change ?", |
173 "Change ?", |
83 "Request status", |
174 "What(if Y)/Why (if N)", |
84 "Informations", |
175 "How" |
85 "Number of contents", |
176 ]) |
86 "Number of projects" |
177 |
87 ]] |
178 force = bool(options['f']) |
88 if options['f']: |
|
89 infos = open('results.txt', 'w') |
|
90 else: |
|
91 infos = open('projectsToChange.txt', 'w') |
|
92 j = 0 |
179 j = 0 |
93 parser = etree.XMLParser(encoding='utf-8') |
|
94 logger = logging.getLogger(__name__) |
|
95 files1 = models.Media.objects.all() #this list contains every media |
180 files1 = models.Media.objects.all() #this list contains every media |
96 for elem1 in files1: |
181 for elem1 in files1: |
97 if numberofcontents(elem1) == 0: |
182 if numberofcontents(elem1) == 0: |
98 if options['f']: |
183 if force: |
99 elem1.delete() #if there is no content |
184 elem1.delete() #if there is no content |
100 #linked to the media, the media is removed for the database |
185 #linked to the media, the media is removed for the database |
101 self.stdout.write(" No content found, media has been removed") |
186 self.stdout.write(" No content found, media has been removed") |
102 else: |
187 else: |
103 forcsv += [[ |
188 self.stdout.write(" No content found, media will be removed") |
104 elem1.src, |
189 Command.mycsvfile.writerow([ |
105 "XXX", |
190 "Media", |
106 "XXX", |
191 elem1.src, |
107 "XXX", |
192 "Yes", |
108 "NO CONTENT : media will be erased", |
193 "deleting object (no content)" |
109 "0", |
194 ]) |
110 "0" |
|
111 ]] |
|
112 j += 1 |
195 j += 1 |
113 continue |
196 continue |
114 if numberofproject(elem1) == 0: |
|
115 if options['f']: |
|
116 mycontentid = models.Content.objects.filter(media_obj_id=elem1.id)[0].iri_id |
|
117 management.call_command('initfrontproject', mycontentid) |
|
118 else: |
|
119 forcsv += [[ |
|
120 elem1.src, |
|
121 "XXX", |
|
122 "XXX", |
|
123 "XXX", |
|
124 "No project : will be initialized", |
|
125 "1", |
|
126 "0" |
|
127 ]] |
|
128 ldtproj = models.Project.objects.filter\ |
|
129 (content=models.Content.objects.filter(media_obj_id=elem1.id)[0]) |
|
130 if elem1.src.lower() == tohttps(elem1.src, elem1.videopath).lower(): |
197 if elem1.src.lower() == tohttps(elem1.src, elem1.videopath).lower(): |
131 if options['f']: |
198 self.cleanmediaproject(elem1, force) |
132 elem1.videopath = '' |
|
133 elem1.save() |
|
134 for singleproject in ldtproj: |
|
135 root = etree.XML(singleproject.ldt.encode('utf-8'), parser) |
|
136 mediapathlen = len(root.xpath('medias/media')) |
|
137 if mediapathlen == 0: |
|
138 self.stdout.write(" .ldt has no media", ending='') |
|
139 continue |
|
140 if options['f']: |
|
141 root.xpath('medias/media')[0].set("video", '') |
|
142 singleproject.ldt = etree.tostring(root) |
|
143 singleproject.save() |
|
144 self.stdout.write(" Project videopath modified!") |
|
145 infos.write(" Project videopath modified ") |
|
146 if re.match(r".*\.youtube\.com.*", elem1.src) != None\ |
199 if re.match(r".*\.youtube\.com.*", elem1.src) != None\ |
147 or re.match(r".*youtu\.be.+", elem1.src) != None: |
200 or re.match(r".*youtu\.be.+", elem1.src) != None: |
148 if re.match(r".*feature=player_embedded.+", elem1.src) != None: |
201 myembed = constructytembed(elem1.src) |
149 myembed = "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\ |
|
150 + elem1.src[len(elem1.src)-11:] +"&format=json" |
|
151 else: |
|
152 myembed = "http://www.youtube.com/oembed?url=" + elem1.src + "&format=json" |
|
153 if requests.get(myembed).status_code == 404: |
202 if requests.get(myembed).status_code == 404: |
154 self.stdout.write("%s : Video doesn't exists"% elem1.src) |
203 self.stdout.write("%s : Video doesn't exists"% elem1.src) |
155 if numberofproject(elem1) > 0: |
204 if numberofproject(elem1) > 0: |
156 ldtproj = models.Project.objects.get(id=models.Content.objects.filter\ |
205 ldtproj = models.Project.objects.get(id=models.Content.objects.filter\ |
157 (media_obj_id=elem1.id)[0].front_project_id).ldt |
206 (media_obj_id=elem1.id)[0].front_project_id).ldt |
158 root = etree.XML(ldtproj.encode('utf-8'), parser) |
207 root = etree.XML(ldtproj.encode('utf-8'), Command.parser) |
159 if root.xpath('annotations/content/ensemble/decoupage/elements/element')\ |
208 if root.xpath('annotations/content/ensemble/decoupage/elements/element')\ |
160 == []: |
209 == []: |
161 if options['f']: |
210 if force: |
162 elem1.delete() |
211 elem1.delete() |
163 self.stdout.write("video doesn't exist anymore : media deleted") |
212 self.stdout.write("video doesn't exist anymore : media deleted") |
164 else: |
213 Command.mycsvfile.writerow([ |
165 forcsv += [[ |
214 "Media/Content/Project", |
166 elem1.src, |
215 elem1.src, |
167 "XXX", |
216 "Yes", |
168 "XXX", |
217 "deleting(Video doesn't exist anymore + empty projects)" |
169 "404", |
218 ]) |
170 "Video doesn't exist anymore + empty projects", |
219 j += 1 |
171 "1", |
220 else: |
172 "1" |
221 self.cleanmediaproject(elem1,force) |
173 ]] |
222 if force: |
174 |
|
175 if options['f']: |
|
176 self.stdout.write("%s files deleted"%j) |
223 self.stdout.write("%s files deleted"%j) |
177 else: |
224 else: |
178 self.stdout.write("%s files to delete"%j) |
225 self.stdout.write("%s files to delete"%j) |
179 i = 0 |
226 i = 0 |
180 files = list(chain( |
227 files = list(chain( |
187 |
234 |
188 for elem in files: |
235 for elem in files: |
189 self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') |
236 self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') |
190 i += 1 |
237 i += 1 |
191 if numberofcontents(elem) == 0: |
238 if numberofcontents(elem) == 0: |
192 self.stdout.write(" no content", ending='') |
|
193 continue |
239 continue |
194 mysrc = elem.src |
240 mysrc = elem.src |
195 newsource = tohttps(elem.src, elem.videopath) |
241 newsource = tohttps(elem.src, elem.videopath) |
196 try: |
242 try: |
197 res = requests.head(newsource, timeout=10).status_code |
243 res = requests.head(newsource, timeout=10).status_code |
198 except requests.ConnectionError: |
244 except requests.ConnectionError: |
199 self.stdout.write(" connection error", ending='') |
245 self.stdout.write(" connection error", ending='') |
200 logger.error("CONNECTION ERROR FOR %s", elem.title) |
246 Command.logger.error("CONNECTION ERROR FOR %s", elem.title) |
201 try: |
247 try: |
202 res = requests.head(elem, timeout=10).status_code |
248 res = requests.head(elem, timeout=10).status_code |
203 except requests.ConnectionError: |
249 except requests.ConnectionError: |
204 res = "connection error" |
250 Command.mycsvfile.writerow([ |
205 forcsv += [[ |
251 "Media", |
206 mysrc, |
252 mysrc, |
207 newsource, |
253 "No", |
208 0, |
254 "connection error", |
209 res, |
255 newsource |
210 ": website doesn't exist anymore", |
256 ]) |
211 numberofcontents(elem), |
|
212 numberofproject(elem) |
|
213 ]] |
|
214 continue |
257 continue |
215 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
258 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
216 forcsv += [[ |
259 Command.mycsvfile.writerow([ |
217 mysrc, |
260 "Media", |
218 newsource, |
261 mysrc, |
219 0, |
262 "No", |
220 "XXX", |
|
221 "missing schema on base source!", |
263 "missing schema on base source!", |
222 numberofcontents(elem), |
264 newsource |
223 numberofproject(elem) |
265 ]) |
224 ]] |
|
225 continue |
266 continue |
226 except requests.exceptions.Timeout: |
267 except requests.exceptions.Timeout: |
227 forcsv += [[ |
268 Command.mycsvfile.writerow([ |
228 mysrc, |
269 "Media", |
229 newsource, |
270 mysrc, |
230 0, |
271 "No", |
231 "XXX", |
|
232 "TIMEOUT!", |
272 "TIMEOUT!", |
233 numberofcontents(elem), |
273 newsource |
234 numberofproject(elem) |
274 ]) |
235 ]] |
|
236 continue |
275 continue |
237 else: |
276 else: |
238 forcsv += [[ |
277 Command.mycsvfile.writerow([ |
239 mysrc, |
278 "Media", |
240 newsource, |
279 mysrc, |
241 0, |
280 "No", |
242 res, |
|
243 "use source link : website doesn't work with https", |
281 "use source link : website doesn't work with https", |
244 numberofcontents(elem), |
282 newsource |
245 numberofproject(elem) |
283 ]) |
246 ]] |
|
247 continue |
284 continue |
248 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
285 except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): |
249 self.stdout.write(" Missing schema !", ending='') |
286 self.stdout.write(" Missing schema !", ending='') |
250 logger.warning("MISSING SCHEMA FOR %s", elem.title) |
287 Command.logger.warning("MISSING SCHEMA FOR %s", elem.title) |
251 forcsv += [[ |
288 Command.mycsvfile.writerow([ |
|
289 "Media", |
252 mysrc, |
290 mysrc, |
253 newsource, |
291 "No", |
254 0, |
|
255 "XXX", |
|
256 "missing schema!", |
292 "missing schema!", |
257 numberofcontents(elem), |
293 newsource |
258 numberofproject(elem) |
294 ]) |
259 ]] |
|
260 continue |
295 continue |
261 except requests.exceptions.Timeout: |
296 except requests.exceptions.Timeout: |
262 self.stdout.write(" Timeout !", ending='') |
297 self.stdout.write(" Timeout !", ending='') |
263 logger.warning("Timeout FOR %s", elem.title) |
298 Command.logger.warning("Timeout FOR %s", elem.title) |
264 forcsv += [[ |
299 Command.mycsvfile.writerow([ |
|
300 "Media", |
265 mysrc, |
301 mysrc, |
266 newsource, |
302 "No", |
267 0, |
|
268 "XXX", |
|
269 "TIMEOUT!", |
303 "TIMEOUT!", |
270 numberofcontents(elem), |
304 newsource |
271 numberofproject(elem) |
305 ]) |
272 ]] |
|
273 continue |
306 continue |
274 if res > 400: |
307 if res > 400: |
275 try: |
308 try: |
276 ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\ |
309 ressrc = requests.head(tohttps(elem.src, elem.videopath, 0),\ |
277 timeout=10).status_code |
310 timeout=10).status_code |
278 except (requests.exceptions.Timeout, requests.ConnectionError): |
311 except (requests.exceptions.Timeout, requests.ConnectionError): |
279 self.stdout.write(" can't access source/new files", ending='') |
312 self.stdout.write(" can't access source/new files", ending='') |
280 logger.warning("can't access %s", elem.title) |
313 Command.logger.warning("can't access %s", elem.title) |
281 res = "connection error" |
314 res = "connection error" |
282 forcsv += [[ |
315 Command.mycsvfile.writerow([ |
283 mysrc, |
316 "Media", |
284 newsource, |
317 mysrc, |
285 0, |
318 "No", |
286 res, |
|
287 "website doesn't exist anymore", |
319 "website doesn't exist anymore", |
288 numberofcontents(elem), |
320 newsource |
289 numberofproject(elem) |
321 ]) |
290 ]] |
|
291 continue |
322 continue |
292 if ressrc == 404: |
323 if ressrc == 404: |
293 self.stdout.write(" can't access source/new files", ending='') |
324 self.stdout.write(" can't access source/new files", ending='') |
294 logger.warning("can't access %s", elem.title) |
325 Command.logger.warning("can't access %s", elem.title) |
295 forcsv += [[ |
326 Command.mycsvfile.writerow([ |
296 mysrc, |
327 "Media", |
297 newsource, |
328 mysrc, |
298 0, |
329 "No", |
299 res, |
|
300 "can't access source/new files", |
330 "can't access source/new files", |
301 numberofcontents(elem), |
331 newsource |
302 numberofproject(elem) |
332 ]) |
303 ]] |
|
304 elif ressrc == 200: |
333 elif ressrc == 200: |
305 self.stdout.write( |
334 self.stdout.write( |
306 " file not transcoded yet :" |
335 " file not transcoded yet :" |
307 "keep source extension or wait transcoding to be done", |
336 "keep source extension or wait transcoding to be done", |
308 ending='') |
337 ending='') |
309 logger.warning("%s not transcoded yet", elem.title) |
338 Command.logger.warning("%s not transcoded yet", elem.title) |
310 forcsv += [[ |
339 Command.mycsvfile.writerow([ |
311 mysrc, |
340 "Media", |
312 newsource, |
341 mysrc, |
313 0, |
342 "No", |
314 res, |
|
315 "file not transcoded yet : keep source extension", |
343 "file not transcoded yet : keep source extension", |
316 numberofcontents(elem), |
344 newsource |
317 numberofproject(elem) |
345 ]) |
318 ]] |
|
319 continue |
346 continue |
320 self.stdout.write(" It works", ending='') |
347 self.stdout.write(" It works", ending='') |
321 if options['f']: |
|
322 elem.src = newsource |
|
323 alreadyin = False |
348 alreadyin = False |
324 for everyelem in models.Media.objects.all(): |
349 for everyelem in models.Media.objects.all(): |
325 if newsource == everyelem.src: |
350 if newsource == everyelem.src: |
326 alreadyin = True |
351 alreadyin = True |
327 break |
352 break |
328 if alreadyin: |
353 if alreadyin: |
329 self.stdout.write(" element already in table", ending='') |
354 self.stdout.write(" element already in table", ending='') |
330 logger.warning("%s already in table", elem.title) |
355 Command.logger.warning("%s already in table", elem.title) |
331 forcsv += [[ |
356 Command.mycsvfile.writerow([ |
332 mysrc, |
357 "Media", |
333 newsource, |
358 newsource, |
334 0, |
359 "No", |
335 res, |
360 "new source already in table" |
336 "element already in table", |
361 ]) |
337 numberofcontents(elem), |
362 continue |
338 numberofproject(elem) |
363 self.cleanmediaproject(elem, force, newsource) |
339 ]] |
364 Command.csvfile.close() |
340 continue |
|
341 if numberofcontents(elem) == 0: |
|
342 self.stdout.write(" source has no content", ending='') |
|
343 forcsv += [[ |
|
344 mysrc, |
|
345 newsource, |
|
346 0, |
|
347 res, |
|
348 "source has no content", |
|
349 numberofcontents(elem), |
|
350 numberofproject(elem) |
|
351 ]] |
|
352 continue |
|
353 if options['f']: |
|
354 elem.videopath = '' |
|
355 elem.save() |
|
356 ldtproj = models.Project.objects.filter\ |
|
357 (content=models.Content.objects.filter(media_obj_id=elem.id)[0]) |
|
358 if numberofproject(elem) == 0: |
|
359 self.stdout.write(" no project", ending='') |
|
360 logger.warning("NO PROJECT FOR %s", elem.title) |
|
361 forcsv += [[ |
|
362 mysrc, |
|
363 newsource, |
|
364 1, |
|
365 res, |
|
366 "It works but there is no project with this media", |
|
367 numberofcontents(elem), |
|
368 numberofproject(elem) |
|
369 ]] |
|
370 continue |
|
371 infos.write("\nProjects : \n") |
|
372 for singleproject in ldtproj: |
|
373 base_url = Site.objects.get_current().domain + settings.BASE_URL |
|
374 ldt_id = singleproject.ldt_id |
|
375 embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ |
|
376 "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ |
|
377 "player_id=player_project_{ldt_id}&" \ |
|
378 "ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id) |
|
379 infos.write("%s \n"%(embedurl)) |
|
380 infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\ |
|
381 % (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id)) |
|
382 if options['f']: |
|
383 for singleproject in ldtproj: |
|
384 root = etree.XML(singleproject.ldt.encode('utf-8'), parser) |
|
385 mediapathlen = len(root.xpath('medias/media')) |
|
386 if mediapathlen == 0: |
|
387 self.stdout.write(" le .ldt ne contient pas de media", ending='') |
|
388 infos.write("le .ldt ne contient pas de media") |
|
389 continue |
|
390 root.xpath('medias/media')[0].set("video", '') |
|
391 singleproject.ldt = etree.tostring(root) |
|
392 singleproject.save() |
|
393 infos.write("\nSuccessful !\n") |
|
394 logger.info("%s DONE\n", embedurl) |
|
395 self.stdout.write(" done", ending='') |
|
396 forcsv += [[ |
|
397 mysrc, |
|
398 newsource, |
|
399 1, |
|
400 res, |
|
401 "It works", |
|
402 numberofcontents(elem), |
|
403 numberofproject(elem) |
|
404 ]] |
|
405 if not options['f']: |
|
406 with open('mediaInformations.csv', 'wb') as csvfile: |
|
407 mycsvfile = csv.writer(csvfile) |
|
408 for mycsv in forcsv: |
|
409 mycsvfile.writerow(mycsv) |
|
410 infos.close() |
|