# HG changeset patch # User bellierp # Date 1490364230 -3600 # Node ID 324717f075f9e1c9f48389844a1d0ee3d9bb3ff3 # Parent 509b57af708d63a2718f9ca857e15dc08fcb4d71 better informations about objects changed diff -r 509b57af708d -r 324717f075f9 src/ldtplatform/management/commands/replacedelete.py --- a/src/ldtplatform/management/commands/replacedelete.py Tue Mar 21 16:17:31 2017 +0100 +++ b/src/ldtplatform/management/commands/replacedelete.py Fri Mar 24 15:03:50 2017 +0100 @@ -22,7 +22,8 @@ ''' to https ''' - if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4' or not re.match(r".*\..{3}$", source): + if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4'\ + or not re.match(r".*\..{3}$", source): tomp4 = 0 if tomp4 == 1: source = source[0:len(source)-3]+"mp4" @@ -41,8 +42,7 @@ if vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/': if re.match(r".*\..{3}$", source): return "https://media.iri.centrepompidou.fr/" + source[4:] - else: - return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4" + return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4" return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source @@ -61,11 +61,102 @@ (content=models.Content.objects.filter(media_obj_id=source.id)[0])) return 0 +def constructytembed(source): + ''' + construct youtube video oembed link + ''' + if re.match(r".*feature=player_embedded.+", source) != None: + return "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\ + + source[len(source)-11:] +"&format=json" + return "http://www.youtube.com/oembed?url=" + source + "&format=json" + class Command(BaseCommand): ''' Command class ''' help = 'delete medias without contents, replace media\'s source by a new URL' + base_url = Site.objects.get_current().domain + settings.BASE_URL + parser = etree.XMLParser(encoding='utf-8') + logger = logging.getLogger(__name__) + csvfile = open('mediaInformations.csv', 'wb') + mycsvfile = csv.writer(csvfile) + + def constructldtembed(self, ldtid): + ''' + construct ldt embed + ''' + return "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ + "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ + "player_id=player_project_{ldt_id}&" \ + "ldt_id={ldt_id}".format(base_url=Command.base_url, ldt_id=ldtid) + + def cleanmediaproject(self, element, force, newsrc=None): + ''' + change media objects' videopath and source if necessary + change project .ldt + ''' + basesrc = element.src + if force: + element.videopath = '' + element.save() + if newsrc != None: + if force: + element.src = newsrc + element.save() + Command.mycsvfile.writerow([ + "Media", + basesrc, + "Yes", + "changing source/videopath", + newsrc, "\'\'" + ]) + if numberofproject(element) == 0: + Command.mycsvfile.writerow([ + "Project", + element.src, + "Yes", + "initializing object(no project)" + ]) + if force: + mycontentid = models.Content.objects.filter(media_obj_id=element.id)[0].iri_id + try: + management.call_command('initfrontproject', mycontentid) + except Exception: + Command.mycsvfile.writerow([ + "Project", + element.src, + "No", + "socket error" + ]) + return + self.stdout.write(" Initializing project", ending='') + else: + self.stdout.write(" Project has to be initialized ", ending='') + return + ldtproj = models.Project.objects.filter\ + (content=models.Content.objects.filter(media_obj_id=element.id)[0]) + for singleproject in ldtproj: + root = etree.XML(singleproject.ldt.encode('utf-8'), Command.parser) + if root.xpath('medias/media') == []: + self.stdout.write(" le .ldt ne contient pas de media", ending='') + continue + if root.xpath('medias/media')[0].get("video") != '': + embedurl = self.constructldtembed(singleproject.ldt_id) + if force: + root.xpath('medias/media')[0].set("video", '') + self.stdout.write(" changing videopath arg in .ldt ") + Command.mycsvfile.writerow([ + "Project", + embedurl, + "Yes", + "changing .ldt /medias/media/video", + "\'\'" + ]) + singleproject.ldt = etree.tostring(root) + singleproject.save() + Command.logger.info("%s DONE\n", embedurl) + element.save() + def add_arguments(self, parser): ''' add arguments @@ -76,103 +167,59 @@ ''' handle ''' - forcsv = [[ - "Source link", - "Tried link", + Command.mycsvfile.writerow([ + "Object type", + "which object", "Change ?", - "Request status", - "Informations", - "Number of contents", - "Number of projects" - ]] - if options['f']: - infos = open('results.txt', 'w') - else: - infos = open('projectsToChange.txt', 'w') + "What(if Y)/Why (if N)", + "How" + ]) + + force = bool(options['f']) j = 0 - parser = etree.XMLParser(encoding='utf-8') - logger = logging.getLogger(__name__) files1 = models.Media.objects.all() #this list contains every media for elem1 in files1: if numberofcontents(elem1) == 0: - if options['f']: + if force: elem1.delete() #if there is no content #linked to the media, the media is removed for the database self.stdout.write(" No content found, media has been removed") else: - forcsv += [[ - elem1.src, - "XXX", - "XXX", - "XXX", - "NO CONTENT : media will be erased", - "0", - "0" - ]] + self.stdout.write(" No content found, media will be removed") + Command.mycsvfile.writerow([ + "Media", + elem1.src, + "Yes", + "deleting object (no content)" + ]) j += 1 continue - if numberofproject(elem1) == 0: - if options['f']: - mycontentid = models.Content.objects.filter(media_obj_id=elem1.id)[0].iri_id - management.call_command('initfrontproject', mycontentid) - else: - forcsv += [[ - elem1.src, - "XXX", - "XXX", - "XXX", - "No project : will be initialized", - "1", - "0" - ]] - ldtproj = models.Project.objects.filter\ - (content=models.Content.objects.filter(media_obj_id=elem1.id)[0]) if elem1.src.lower() == tohttps(elem1.src, elem1.videopath).lower(): - if options['f']: - elem1.videopath = '' - elem1.save() - for singleproject in ldtproj: - root = etree.XML(singleproject.ldt.encode('utf-8'), parser) - mediapathlen = len(root.xpath('medias/media')) - if mediapathlen == 0: - self.stdout.write(" .ldt has no media", ending='') - continue - if options['f']: - root.xpath('medias/media')[0].set("video", '') - singleproject.ldt = etree.tostring(root) - singleproject.save() - self.stdout.write(" Project videopath modified!") - infos.write(" Project videopath modified ") + self.cleanmediaproject(elem1, force) if re.match(r".*\.youtube\.com.*", elem1.src) != None\ or re.match(r".*youtu\.be.+", elem1.src) != None: - if re.match(r".*feature=player_embedded.+", elem1.src) != None: - myembed = "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\ - + elem1.src[len(elem1.src)-11:] +"&format=json" - else: - myembed = "http://www.youtube.com/oembed?url=" + elem1.src + "&format=json" + myembed = constructytembed(elem1.src) if requests.get(myembed).status_code == 404: self.stdout.write("%s : Video doesn't exists"% elem1.src) if numberofproject(elem1) > 0: ldtproj = models.Project.objects.get(id=models.Content.objects.filter\ (media_obj_id=elem1.id)[0].front_project_id).ldt - root = etree.XML(ldtproj.encode('utf-8'), parser) + root = etree.XML(ldtproj.encode('utf-8'), Command.parser) if root.xpath('annotations/content/ensemble/decoupage/elements/element')\ == []: - if options['f']: + if force: elem1.delete() self.stdout.write("video doesn't exist anymore : media deleted") - else: - forcsv += [[ - elem1.src, - "XXX", - "XXX", - "404", - "Video doesn't exist anymore + empty projects", - "1", - "1" - ]] - - if options['f']: + Command.mycsvfile.writerow([ + "Media/Content/Project", + elem1.src, + "Yes", + "deleting(Video doesn't exist anymore + empty projects)" + ]) + j += 1 + else: + self.cleanmediaproject(elem1,force) + if force: self.stdout.write("%s files deleted"%j) else: self.stdout.write("%s files to delete"%j) @@ -189,7 +236,6 @@ self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='') i += 1 if numberofcontents(elem) == 0: - self.stdout.write(" no content", ending='') continue mysrc = elem.src newsource = tohttps(elem.src, elem.videopath) @@ -197,79 +243,66 @@ res = requests.head(newsource, timeout=10).status_code except requests.ConnectionError: self.stdout.write(" connection error", ending='') - logger.error("CONNECTION ERROR FOR %s", elem.title) + Command.logger.error("CONNECTION ERROR FOR %s", elem.title) try: res = requests.head(elem, timeout=10).status_code except requests.ConnectionError: - res = "connection error" - forcsv += [[ + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - res, - ": website doesn't exist anymore", - numberofcontents(elem), - numberofproject(elem) - ]] + "No", + "connection error", + newsource + ]) continue except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): - forcsv += [[ + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - "XXX", + "No", "missing schema on base source!", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue except requests.exceptions.Timeout: - forcsv += [[ + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - "XXX", + "No", "TIMEOUT!", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue else: - forcsv += [[ + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - res, + "No", "use source link : website doesn't work with https", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema): self.stdout.write(" Missing schema !", ending='') - logger.warning("MISSING SCHEMA FOR %s", elem.title) - forcsv += [[ + Command.logger.warning("MISSING SCHEMA FOR %s", elem.title) + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - "XXX", + "No", "missing schema!", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue except requests.exceptions.Timeout: self.stdout.write(" Timeout !", ending='') - logger.warning("Timeout FOR %s", elem.title) - forcsv += [[ + Command.logger.warning("Timeout FOR %s", elem.title) + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - "XXX", + "No", "TIMEOUT!", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue if res > 400: try: @@ -277,49 +310,41 @@ timeout=10).status_code except (requests.exceptions.Timeout, requests.ConnectionError): self.stdout.write(" can't access source/new files", ending='') - logger.warning("can't access %s", elem.title) + Command.logger.warning("can't access %s", elem.title) res = "connection error" - forcsv += [[ + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - res, + "No", "website doesn't exist anymore", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue if ressrc == 404: self.stdout.write(" can't access source/new files", ending='') - logger.warning("can't access %s", elem.title) - forcsv += [[ + Command.logger.warning("can't access %s", elem.title) + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - res, + "No", "can't access source/new files", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) elif ressrc == 200: self.stdout.write( " file not transcoded yet :" "keep source extension or wait transcoding to be done", ending='') - logger.warning("%s not transcoded yet", elem.title) - forcsv += [[ + Command.logger.warning("%s not transcoded yet", elem.title) + Command.mycsvfile.writerow([ + "Media", mysrc, - newsource, - 0, - res, + "No", "file not transcoded yet : keep source extension", - numberofcontents(elem), - numberofproject(elem) - ]] + newsource + ]) continue self.stdout.write(" It works", ending='') - if options['f']: - elem.src = newsource alreadyin = False for everyelem in models.Media.objects.all(): if newsource == everyelem.src: @@ -327,84 +352,13 @@ break if alreadyin: self.stdout.write(" element already in table", ending='') - logger.warning("%s already in table", elem.title) - forcsv += [[ - mysrc, - newsource, - 0, - res, - "element already in table", - numberofcontents(elem), - numberofproject(elem) - ]] - continue - if numberofcontents(elem) == 0: - self.stdout.write(" source has no content", ending='') - forcsv += [[ - mysrc, + Command.logger.warning("%s already in table", elem.title) + Command.mycsvfile.writerow([ + "Media", newsource, - 0, - res, - "source has no content", - numberofcontents(elem), - numberofproject(elem) - ]] - continue - if options['f']: - elem.videopath = '' - elem.save() - ldtproj = models.Project.objects.filter\ - (content=models.Content.objects.filter(media_obj_id=elem.id)[0]) - if numberofproject(elem) == 0: - self.stdout.write(" no project", ending='') - logger.warning("NO PROJECT FOR %s", elem.title) - forcsv += [[ - mysrc, - newsource, - 1, - res, - "It works but there is no project with this media", - numberofcontents(elem), - numberofproject(elem) - ]] + "No", + "new source already in table" + ]) continue - infos.write("\nProjects : \n") - for singleproject in ldtproj: - base_url = Site.objects.get_current().domain + settings.BASE_URL - ldt_id = singleproject.ldt_id - embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \ - "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \ - "player_id=player_project_{ldt_id}&" \ - "ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id) - infos.write("%s \n"%(embedurl)) - infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\ - % (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id)) - if options['f']: - for singleproject in ldtproj: - root = etree.XML(singleproject.ldt.encode('utf-8'), parser) - mediapathlen = len(root.xpath('medias/media')) - if mediapathlen == 0: - self.stdout.write(" le .ldt ne contient pas de media", ending='') - infos.write("le .ldt ne contient pas de media") - continue - root.xpath('medias/media')[0].set("video", '') - singleproject.ldt = etree.tostring(root) - singleproject.save() - infos.write("\nSuccessful !\n") - logger.info("%s DONE\n", embedurl) - self.stdout.write(" done", ending='') - forcsv += [[ - mysrc, - newsource, - 1, - res, - "It works", - numberofcontents(elem), - numberofproject(elem) - ]] - if not options['f']: - with open('mediaInformations.csv', 'wb') as csvfile: - mycsvfile = csv.writer(csvfile) - for mycsv in forcsv: - mycsvfile.writerow(mycsv) - infos.close() + self.cleanmediaproject(elem, force, newsource) + Command.csvfile.close()