--- a/src/ldtplatform/management/commands/replacedelete.py Tue Mar 21 16:17:31 2017 +0100
+++ b/src/ldtplatform/management/commands/replacedelete.py Fri Mar 24 15:03:50 2017 +0100
@@ -22,7 +22,8 @@
'''
to https
'''
- if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4' or not re.match(r".*\..{3}$", source):
+ if source[len(source)-3:len(source)] == 'MP4' or source[len(source)-3:len(source)] == 'mp4'\
+ or not re.match(r".*\..{3}$", source):
tomp4 = 0
if tomp4 == 1:
source = source[0:len(source)-3]+"mp4"
@@ -41,8 +42,7 @@
if vidpath == 'rtmp://media.iri.centrepompidou.fr/ddc_player/':
if re.match(r".*\..{3}$", source):
return "https://media.iri.centrepompidou.fr/" + source[4:]
- else:
- return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4"
+ return "https://media.iri.centrepompidou.fr/" + source[4:] + ".mp4"
return "https://media.iri.centrepompidou.fr/video/ldtplatform/"+source
@@ -61,11 +61,102 @@
(content=models.Content.objects.filter(media_obj_id=source.id)[0]))
return 0
+def constructytembed(source):
+ '''
+ construct youtube video oembed link
+ '''
+ if re.match(r".*feature=player_embedded.+", source) != None:
+ return "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\
+ + source[len(source)-11:] +"&format=json"
+ return "http://www.youtube.com/oembed?url=" + source + "&format=json"
+
class Command(BaseCommand):
'''
Command class
'''
help = 'delete medias without contents, replace media\'s source by a new URL'
+ base_url = Site.objects.get_current().domain + settings.BASE_URL
+ parser = etree.XMLParser(encoding='utf-8')
+ logger = logging.getLogger(__name__)
+ csvfile = open('mediaInformations.csv', 'wb')
+ mycsvfile = csv.writer(csvfile)
+
+ def constructldtembed(self, ldtid):
+ '''
+ construct ldt embed
+ '''
+ return "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \
+ "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \
+ "player_id=player_project_{ldt_id}&" \
+ "ldt_id={ldt_id}".format(base_url=Command.base_url, ldt_id=ldtid)
+
+ def cleanmediaproject(self, element, force, newsrc=None):
+ '''
+ change media objects' videopath and source if necessary
+ change project .ldt
+ '''
+ basesrc = element.src
+ if force:
+ element.videopath = ''
+ element.save()
+ if newsrc != None:
+ if force:
+ element.src = newsrc
+ element.save()
+ Command.mycsvfile.writerow([
+ "Media",
+ basesrc,
+ "Yes",
+ "changing source/videopath",
+ newsrc, "\'\'"
+ ])
+ if numberofproject(element) == 0:
+ Command.mycsvfile.writerow([
+ "Project",
+ element.src,
+ "Yes",
+ "initializing object(no project)"
+ ])
+ if force:
+ mycontentid = models.Content.objects.filter(media_obj_id=element.id)[0].iri_id
+ try:
+ management.call_command('initfrontproject', mycontentid)
+ except Exception:
+ Command.mycsvfile.writerow([
+ "Project",
+ element.src,
+ "No",
+ "socket error"
+ ])
+ return
+ self.stdout.write(" Initializing project", ending='')
+ else:
+ self.stdout.write(" Project has to be initialized ", ending='')
+ return
+ ldtproj = models.Project.objects.filter\
+ (content=models.Content.objects.filter(media_obj_id=element.id)[0])
+ for singleproject in ldtproj:
+ root = etree.XML(singleproject.ldt.encode('utf-8'), Command.parser)
+ if root.xpath('medias/media') == []:
+ self.stdout.write(" le .ldt ne contient pas de media", ending='')
+ continue
+ if root.xpath('medias/media')[0].get("video") != '':
+ embedurl = self.constructldtembed(singleproject.ldt_id)
+ if force:
+ root.xpath('medias/media')[0].set("video", '')
+ self.stdout.write(" changing videopath arg in .ldt ")
+ Command.mycsvfile.writerow([
+ "Project",
+ embedurl,
+ "Yes",
+ "changing .ldt /medias/media/video",
+ "\'\'"
+ ])
+ singleproject.ldt = etree.tostring(root)
+ singleproject.save()
+ Command.logger.info("%s DONE\n", embedurl)
+ element.save()
+
def add_arguments(self, parser):
'''
add arguments
@@ -76,103 +167,59 @@
'''
handle
'''
- forcsv = [[
- "Source link",
- "Tried link",
+ Command.mycsvfile.writerow([
+ "Object type",
+ "which object",
"Change ?",
- "Request status",
- "Informations",
- "Number of contents",
- "Number of projects"
- ]]
- if options['f']:
- infos = open('results.txt', 'w')
- else:
- infos = open('projectsToChange.txt', 'w')
+ "What(if Y)/Why (if N)",
+ "How"
+ ])
+
+ force = bool(options['f'])
j = 0
- parser = etree.XMLParser(encoding='utf-8')
- logger = logging.getLogger(__name__)
files1 = models.Media.objects.all() #this list contains every media
for elem1 in files1:
if numberofcontents(elem1) == 0:
- if options['f']:
+ if force:
elem1.delete() #if there is no content
#linked to the media, the media is removed for the database
self.stdout.write(" No content found, media has been removed")
else:
- forcsv += [[
- elem1.src,
- "XXX",
- "XXX",
- "XXX",
- "NO CONTENT : media will be erased",
- "0",
- "0"
- ]]
+ self.stdout.write(" No content found, media will be removed")
+ Command.mycsvfile.writerow([
+ "Media",
+ elem1.src,
+ "Yes",
+ "deleting object (no content)"
+ ])
j += 1
continue
- if numberofproject(elem1) == 0:
- if options['f']:
- mycontentid = models.Content.objects.filter(media_obj_id=elem1.id)[0].iri_id
- management.call_command('initfrontproject', mycontentid)
- else:
- forcsv += [[
- elem1.src,
- "XXX",
- "XXX",
- "XXX",
- "No project : will be initialized",
- "1",
- "0"
- ]]
- ldtproj = models.Project.objects.filter\
- (content=models.Content.objects.filter(media_obj_id=elem1.id)[0])
if elem1.src.lower() == tohttps(elem1.src, elem1.videopath).lower():
- if options['f']:
- elem1.videopath = ''
- elem1.save()
- for singleproject in ldtproj:
- root = etree.XML(singleproject.ldt.encode('utf-8'), parser)
- mediapathlen = len(root.xpath('medias/media'))
- if mediapathlen == 0:
- self.stdout.write(" .ldt has no media", ending='')
- continue
- if options['f']:
- root.xpath('medias/media')[0].set("video", '')
- singleproject.ldt = etree.tostring(root)
- singleproject.save()
- self.stdout.write(" Project videopath modified!")
- infos.write(" Project videopath modified ")
+ self.cleanmediaproject(elem1, force)
if re.match(r".*\.youtube\.com.*", elem1.src) != None\
or re.match(r".*youtu\.be.+", elem1.src) != None:
- if re.match(r".*feature=player_embedded.+", elem1.src) != None:
- myembed = "http://www.youtube.com/oembed?url=http://youtube.com/watch?v="\
- + elem1.src[len(elem1.src)-11:] +"&format=json"
- else:
- myembed = "http://www.youtube.com/oembed?url=" + elem1.src + "&format=json"
+ myembed = constructytembed(elem1.src)
if requests.get(myembed).status_code == 404:
self.stdout.write("%s : Video doesn't exists"% elem1.src)
if numberofproject(elem1) > 0:
ldtproj = models.Project.objects.get(id=models.Content.objects.filter\
(media_obj_id=elem1.id)[0].front_project_id).ldt
- root = etree.XML(ldtproj.encode('utf-8'), parser)
+ root = etree.XML(ldtproj.encode('utf-8'), Command.parser)
if root.xpath('annotations/content/ensemble/decoupage/elements/element')\
== []:
- if options['f']:
+ if force:
elem1.delete()
self.stdout.write("video doesn't exist anymore : media deleted")
- else:
- forcsv += [[
- elem1.src,
- "XXX",
- "XXX",
- "404",
- "Video doesn't exist anymore + empty projects",
- "1",
- "1"
- ]]
-
- if options['f']:
+ Command.mycsvfile.writerow([
+ "Media/Content/Project",
+ elem1.src,
+ "Yes",
+ "deleting(Video doesn't exist anymore + empty projects)"
+ ])
+ j += 1
+ else:
+ self.cleanmediaproject(elem1,force)
+ if force:
self.stdout.write("%s files deleted"%j)
else:
self.stdout.write("%s files to delete"%j)
@@ -189,7 +236,6 @@
self.stdout.write(" \n%s/%s files done"%(i+1, len(files)), ending='')
i += 1
if numberofcontents(elem) == 0:
- self.stdout.write(" no content", ending='')
continue
mysrc = elem.src
newsource = tohttps(elem.src, elem.videopath)
@@ -197,79 +243,66 @@
res = requests.head(newsource, timeout=10).status_code
except requests.ConnectionError:
self.stdout.write(" connection error", ending='')
- logger.error("CONNECTION ERROR FOR %s", elem.title)
+ Command.logger.error("CONNECTION ERROR FOR %s", elem.title)
try:
res = requests.head(elem, timeout=10).status_code
except requests.ConnectionError:
- res = "connection error"
- forcsv += [[
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- res,
- ": website doesn't exist anymore",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ "No",
+ "connection error",
+ newsource
+ ])
continue
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
- forcsv += [[
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- "XXX",
+ "No",
"missing schema on base source!",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
except requests.exceptions.Timeout:
- forcsv += [[
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- "XXX",
+ "No",
"TIMEOUT!",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
else:
- forcsv += [[
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- res,
+ "No",
"use source link : website doesn't work with https",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema):
self.stdout.write(" Missing schema !", ending='')
- logger.warning("MISSING SCHEMA FOR %s", elem.title)
- forcsv += [[
+ Command.logger.warning("MISSING SCHEMA FOR %s", elem.title)
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- "XXX",
+ "No",
"missing schema!",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
except requests.exceptions.Timeout:
self.stdout.write(" Timeout !", ending='')
- logger.warning("Timeout FOR %s", elem.title)
- forcsv += [[
+ Command.logger.warning("Timeout FOR %s", elem.title)
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- "XXX",
+ "No",
"TIMEOUT!",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
if res > 400:
try:
@@ -277,49 +310,41 @@
timeout=10).status_code
except (requests.exceptions.Timeout, requests.ConnectionError):
self.stdout.write(" can't access source/new files", ending='')
- logger.warning("can't access %s", elem.title)
+ Command.logger.warning("can't access %s", elem.title)
res = "connection error"
- forcsv += [[
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- res,
+ "No",
"website doesn't exist anymore",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
if ressrc == 404:
self.stdout.write(" can't access source/new files", ending='')
- logger.warning("can't access %s", elem.title)
- forcsv += [[
+ Command.logger.warning("can't access %s", elem.title)
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- res,
+ "No",
"can't access source/new files",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
elif ressrc == 200:
self.stdout.write(
" file not transcoded yet :"
"keep source extension or wait transcoding to be done",
ending='')
- logger.warning("%s not transcoded yet", elem.title)
- forcsv += [[
+ Command.logger.warning("%s not transcoded yet", elem.title)
+ Command.mycsvfile.writerow([
+ "Media",
mysrc,
- newsource,
- 0,
- res,
+ "No",
"file not transcoded yet : keep source extension",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ newsource
+ ])
continue
self.stdout.write(" It works", ending='')
- if options['f']:
- elem.src = newsource
alreadyin = False
for everyelem in models.Media.objects.all():
if newsource == everyelem.src:
@@ -327,84 +352,13 @@
break
if alreadyin:
self.stdout.write(" element already in table", ending='')
- logger.warning("%s already in table", elem.title)
- forcsv += [[
- mysrc,
- newsource,
- 0,
- res,
- "element already in table",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
- continue
- if numberofcontents(elem) == 0:
- self.stdout.write(" source has no content", ending='')
- forcsv += [[
- mysrc,
+ Command.logger.warning("%s already in table", elem.title)
+ Command.mycsvfile.writerow([
+ "Media",
newsource,
- 0,
- res,
- "source has no content",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
- continue
- if options['f']:
- elem.videopath = ''
- elem.save()
- ldtproj = models.Project.objects.filter\
- (content=models.Content.objects.filter(media_obj_id=elem.id)[0])
- if numberofproject(elem) == 0:
- self.stdout.write(" no project", ending='')
- logger.warning("NO PROJECT FOR %s", elem.title)
- forcsv += [[
- mysrc,
- newsource,
- 1,
- res,
- "It works but there is no project with this media",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
+ "No",
+ "new source already in table"
+ ])
continue
- infos.write("\nProjects : \n")
- for singleproject in ldtproj:
- base_url = Site.objects.get_current().domain + settings.BASE_URL
- ldt_id = singleproject.ldt_id
- embedurl = "http://{base_url}ldtplatform/ldt/embed/v3/config?json_url=" \
- "http://{base_url}ldtplatform/ldt/cljson/id/{ldt_id}&" \
- "player_id=player_project_{ldt_id}&" \
- "ldt_id={ldt_id}".format(base_url=base_url, ldt_id=ldt_id)
- infos.write("%s \n"%(embedurl))
- infos.write("having as old media %s \nAs new media %s \nAs content %s \n"\
- % (elem.src, newsource, models.Content.objects.filter(media_obj_id=elem.id)[0].iri_id))
- if options['f']:
- for singleproject in ldtproj:
- root = etree.XML(singleproject.ldt.encode('utf-8'), parser)
- mediapathlen = len(root.xpath('medias/media'))
- if mediapathlen == 0:
- self.stdout.write(" le .ldt ne contient pas de media", ending='')
- infos.write("le .ldt ne contient pas de media")
- continue
- root.xpath('medias/media')[0].set("video", '')
- singleproject.ldt = etree.tostring(root)
- singleproject.save()
- infos.write("\nSuccessful !\n")
- logger.info("%s DONE\n", embedurl)
- self.stdout.write(" done", ending='')
- forcsv += [[
- mysrc,
- newsource,
- 1,
- res,
- "It works",
- numberofcontents(elem),
- numberofproject(elem)
- ]]
- if not options['f']:
- with open('mediaInformations.csv', 'wb') as csvfile:
- mycsvfile = csv.writer(csvfile)
- for mycsv in forcsv:
- mycsvfile.writerow(mycsv)
- infos.close()
+ self.cleanmediaproject(elem, force, newsource)
+ Command.csvfile.close()