diff -r 1fcf0bd88ba3 -r 24397932219d web/ldt/ldt_utils/fileimport.py --- a/web/ldt/ldt_utils/fileimport.py Thu Jun 17 11:24:33 2010 +0200 +++ b/web/ldt/ldt_utils/fileimport.py Thu Jun 17 15:04:36 2010 +0200 @@ -1,379 +1,376 @@ -import tempfile -import os.path -import shutil -from django.core.exceptions import ObjectDoesNotExist -from ldt.utils import zipfileext -from django.conf import settings -from models import Content -import xml.dom.minidom -import xml.dom.ext #@UnresolvedImport -import xml.xpath #@UnresolvedImport -import fnmatch -import uuid -import urllib - -class FileImportError(Exception): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - - -def Property(func): - return property(**func()) - -class IriInfo(object): - - - def __init__(self, id, order, titledesc, basepath="", videopath=settings.STREAM_URL, decoupage_blacklist = settings.DECOUPAGE_BLACKLIST, flatten = True): - self.id = id - self.basepath = basepath - self.order = order - self.src = "" - self.annotations = None - self.videopath = videopath - self.videourl = "" - self.title = None - self.desc = None - self.duration = None - self.created = False - self.content = None - self.decoupage_blacklist = decoupage_blacklist - if self.decoupage_blacklist is None: - self.decoupage_blacklist = () - self.flatten = flatten - - - - def processIri(self): - # for just import a file ldt and get the title for every media - if 'http' in self.src: - url = urllib.urlopen(self.src) - doc = xml.dom.minidom.parse(url) - doc = Ft.Xml.Domlette.ConvertDocument(doc) - con = xml.xpath.Context.Context(doc, 1, 1, None) - #open .iri and get the title - res = xml.xpath.Evaluate("/iri/head/meta[@name='title']/@content", context=con) - self.title = res[0].value - - #for import a zip, get title and copy file .iri in the media directory - else: - path = os.path.join(self.basepath, self.src) - doc = xml.dom.minidom.parse(path) - - con = xml.xpath.Context.Context(doc, 1, 1, None) - - res = xml.xpath.Evaluate("/iri/head/meta[@name='title']/@content", context=con) - self.title = res[0].value - - res = xml.xpath.Evaluate("/iri/body/ensembles",context=con) - ensemblesnode = res[0] - - ensembleids = [] - - for node in ensemblesnode.childNodes: - if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "ensemble": - id = node.getAttributeNS(None,u"id") - if id not in ensembleids: - ensembleids.append(id) - - if self.annotations is not None: - newEnsemble = None - for cnode in self.annotations.childNodes: - if cnode.nodeType == xml.dom.Node.ELEMENT_NODE and cnode.tagName == "decoupage": - if newEnsemble is None: - newensemble = doc.createElement('ensemble') - ensembleid = self.id+"_"+str(uuid.uuid1()) - newensemble.setAttributeNS(None,'id',ensembleid) - - newensemble.setAttributeNS(None,'title', self.annotations.getAttribute('title')) - newensemble.setAttributeNS(None,'author', self.annotations.getAttribute('author')) - newensemble.setAttributeNS(None,'date', self.annotations.getAttribute('date')) - newensemble.setAttributeNS(None,'abstract', self.annotations.getAttribute('abstract')) - ensemblesnode.appendChild(newensemble) - ensembleids.append(ensembleid) - newDecoupageNode = cnode.cloneNode(True) - newensemble.appendChild(newDecoupageNode) - elif cnode.nodeType == xml.dom.Node.ELEMENT_NODE and cnode.tagName == "ensemble": - ensembleid = cnode.getAttribute(u"id") - cloneNode = cnode.cloneNode(True) - if ensembleid in ensembleids: - ensembleid = self.id+"_"+str(uuid.uuid1()) - cloneNode.setAttribute(u"id", ensembleid) - ensembleids.append(ensembleid) - ensemblesnode.appendChild(cloneNode) - - res = xml.xpath.Evaluate("/iri/body/medias/media[@id='video']/video", context=con) - if self.flatten: - src_video = res[0].getAttribute('src') - self.videourl = os.path.basename(src_video) - res[0].setAttributeNS(None,'src', self.videourl) - self.duration = res[0].getAttributeNS(None, u'dur') - - f = open(path, "w") - try: - xml.dom.ext.Print(doc, stream=f) - finally: - f.close() - - - destPath = os.path.join(os.path.join(os.path.join(settings.MEDIA_ROOT, "media"), "ldt"), self.id); - if not os.path.exists(destPath): - os.makedirs(destPath) - shutil.move(os.path.join(self.basepath, self.src), os.path.join(destPath, os.path.basename(self.src))) - self.src = self.id + u"/" + os.path.basename(self.src) - - - - def saveContent(self): - #if 'http' in self.src: - # url = self.src - #else: - # url = self.id + u"/" + os.path.basename(self.src) - content, self.created = Content.objects.get_or_create(iri_id=self.id, defaults = {'src':self.videourl, 'iriurl': self.src, 'title':self.title, 'description':self.desc, 'videopath': self.videopath}) - if not self.created: - content.iriurl = self.src - content.title = self.title - content.description = self.desc - content.save() - - content.iriurl = self.src - content.videopath = self.videopath.rstrip("/") + "/" - - content.iri = self.id + u"/" + os.path.basename(self.src) - content.title = self.title - content.description = self.desc - content.duration = int(self.duration) - content.save() - - self.content = content - - def process(self): - self.processIri() - self.saveContent() - -class BaseFileImport(object): - - def __init__(self, filepath, videopath): - self.__filepath = filepath - self.__tempdir = "" - self.__videopath = videopath - self.__author = None - - - def filepath(): #@NoSelf - doc = """Docstring""" #@UnusedVariable - - def fget(self): - return self.__filepath - - def fset(self, value): - self.__filepath = value - - def fdel(self): - del self.__filepath - - return locals() - - filepath = property(**filepath()) - - def videopath(): #@NoSelf - doc = """Docstring""" #@UnusedVariable - - def fget(self): - return self.__videopath - - def fset(self, value): - self.__videopath = value - - def fdel(self): - del self.__videopath - - return locals() - - videopath = property(**videopath()) - - def author(): #@NoSelf - doc = """Docstring""" #@UnusedVariable - - def fget(self): - return self.__author - - def fset(self, value): - self.__author = value - - def fdel(self): - del self.__author - - return locals() - - author = property(**author()) - - -class FileImport(BaseFileImport): - - def __init__(self, filepath, videopath, flatten): - BaseFileImport.__init__(self, filepath, videopath) - self.__checkExistingMedia = False - self.__flatten = flatten - - def checkExistingMedia(): #@NoSelf - doc = """Docstring""" #@UnusedVariable - - def fget(self): - return self.__checkExistingMedia - - def fset(self, value): - self.__checkExistingMedia = value - - def fdel(self): - del self.__checkExistingMedia - - return locals() - - checkExistingMedia = property(**checkExistingMedia()) - - def flatten(): #@NoSelf - doc = """Docstring""" #@UnusedVariable - - def fget(self): - return self.__flatten - - def fset(self, value): - self.__flatten = value - - def fdel(self): - del self.__flatten - - return locals() - flatten = property(**flatten()) - - def processLdt(self, ldtpath=None): - - # list iri - # see if there is some comments - # inject comment in iri - # copy iri in folder - # create or update content - contents = {} - if ldtpath: - doc = xml.dom.minidom.parse(ldtpath) - else: - doc = xml.dom.minidom.parse(self.filepath) - - con = xml.xpath.Context.Context(doc, 1, 1, None) - - #get author from file ldt - result = xml.xpath.Evaluate("/iri/project", context=con) - for pnode in result: - author = pnode.getAttributeNS(None,u"user") - if author: - self.author = unicode(author) - break - - result = xml.xpath.Evaluate("/iri/medias/media", context=con) - - for i, medianode in enumerate(result): - # get iri file's id from file ldt - id = medianode.attributes['id'].value - if self.checkExistingMedia: - try: - Content.objects.get(iri_id=id) - do_pass = True - except ObjectDoesNotExist: #Content.DoesNotExist - do_pass = False - else: - do_pass = False - if not do_pass: - if not (contents.has_key(id)): - # Create instance iriInfo(id, order, titledesc, basepath="", videopath=settings.STREAM_URL) - if ldtpath: - contents[id] = IriInfo(id, i, "", os.path.dirname(ldtpath), flatten=self.flatten) - else: - contents[id] = IriInfo(id, i, "", flatten=self.flatten) - # Get iri file's url from ldt. This url can be relative path or absolute path. - contents[id].src = medianode.attributes['src'].value - if medianode.attributes['video'].value !="": - contents[id].videopath = medianode.attributes['video'].value - elif self.videopath !="" or self.videopath: - contents[id].videopath = self.videopath - else: - contents[id].videopath =settings.STREAM_URL - - - #get annotation of file ldt - result = xml.xpath.Evaluate("/iri/annotations/content", context=con) - - for contentnode in result: - id = contentnode.attributes['id'].value - # pocketfilms.utils.log.debug("ID : " + str(id)) - if contents.has_key(id): - if self.author: - contentnode.setAttributeNS(None,"author", unicode(self.author)) - contents[id].annotations = contentnode - - #go throught values - for iriinfo in contents.values(): - - iriinfo.process() - - # if yes update - # if no create - # move iri file to the proper place - #return list of iriInfo - - def processFile(self): - if self.filepath.name.endswith(".ldt"): - self.processLdt() - elif self.filepath.name.endswith(".zip"): - self.processZip() - else: - raise FileImportError("Bad file type") - - - def processZip(self): - # """ extraire .zip, pass to method processLdt""" - # create temp directory - self.__tempdir = tempfile.mkdtemp() - openfiles = [] - flvfiles = [] - processedids = [] - - try: - zipfile = zipfileext.ZipFileExt(self.filepath) - zipfile.unzip_into_dir(self.__tempdir) - #load ldt - foldersToProcess = [self.__tempdir] - while len(foldersToProcess): - # pocketfilms.utils.log.debug("folder stack length : "+ str(len(foldersToProcess))) - currentFolder = foldersToProcess.pop() - for entry in os.listdir(currentFolder): - if entry in settings.ZIP_BLACKLIST: - continue - entryPath = os.path.join(currentFolder, entry) - if(os.path.isdir(entryPath)): - # pocketfilms.utils.log.debug("Push folder : " + entryPath) - foldersToProcess.append(entryPath) - elif fnmatch.fnmatch(entry, "*.ldt"): - # pocketfilms.utils.log.debug("Process file : " + entryPath) - ldtid = self.processLdt(entryPath) - processedids.append(ldtid) - elif fnmatch.fnmatch(entry, "*.flv"): - flvfiles.append(entryPath) - - if settings.CONTENT_ROOT and os.path.exists(settings.CONTENT_ROOT): - for entry in flvfiles: - shutil.copy(entry, settings.CONTENT_ROOT) - - finally: - for f in openfiles: - f.close() - shutil.rmtree(self.__tempdir) - # delete directory - return processedids - - # def processFileLdt(self): - # processedids = [] - # ldtid = self.processLdt(self.filepath) - # processedids.append(ldtid) - # return processedids +import tempfile +import os.path +import shutil +from django.core.exceptions import ObjectDoesNotExist +from ldt.utils import zipfileext +from django.conf import settings +from models import Content +import xml.dom.minidom +import xml.dom.ext #@UnresolvedImport +import xml.xpath #@UnresolvedImport +import fnmatch +import uuid +import urllib + +class FileImportError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + + +def Property(func): + return property(**func()) + +class IriInfo(object): + + + def __init__(self, id, order, titledesc, basepath="", videopath=settings.STREAM_URL, decoupage_blacklist = settings.DECOUPAGE_BLACKLIST, flatten = True): + self.id = id + self.basepath = basepath + self.order = order + self.src = "" + self.annotations = None + self.videopath = videopath + self.videourl = "" + self.title = None + self.desc = None + self.duration = None + self.created = False + self.content = None + self.decoupage_blacklist = decoupage_blacklist + if self.decoupage_blacklist is None: + self.decoupage_blacklist = () + self.flatten = flatten + + + + def processIri(self): + # for just import a file ldt and get the title for every media + if 'http' in self.src: + url = urllib.urlopen(self.src) + doc = xml.dom.minidom.parse(url) + #for import a zip, get title and copy file .iri in the media directory + else: + path = os.path.join(self.basepath, self.src) + doc = xml.dom.minidom.parse(path) + + + doc = Ft.Xml.Domlette.ConvertDocument(doc) + con = xml.xpath.Context.Context(doc, 1, 1, None) + + res = xml.xpath.Evaluate("/iri/head/meta[@name='title']/@content", context=con) + self.title = res[0].value + + + res = xml.xpath.Evaluate("/iri/body/ensembles",context=con) + ensemblesnode = res[0] + + ensembleids = [] + + for node in ensemblesnode.childNodes: + if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "ensemble": + id = node.getAttributeNS(None,u"id") + if id not in ensembleids: + ensembleids.append(id) + + if self.annotations is not None: + newEnsemble = None + for cnode in self.annotations.childNodes: + if cnode.nodeType == xml.dom.Node.ELEMENT_NODE and cnode.tagName == "decoupage": + if newEnsemble is None: + newensemble = doc.createElement('ensemble') + ensembleid = self.id+"_"+str(uuid.uuid1()) + newensemble.setAttributeNS(None,'id',ensembleid) + + newensemble.setAttributeNS(None,'title', self.annotations.getAttribute('title')) + newensemble.setAttributeNS(None,'author', self.annotations.getAttribute('author')) + newensemble.setAttributeNS(None,'date', self.annotations.getAttribute('date')) + newensemble.setAttributeNS(None,'abstract', self.annotations.getAttribute('abstract')) + ensemblesnode.appendChild(newensemble) + ensembleids.append(ensembleid) + newDecoupageNode = cnode.cloneNode(True) + newensemble.appendChild(newDecoupageNode) + elif cnode.nodeType == xml.dom.Node.ELEMENT_NODE and cnode.tagName == "ensemble": + ensembleid = cnode.getAttribute(u"id") + cloneNode = cnode.cloneNode(True) + if ensembleid in ensembleids: + ensembleid = self.id+"_"+str(uuid.uuid1()) + cloneNode.setAttribute(u"id", ensembleid) + ensembleids.append(ensembleid) + ensemblesnode.appendChild(cloneNode) + + res = xml.xpath.Evaluate("/iri/body/medias/media[@id='video']/video", context=con) + if self.flatten: + src_video = res[0].getAttributeNS(None,'src') + self.videourl = os.path.basename(src_video) + res[0].setAttributeNS(None,'src', self.videourl) + self.duration = res[0].getAttributeNS(None, u'dur') + + f = open(path, "w") + try: + xml.dom.ext.Print(doc, stream=f) + finally: + f.close() + + + destPath = os.path.join(os.path.join(os.path.join(settings.MEDIA_ROOT, "media"), "ldt"), self.id); + if not os.path.exists(destPath): + os.makedirs(destPath) + shutil.move(os.path.join(self.basepath, self.src), os.path.join(destPath, os.path.basename(self.src))) + self.src = self.id + u"/" + os.path.basename(self.src) + + + + def saveContent(self): + #if 'http' in self.src: + # url = self.src + #else: + # url = self.id + u"/" + os.path.basename(self.src) + content, self.created = Content.objects.get_or_create(iri_id=self.id, defaults = {'src':self.videourl, 'iriurl': self.src, 'title':self.title, 'description':self.desc, 'videopath': self.videopath}) + if not self.created: + content.iriurl = self.src + content.title = self.title + content.description = self.desc + content.save() + + content.iriurl = self.src + content.videopath = self.videopath.rstrip("/") + "/" + + content.iri = self.id + u"/" + os.path.basename(self.src) + content.title = self.title + content.description = self.desc + content.duration = int(self.duration) + content.save() + + self.content = content + + def process(self): + self.processIri() + self.saveContent() + +class BaseFileImport(object): + + def __init__(self, filepath, videopath): + self.__filepath = filepath + self.__tempdir = "" + self.__videopath = videopath + self.__author = None + + + def filepath(): #@NoSelf + doc = """Docstring""" #@UnusedVariable + + def fget(self): + return self.__filepath + + def fset(self, value): + self.__filepath = value + + def fdel(self): + del self.__filepath + + return locals() + + filepath = property(**filepath()) + + def videopath(): #@NoSelf + doc = """Docstring""" #@UnusedVariable + + def fget(self): + return self.__videopath + + def fset(self, value): + self.__videopath = value + + def fdel(self): + del self.__videopath + + return locals() + + videopath = property(**videopath()) + + def author(): #@NoSelf + doc = """Docstring""" #@UnusedVariable + + def fget(self): + return self.__author + + def fset(self, value): + self.__author = value + + def fdel(self): + del self.__author + + return locals() + + author = property(**author()) + + +class FileImport(BaseFileImport): + + def __init__(self, filepath, videopath, flatten): + BaseFileImport.__init__(self, filepath, videopath) + self.__checkExistingMedia = False + self.__flatten = flatten + + def checkExistingMedia(): #@NoSelf + doc = """Docstring""" #@UnusedVariable + + def fget(self): + return self.__checkExistingMedia + + def fset(self, value): + self.__checkExistingMedia = value + + def fdel(self): + del self.__checkExistingMedia + + return locals() + + checkExistingMedia = property(**checkExistingMedia()) + + def flatten(): #@NoSelf + doc = """Docstring""" #@UnusedVariable + + def fget(self): + return self.__flatten + + def fset(self, value): + self.__flatten = value + + def fdel(self): + del self.__flatten + + return locals() + flatten = property(**flatten()) + + def processLdt(self, ldtpath=None): + + # list iri + # see if there is some comments + # inject comment in iri + # copy iri in folder + # create or update content + contents = {} + if ldtpath: + doc = xml.dom.minidom.parse(ldtpath) + else: + doc = xml.dom.minidom.parse(self.filepath) + + con = xml.xpath.Context.Context(doc, 1, 1, None) + + #get author from file ldt + result = xml.xpath.Evaluate("/iri/project", context=con) + for pnode in result: + author = pnode.getAttributeNS(None,u"user") + if author: + self.author = unicode(author) + break + + result = xml.xpath.Evaluate("/iri/medias/media", context=con) + + for i, medianode in enumerate(result): + # get iri file's id from file ldt + id = medianode.attributes['id'].value + if self.checkExistingMedia: + try: + Content.objects.get(iri_id=id) + do_pass = True + except ObjectDoesNotExist: #Content.DoesNotExist + do_pass = False + else: + do_pass = False + if not do_pass: + if not (contents.has_key(id)): + # Create instance iriInfo(id, order, titledesc, basepath="", videopath=settings.STREAM_URL) + if ldtpath: + contents[id] = IriInfo(id, i, "", os.path.dirname(ldtpath), flatten=self.flatten) + else: + contents[id] = IriInfo(id, i, "", flatten=self.flatten) + # Get iri file's url from ldt. This url can be relative path or absolute path. + contents[id].src = medianode.attributes['src'].value + if medianode.attributes['video'].value !="": + contents[id].videopath = medianode.attributes['video'].value + elif self.videopath !="" or self.videopath: + contents[id].videopath = self.videopath + else: + contents[id].videopath =settings.STREAM_URL + + + #get annotation of file ldt + result = xml.xpath.Evaluate("/iri/annotations/content", context=con) + + for contentnode in result: + id = contentnode.attributes['id'].value + # pocketfilms.utils.log.debug("ID : " + str(id)) + if contents.has_key(id): + if self.author: + contentnode.setAttributeNS(None,"author", unicode(self.author)) + contents[id].annotations = contentnode + + #go throught values + for iriinfo in contents.values(): + + iriinfo.process() + + # if yes update + # if no create + # move iri file to the proper place + #return list of iriInfo + + def processFile(self): + if self.filepath.name.endswith(".ldt"): + self.processLdt() + elif self.filepath.name.endswith(".zip"): + self.processZip() + else: + raise FileImportError("Bad file type") + + + def processZip(self): + # """ extraire .zip, pass to method processLdt""" + # create temp directory + self.__tempdir = tempfile.mkdtemp() + openfiles = [] + flvfiles = [] + processedids = [] + + try: + zipfile = zipfileext.ZipFileExt(self.filepath) + zipfile.unzip_into_dir(self.__tempdir) + #load ldt + foldersToProcess = [self.__tempdir] + while len(foldersToProcess): + # pocketfilms.utils.log.debug("folder stack length : "+ str(len(foldersToProcess))) + currentFolder = foldersToProcess.pop() + for entry in os.listdir(currentFolder): + if entry in settings.ZIP_BLACKLIST: + continue + entryPath = os.path.join(currentFolder, entry) + if(os.path.isdir(entryPath)): + # pocketfilms.utils.log.debug("Push folder : " + entryPath) + foldersToProcess.append(entryPath) + elif fnmatch.fnmatch(entry, "*.ldt"): + # pocketfilms.utils.log.debug("Process file : " + entryPath) + ldtid = self.processLdt(entryPath) + processedids.append(ldtid) + elif fnmatch.fnmatch(entry, "*.flv"): + flvfiles.append(entryPath) + + if settings.CONTENT_ROOT and os.path.exists(settings.CONTENT_ROOT): + for entry in flvfiles: + shutil.copy(entry, settings.CONTENT_ROOT) + + finally: + for f in openfiles: + f.close() + shutil.rmtree(self.__tempdir) + # delete directory + return processedids + + # def processFileLdt(self): + # processedids = [] + # ldtid = self.processLdt(self.filepath) + # processedids.append(ldtid) + # return processedids