| author | ymh <ymh.work@gmail.com> |
| Wed, 20 Jan 2010 12:37:40 +0100 | |
| changeset 3 | 526ebd3988b0 |
| parent 0 | 0d40e90630ef |
| child 29 | cc9b7e14412b |
| permissions | -rw-r--r-- |
| 0 | 1 |
import tempfile |
2 |
import os |
|
3 |
import os.path |
|
4 |
import shutil |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
5 |
from blinkster.utils import zipfileext |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
6 |
import blinkster.utils.log |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
7 |
import blinkster.utils.xml |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
8 |
from blinkster import settings |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
9 |
from blinkster.ldt.models import Content, LdtProject |
| 0 | 10 |
import xml |
11 |
import xml.dom |
|
12 |
import xml.dom.minidom |
|
13 |
import xml.dom.ext |
|
14 |
import xml.xpath |
|
15 |
import fnmatch |
|
16 |
import uuid |
|
17 |
import shutil |
|
18 |
import lucene |
|
19 |
import uuid |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
20 |
from blinkster.ldt import STORE |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
21 |
from blinkster.ldt import ANALYZER |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
22 |
from blinkster.ldt.contentindexer import ContentIndexer |
| 0 | 23 |
|
24 |
def Property(func): |
|
25 |
return property(**func()) |
|
26 |
||
27 |
class IriInfo(object): |
|
28 |
||
29 |
||
30 |
def __init__(self, id, basepath, order, titledesc, decoupage_blacklist = settings.DECOUPAGE_BLACKLIST): |
|
31 |
self.id = id |
|
32 |
self.basepath = basepath |
|
33 |
self.order = order |
|
34 |
self.src = "" |
|
35 |
self.annotations = None |
|
36 |
self.videopath = settings.STREAM_URL |
|
37 |
self.videourl = "" |
|
38 |
self.title = None |
|
39 |
self.desc = None |
|
40 |
self.decoupageId = None |
|
41 |
self.ensembleId = None |
|
42 |
self.created = False |
|
43 |
self.content = None |
|
44 |
self.decoupage_blacklist = decoupage_blacklist |
|
45 |
if self.decoupage_blacklist is None: |
|
46 |
self.decoupage_blacklist = () |
|
47 |
||
48 |
||
49 |
||
50 |
def processIri(self): |
|
51 |
||
52 |
path = os.path.join(self.basepath, self.src) |
|
53 |
doc = xml.dom.minidom.parse(path) |
|
54 |
||
55 |
con = xml.xpath.Context.Context(doc, 1, 1, None) |
|
56 |
||
57 |
res = xml.xpath.Evaluate("/iri/head/meta[@name='title']/@content", context=con) |
|
58 |
self.title = res[0].value |
|
59 |
||
60 |
res = xml.xpath.Evaluate("/iri/body/ensembles",context=con) |
|
61 |
ensemblesnode = res[0] |
|
62 |
||
63 |
if self.annotations is not None: |
|
64 |
newensemble = doc.createElement('ensemble') |
|
65 |
self.ensembleId = self.id+"_"+str(uuid.uuid1()) |
|
66 |
newensemble.setAttribute('id',self.ensembleId) |
|
67 |
||
68 |
newensemble.setAttribute('title', self.annotations.getAttribute('title')) |
|
69 |
newensemble.setAttribute('author', self.annotations.getAttribute('author')) |
|
70 |
newensemble.setAttribute('date', self.annotations.getAttribute('date')) |
|
71 |
newensemble.setAttribute('abstract', self.annotations.getAttribute('abstract')) |
|
72 |
||
73 |
decoupageNode = None |
|
74 |
for node in self.annotations.childNodes: |
|
75 |
if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "decoupage": |
|
76 |
decoupageNode = node |
|
77 |
newDecoupageNode = decoupageNode.cloneNode(True) |
|
78 |
newensemble.appendChild(newDecoupageNode) |
|
79 |
||
80 |
if not decoupageNode is None: |
|
81 |
self.decoupageId = decoupageNode.getAttribute(u"id") |
|
82 |
||
83 |
ensemblesnode.appendChild(newensemble) |
|
84 |
else: |
|
85 |
# find an ensemble child node |
|
86 |
ensemble = None |
|
87 |
for node in ensemblesnode.childNodes: |
|
88 |
if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "ensemble": |
|
89 |
ensemble = node |
|
90 |
break |
|
91 |
if ensemble is not None: |
|
92 |
self.ensembleId = ensemble.getAttribute(u"id") |
|
93 |
decoupageNode = None |
|
94 |
for node in ensemble.childNodes: |
|
95 |
if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "decoupage": |
|
96 |
decoupageNode = node |
|
97 |
break |
|
98 |
if not decoupageNode is None: |
|
99 |
self.decoupageId = decoupageNode.getAttribute(u"id") |
|
100 |
||
101 |
||
102 |
res = xml.xpath.Evaluate("/iri/body/medias/media[@id='video']/video", context=con) |
|
103 |
src_video = res[0].getAttribute('src') |
|
104 |
self.videourl = os.path.basename(src_video) |
|
105 |
srcnode = res[0].setAttribute('src', self.videourl) |
|
106 |
||
107 |
f = open(path, "w") |
|
108 |
try: |
|
109 |
xml.dom.ext.Print(doc, stream=f) |
|
110 |
finally: |
|
111 |
f.close() |
|
112 |
||
113 |
||
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
114 |
destPath = os.path.join(os.path.join(os.path.join(blinkster.settings.MEDIA_ROOT, "media"), "ldt"), self.id); |
| 0 | 115 |
if not os.path.exists(destPath): |
116 |
os.makedirs(destPath) |
|
117 |
shutil.move(os.path.join(self.basepath, self.src), os.path.join(destPath, os.path.basename(self.src))) |
|
118 |
||
119 |
||
120 |
||
121 |
def saveContent(self): |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
122 |
blinkster.utils.log.debug("ID of object to create : "+str(self.id)) |
| 0 | 123 |
content, self.created = Content.objects.get_or_create(iri_id=self.id, defaults = { 'iri' : self.id + u"/" + os.path.basename(self.src), 'title':self.title, 'description':self.desc, })#'decoupage_id': self.decoupageId, 'ensemble_id': self.ensembleId}); |
124 |
if not self.created: |
|
125 |
content.iri = self.id + u"/" + os.path.basename(self.src) |
|
126 |
content.title = self.title |
|
127 |
content.description = self.desc |
|
128 |
#content.decoupage_id = unicode(self.decoupageId) |
|
129 |
#content.ensemble_id = unicode(self.ensembleId) |
|
130 |
content.save() |
|
131 |
self.content = content |
|
132 |
||
133 |
def indexDecoupage(self, writer): |
|
134 |
indexer = ContentIndexer([self.content],writer, settings.DECOUPAGE_BLACKLIST) |
|
135 |
indexer.index_all() |
|
136 |
||
137 |
def process(self, writer): |
|
138 |
self.processIri() |
|
139 |
self.saveContent() |
|
140 |
self.indexDecoupage(writer) |
|
141 |
||
142 |
class BaseFileImport(object): |
|
143 |
||
144 |
def __init__(self, filepath, author): |
|
145 |
self.__filepath = filepath |
|
146 |
self.__tempdir = "" |
|
147 |
self.__author = author |
|
148 |
self.__writer = lucene.IndexModifier(STORE, ANALYZER, True) |
|
149 |
self.__writer.setMaxFieldLength(1048576) |
|
150 |
||
151 |
||
152 |
||
153 |
def filepath(): #@NoSelf |
|
154 |
doc = """Docstring""" #@UnusedVariable |
|
155 |
||
156 |
def fget(self): |
|
157 |
return self.__filepath |
|
158 |
||
159 |
def fset(self, value): |
|
160 |
self.__filepath = value |
|
161 |
||
162 |
def fdel(self): |
|
163 |
del self.__filepath |
|
164 |
||
165 |
return locals() |
|
166 |
||
167 |
filepath = property(**filepath()) |
|
168 |
||
169 |
def author(): #@NoSelf |
|
170 |
doc = """Docstring""" #@UnusedVariable |
|
171 |
||
172 |
def fget(self): |
|
173 |
return self.__author |
|
174 |
||
175 |
def fset(self, value): |
|
176 |
self.__author = value |
|
177 |
||
178 |
def fdel(self): |
|
179 |
del self.__author |
|
180 |
||
181 |
return locals() |
|
182 |
||
183 |
author = property(**author()) |
|
184 |
||
185 |
def writer(): |
|
186 |
||
187 |
def fget(self): |
|
188 |
return self.__writer |
|
189 |
||
190 |
return locals() |
|
191 |
||
192 |
writer = property(**writer()) |
|
193 |
||
194 |
||
195 |
def close(self): |
|
196 |
self.__writer.optimize() |
|
197 |
self.__writer.close() |
|
198 |
||
199 |
||
200 |
class FileImport(BaseFileImport): |
|
201 |
||
202 |
def __init__(self, filepath, author): |
|
203 |
BaseFileImport.__init__(self,filepath,author) |
|
204 |
self.__checkExistingMedia = False |
|
205 |
||
206 |
def checkExistingMedia(): #@NoSelf |
|
207 |
doc = """Docstring""" #@UnusedVariable |
|
208 |
||
209 |
def fget(self): |
|
210 |
return self.__checkExistingMedia |
|
211 |
||
212 |
def fset(self, value): |
|
213 |
self.__checkExistingMedia = value |
|
214 |
||
215 |
def fdel(self): |
|
216 |
del self.__checkExistingMedia |
|
217 |
||
218 |
return locals() |
|
219 |
||
220 |
checkExistingMedia = property(**checkExistingMedia()) |
|
221 |
||
222 |
||
223 |
def processLdt(self, ldtpath): |
|
224 |
||
225 |
# list iri |
|
226 |
# see if there is some comments |
|
227 |
# inject comment in iri |
|
228 |
# copy iri in folder |
|
229 |
# create or update content |
|
230 |
contents = {} |
|
231 |
doc = xml.dom.minidom.parse(ldtpath) |
|
232 |
||
233 |
con = xml.xpath.Context.Context(doc, 1, 1, None) |
|
234 |
result = xml.xpath.Evaluate("/iri/project", context=con) |
|
235 |
for pnode in result: |
|
236 |
author = pnode.getAttribute("user") |
|
237 |
if author: |
|
238 |
self.author = unicode(author) |
|
239 |
break |
|
240 |
||
241 |
result = xml.xpath.Evaluate("/iri/medias/media", context=con) |
|
242 |
||
243 |
for i, medianode in enumerate(result): |
|
244 |
id = medianode.attributes['id'].value |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
245 |
blinkster.utils.log.debug("FIRST ID : " + str(id)) |
| 0 | 246 |
if self.checkExistingMedia: |
247 |
try: |
|
248 |
Content.objects.get(iri_id=id) |
|
249 |
do_pass = True |
|
250 |
except Content.DoesNotExist: |
|
251 |
do_pass = False |
|
252 |
else: |
|
253 |
do_pass = False |
|
254 |
if not do_pass: |
|
255 |
if not (contents.has_key(id)): |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
256 |
blinkster.utils.log.debug("CREATE IRI INFO ID : " + str(id)) |
| 0 | 257 |
contents[id] = IriInfo(id, os.path.dirname(ldtpath), i, "") |
258 |
contents[id].src = medianode.attributes['src'].value |
|
259 |
||
260 |
result = xml.xpath.Evaluate("/iri/annotations/content", context=con) |
|
261 |
||
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
262 |
blinkster.utils.log.debug("content xpath result : " + str(len(result))) |
| 0 | 263 |
for contentnode in result: |
264 |
id = contentnode.attributes['id'].value |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
265 |
blinkster.utils.log.debug("ID : " + str(id)) |
| 0 | 266 |
if contents.has_key(id): |
267 |
if self.author: |
|
268 |
contentnode.setAttribute("author", unicode(self.author)) |
|
269 |
contents[id].annotations = contentnode |
|
270 |
||
271 |
#go throught values |
|
272 |
for iriinfo in contents.values(): |
|
273 |
||
274 |
iriinfo.process(self.writer) |
|
275 |
||
276 |
# if yes update |
|
277 |
# if no create |
|
278 |
# move iri file to the proper place |
|
279 |
#return list of iriInfo |
|
280 |
||
281 |
||
282 |
def processFile(self): |
|
283 |
# create temp directory |
|
284 |
self.__tempdir = tempfile.mkdtemp() |
|
285 |
openfiles = [] |
|
286 |
flvfiles = [] |
|
287 |
processedids = [] |
|
288 |
||
289 |
try: |
|
290 |
zipfile = zipfileext.ZipFileExt(self.filepath) |
|
291 |
zipfile.unzip_into_dir(self.__tempdir) |
|
292 |
#load ldt |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
293 |
blinkster.utils.log.debug("Processing files : ") |
|
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
294 |
blinkster.utils.log.debug(os.listdir(self.__tempdir)) |
| 0 | 295 |
foldersToProcess = [self.__tempdir] |
296 |
while len(foldersToProcess): |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
297 |
blinkster.utils.log.debug("folder stack length : "+ str(len(foldersToProcess))) |
| 0 | 298 |
currentFolder = foldersToProcess.pop() |
299 |
for entry in os.listdir(currentFolder): |
|
300 |
if entry in settings.ZIP_BLACKLIST: |
|
301 |
continue |
|
302 |
entryPath = os.path.join(currentFolder, entry) |
|
303 |
if(os.path.isdir(entryPath)): |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
304 |
blinkster.utils.log.debug("Push folder : " + entryPath) |
| 0 | 305 |
foldersToProcess.append(entryPath) |
306 |
elif fnmatch.fnmatch(entry, "*.ldt"): |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
307 |
blinkster.utils.log.debug("Process file : " + entryPath) |
| 0 | 308 |
ldtid = self.processLdt(entryPath) |
309 |
processedids.append(ldtid) |
|
310 |
elif fnmatch.fnmatch(entry, "*.flv"): |
|
311 |
flvfiles.append(entryPath) |
|
312 |
||
313 |
if settings.CONTENT_ROOT and os.path.exists(settings.CONTENT_ROOT): |
|
314 |
for entry in flvfiles: |
|
315 |
shutil.copy(entry, settings.CONTENT_ROOT) |
|
316 |
||
317 |
finally: |
|
318 |
for f in openfiles: |
|
319 |
f.close() |
|
320 |
shutil.rmtree(self.__tempdir) |
|
321 |
# delete directory |
|
322 |
return processedids |
|
323 |
||
324 |
||
325 |
class ProjectFileImport(BaseFileImport): |
|
326 |
||
327 |
||
328 |
def processLdt(self): |
|
329 |
||
330 |
self.contents = [] |
|
331 |
||
332 |
doc = xml.dom.minidom.parse(self.filepath) |
|
333 |
||
334 |
con = xml.xpath.Context.Context(doc, 1, 1, None) |
|
335 |
result = xml.xpath.Evaluate("/iri/project", context=con) |
|
336 |
for pnode in result: |
|
337 |
author = pnode.setAttribute("user", unicode(self.author)) |
|
338 |
break |
|
339 |
||
340 |
result = xml.xpath.Evaluate("/iri/medias/media", context=con) |
|
341 |
||
342 |
for i, medianode in enumerate(result): |
|
343 |
id = medianode.attributes['id'].value |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
344 |
blinkster.utils.log.debug("FIRST ID : " + str(id)) |
| 0 | 345 |
content = Content.objects.get(iri_id = id) |
346 |
self.contents.append(content) |
|
347 |
||
348 |
medianode.setAttribute('src', content.iri_url_template()) |
|
349 |
medianode.setAttribute('video', "${stream_url}") |
|
350 |
||
351 |
return doc.toprettyxml(encoding="utf-8") |
|
352 |
||
353 |
def processFile(self): |
|
354 |
||
355 |
ldt_id = unicode(uuid.uuid1()) |
|
356 |
ldt = self.processLdt() |
|
357 |
authors = [self.author] |
|
358 |
contents = self.contents |
|
359 |
||
360 |
ldtproject = LdtProject(ldt_id = ldt_id, ldt=ldt) |
|
361 |
ldtproject.save() |
|
362 |
idldt = ldtproject.id |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
363 |
blinkster.utils.log.debug("ProjectFileImport.processFile ID : " + str(idldt)) |
| 0 | 364 |
for author in authors: |
365 |
ldtproject.authors.add(author) |
|
366 |
for content in contents: |
|
367 |
ldtproject.contents.add(content) |
|
368 |
ldtproject.save() |
|
369 |
||
370 |
return [idldt] |
|
371 |
||
372 |
||
373 |
||
374 |
class ZipProjectFileImport(FileImport): |
|
375 |
||
376 |
def __init__(self, filepath, author): |
|
377 |
FileImport.__init__(self,filepath,author) |
|
378 |
self.checkExistingMedia = True |
|
379 |
||
380 |
def processLdt(self, ldtPath): |
|
381 |
fi = ProjectFileImport(ldtPath, self.author) |
|
382 |
FileImport.processLdt(self, ldtPath) |
|
383 |
idproject = fi.processFile() |
|
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
384 |
blinkster.utils.log.debug("ZipProjectFileImport.processLdt ID : " + str(idproject)) |
| 0 | 385 |
fi.close() |
386 |
||
387 |
return idproject |
|
388 |
||
|
3
526ebd3988b0
replace pocketfilms occurence by blinkster
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
389 |