| author | ymh <ymh.work@gmail.com> |
| Sun, 01 May 2011 03:30:40 +0200 | |
| changeset 77 | 7923feb2e362 |
| parent 63 | 93325a5d61f0 |
| child 90 | 4ddb88f103ad |
| permissions | -rw-r--r-- |
| 24 | 1 |
from django.conf import settings |
| 77 | 2 |
from django.db.models.signals import post_save |
3 |
from django.dispatch import receiver |
|
4 |
from ldt.ldt_utils.models import Segment, Content, Project |
|
| 63 | 5 |
from ldt.ldt_utils.utils import reduce_text_node |
| 77 | 6 |
import ldt.indexation |
| 24 | 7 |
import lucene |
8 |
import lxml.etree |
|
| 54 | 9 |
import urllib #@UnresolvedImport |
| 0 | 10 |
# import ldt.utils.log |
11 |
||
| 77 | 12 |
|
| 0 | 13 |
def Property(func): |
14 |
return property(**func()) |
|
| 54 | 15 |
|
| 0 | 16 |
|
17 |
class ContentIndexer(object): |
|
18 |
||
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
19 |
def __init__(self, contentList, writer, decoupage_blackList=settings.DECOUPAGE_BLACKLIST): |
| 0 | 20 |
self.__contentList = contentList |
21 |
self.__decoupage_blacklist = decoupage_blackList |
|
22 |
self.__writer = writer |
|
23 |
||
24 |
@Property |
|
25 |
def decoupage_blacklist(): #@NoSelf |
|
26 |
doc = """get blacklist""" #@UnusedVariable |
|
27 |
||
28 |
def fget(self): |
|
29 |
if self.__decoupage_blacklist is None: |
|
30 |
self.__decoupage_blacklist = () |
|
31 |
return self.__decoupage_blacklist |
|
32 |
||
33 |
def fset(self, value): |
|
34 |
self.__decoupage_blacklist = value |
|
35 |
||
36 |
def fdel(self): |
|
37 |
del self.__decoupage_blacklist |
|
38 |
||
39 |
return locals() |
|
40 |
||
41 |
def index_all(self): |
|
42 |
for content in self.__contentList: |
|
43 |
self.index_content(content) |
|
44 |
||
45 |
def index_content(self, content): |
|
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
46 |
url = content.iri_url() |
| 0 | 47 |
filepath = urllib.urlopen(url) |
| 54 | 48 |
doc = lxml.etree.parse(filepath) #@UndefinedVariable |
| 0 | 49 |
|
50 |
self.__writer.deleteDocuments(lucene.Term("iri_id", content.iri_id)) |
|
| 77 | 51 |
Segment.objects.filter(iri_id=content.iri_id).delete() #@UndefinedVariable |
| 0 | 52 |
|
53 |
res = doc.xpath("/iri/body/ensembles/ensemble") |
|
54 |
||
55 |
for ensemble in res: |
|
| 54 | 56 |
ensembleId = ensemble.get(u"id", None) |
| 0 | 57 |
|
58 |
for decoupageNode in ensemble.getchildren(): |
|
| 54 | 59 |
if decoupageNode.tag != "decoupage" or decoupageNode.get(u"id", None) in self.decoupage_blacklist: |
| 0 | 60 |
|
61 |
continue |
|
62 |
||
| 54 | 63 |
decoupId = decoupageNode.get(u"id", None) |
| 0 | 64 |
res = decoupageNode.xpath("elements/element") |
65 |
for elementNode in res: |
|
66 |
doc = lucene.Document() |
|
| 54 | 67 |
elementId = elementNode.get(u"id", None) |
68 |
tags = elementNode.get(u"tags", None) |
|
| 0 | 69 |
|
70 |
if tags is not None: |
|
71 |
tags.replace(",", ";") |
|
72 |
||
73 |
if tags is None or len(tags) == 0: |
|
| 54 | 74 |
tags = u"" |
75 |
restagnode = elementNode.xpath("tag/text()", smart_strings=False) |
|
| 0 | 76 |
for tagnode in restagnode: |
| 54 | 77 |
tags = tags + u" ; " + tagnode |
| 0 | 78 |
|
79 |
if tags is None or len(tags) == 0: |
|
| 54 | 80 |
tags = u"" |
81 |
restagnode = elementNode.xpath("tags/tag/text()", smart_strings=False) |
|
| 0 | 82 |
|
83 |
for tagnode in restagnode: |
|
| 54 | 84 |
tags = tags + u" ; " + tagnode |
| 0 | 85 |
|
| 54 | 86 |
title = reduce_text_node(elementNode, "title/text()") |
| 63 | 87 |
abstract = reduce_text_node(elementNode, "abstract/text()") |
| 54 | 88 |
|
89 |
author = elementNode.get("author", "") |
|
90 |
start_ts = int(elementNode.get("begin", "-1")) |
|
91 |
duration = int(elementNode.get("dur", "-1")) |
|
92 |
date_str = elementNode.get("date", "") |
|
93 |
||
| 0 | 94 |
|
95 |
doc.add(lucene.Field("iri_id", content.iri_id, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
96 |
doc.add(lucene.Field("ensemble_id", ensembleId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
97 |
doc.add(lucene.Field("decoupage_id", decoupId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
98 |
doc.add(lucene.Field("element_id", elementId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
99 |
doc.add(lucene.Field("tags", tags, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
100 |
doc.add(lucene.Field("title", title, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
101 |
doc.add(lucene.Field("abstract", abstract, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
102 |
doc.add(lucene.Field("all", " ".join([tags, title, abstract]), lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
103 |
||
104 |
seg = Segment(content=content, |
|
105 |
iri_id=content.iri_id, |
|
106 |
ensemble_id=ensembleId, |
|
107 |
cutting_id=decoupId, |
|
108 |
element_id=elementId, |
|
109 |
tags=tags, |
|
110 |
title=title, |
|
111 |
abstract=abstract, |
|
112 |
duration=duration, |
|
113 |
author=author, |
|
114 |
start_ts=start_ts, |
|
115 |
date=date_str) |
|
116 |
seg.save() |
|
117 |
||
118 |
||
119 |
self.__writer.addDocument(doc) |
|
120 |
||
121 |
self.__writer.commit() |
|
122 |
||
123 |
||
124 |
class ProjectIndexer(object): |
|
125 |
||
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
126 |
def __init__(self, projectList, writer, decoupage_blackList=settings.DECOUPAGE_BLACKLIST): |
| 0 | 127 |
self.__projectList = projectList |
128 |
self.__decoupage_blacklist = decoupage_blackList |
|
129 |
self.__writer = writer |
|
130 |
||
131 |
@Property |
|
132 |
def decoupage_blacklist(): #@NoSelf |
|
133 |
doc = """get blacklist""" #@UnusedVariable |
|
134 |
||
135 |
def fget(self): |
|
136 |
if self.__decoupage_blacklist is None: |
|
137 |
self.__decoupage_blacklist = () |
|
138 |
return self.__decoupage_blacklist |
|
139 |
||
140 |
def fset(self, value): |
|
141 |
self.__decoupage_blacklist = value |
|
142 |
||
143 |
def fdel(self): |
|
144 |
del self.__decoupage_blacklist |
|
145 |
||
146 |
return locals() |
|
147 |
||
148 |
def index_all(self): |
|
149 |
for project in self.__projectList: |
|
150 |
self.index_project(project) |
|
151 |
||
152 |
def index_project(self, project): |
|
153 |
||
154 |
# pocketfilms.utils.log.debug("Indexing project : "+str(project.iri_id)) |
|
| 54 | 155 |
doc = lxml.etree.fromstring(project.ldt) #@UndefinedVariable |
| 0 | 156 |
|
| 77 | 157 |
self.__writer.deleteDocuments(lucene.Term("project_id", project.ldt_id)) |
158 |
Segment.objects.filter(project_obj__ldt_id=project.ldt_id).delete() |
|
| 0 | 159 |
|
160 |
res = doc.xpath("/iri/annotations/content") |
|
161 |
||
162 |
for content in res: |
|
| 54 | 163 |
contentId = content.get(u"id", None) |
| 77 | 164 |
content_obj = None |
| 0 | 165 |
|
| 77 | 166 |
clist = Content.objects.filter(iri_id = contentId) |
167 |
if len(clist) > 0: |
|
168 |
content_obj = clist[0] |
|
169 |
||
170 |
for ensembleNode in content.getchildren(): |
|
171 |
ensembleId = ensembleNode.get(u"id",None) |
|
| 0 | 172 |
|
| 77 | 173 |
for decoupageNode in ensembleNode.getchildren(): |
174 |
# pocketfilms.utils.log.debug("Indexing content decoupage : "+ repr(decoupageNode.nodeType) + " in " + repr(self.decoupage_blacklist)) |
|
175 |
if decoupageNode.tag != "decoupage" or decoupageNode.get(u"id", None) in self.decoupage_blacklist: |
|
176 |
continue |
|
| 0 | 177 |
|
| 77 | 178 |
decoupId = decoupageNode.get(u"id", None) |
179 |
res = decoupageNode.xpath("elements/element") |
|
180 |
for elementNode in res: |
|
181 |
doc = lucene.Document() |
|
182 |
elementId = elementNode.get(u"id", None) |
|
183 |
tags = elementNode.get(u"tags", None) |
|
184 |
||
185 |
if tags is not None: |
|
186 |
tags.replace(",", ";") |
|
187 |
||
188 |
if tags is None or len(tags) == 0: |
|
189 |
tags = u"" |
|
190 |
restagnode = elementNode.xpath("tag/text()", smart_strings=False) |
|
191 |
for tagnode in restagnode: |
|
192 |
tags = tags + u" ; " + tagnode |
|
| 0 | 193 |
|
| 77 | 194 |
if tags is None or len(tags) == 0: |
195 |
tags = u"" |
|
196 |
restagnode = elementNode.xpath("tags/tag/text()", smart_strings=False) |
|
197 |
||
198 |
for tagnode in restagnode: |
|
199 |
tags = tags + u" ; " + tagnode |
|
200 |
||
201 |
title = reduce_text_node(elementNode, "title/text()") |
|
202 |
abstract = reduce_text_node(elementNode, "abstract/text()") |
|
203 |
||
204 |
author = elementNode.get("author", "") |
|
205 |
start_ts = int(elementNode.get("begin", "-1")) |
|
206 |
duration = int(elementNode.get("dur", "-1")) |
|
207 |
date_str = elementNode.get("date", "") |
|
208 |
||
209 |
||
210 |
doc.add(lucene.Field("type_doc", "annotation", lucene.Field.Store.NO, lucene.Field.Index.NOT_ANALYZED)) |
|
211 |
doc.add(lucene.Field("project_id", project.ldt_id, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
212 |
doc.add(lucene.Field("iri_id", contentId, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
213 |
doc.add(lucene.Field("ensemble_id", ensembleId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
214 |
doc.add(lucene.Field("decoupage_id", decoupId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
215 |
doc.add(lucene.Field("element_id", elementId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
216 |
doc.add(lucene.Field("tags", tags, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
217 |
doc.add(lucene.Field("title", title, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
218 |
doc.add(lucene.Field("abstract", abstract, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
219 |
doc.add(lucene.Field("all", " ".join([tags, title, abstract]), lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
220 |
||
221 |
seg = Segment(content=content_obj, |
|
222 |
iri_id=contentId, |
|
223 |
ensemble_id=ensembleId, |
|
224 |
cutting_id=decoupId, |
|
225 |
element_id=elementId, |
|
226 |
tags=tags, |
|
227 |
title=title, |
|
228 |
abstract=abstract, |
|
229 |
duration=duration, |
|
230 |
author=author, |
|
231 |
start_ts=start_ts, |
|
232 |
date=date_str, |
|
233 |
project_obj = project) |
|
234 |
seg.save() |
|
235 |
||
236 |
self.__writer.addDocument(doc) |
|
| 0 | 237 |
|
238 |
self.__writer.commit() |
|
| 77 | 239 |
|
240 |
@receiver(post_save, sender=Project) |
|
241 |
def index_project(sender, **kwargs): |
|
242 |
instance = kwargs['instance'] |
|
243 |
writer = ldt.indexation.get_writer() |
|
244 |
if instance.state != 2: |
|
245 |
writer.deleteDocuments(lucene.Term("project_id", instance.ldt_id)) |
|
246 |
Segment.objects.filter(project_obj__ldt_id=instance.ldt_id).delete() |
|
247 |
else: |
|
248 |
projectIndexer = ProjectIndexer([instance], writer) |
|
249 |
projectIndexer.index_all() |
|
250 |
||
251 |