| author | ymh <ymh.work@gmail.com> |
| Thu, 19 May 2011 10:38:36 +0200 | |
| changeset 113 | cf3bdb2a4216 |
| parent 63 | 93325a5d61f0 |
| child 560 | 1cb2a4a573e1 |
| permissions | -rw-r--r-- |
| 0 | 1 |
from ldt import settings |
2 |
import lucene |
|
3 |
import lxml.etree |
|
4 |
||
5 |
def Property(func): |
|
6 |
return property(**func()) |
|
7 |
||
8 |
class ProjectIndexer(object): |
|
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
9 |
def __init__(self, projectList, writer, decoupage_blackList=settings.DECOUPAGE_BLACKLIST): |
| 0 | 10 |
self.__projectList = projectList |
11 |
self.__decoupage_blacklist = decoupage_blackList |
|
12 |
self.__writer = writer |
|
13 |
||
14 |
@Property |
|
15 |
def decoupage_blacklist(): #@NoSelf |
|
16 |
doc = """get blacklist""" #@UnusedVariable |
|
17 |
||
18 |
def fget(self): |
|
19 |
if self.__decoupage_blacklist is None: |
|
20 |
self.__decoupage_blacklist = () |
|
21 |
return self.__decoupage_blacklist |
|
22 |
||
23 |
def fset(self, value): |
|
24 |
self.__decoupage_blacklist = value |
|
25 |
||
26 |
def fdel(self): |
|
27 |
del self.__decoupage_blacklist |
|
28 |
||
29 |
return locals() |
|
30 |
||
31 |
def index_all(self): |
|
32 |
for project in self.__projectList: |
|
33 |
self.index_project(project) |
|
34 |
||
35 |
def index_project(self, project): |
|
36 |
# ldt.utils.log.debug("Indexing project : "+str(project.ldt_id)) |
|
37 |
||
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
38 |
ldt = project.ldt |
|
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
39 |
doc = lxml.etree.fromstring(ldt.encode("utf-8")) |
| 0 | 40 |
|
41 |
self.__writer.deleteDocuments(lucene.Term("ldt_id", project.ldt_id)) |
|
42 |
||
43 |
res = doc.xpath("/iri/annotations/content") |
|
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
44 |
project.ldt.encode("utf-8 ") |
| 0 | 45 |
|
46 |
for content in res: |
|
47 |
contentId = content.get("id") |
|
48 |
||
|
13
97ab7b3191cf
add api to update project, uses psiton
ymh <ymh.work@gmail.com>
parents:
0
diff
changeset
|
49 |
res = content.xpath("ensemble") |
| 0 | 50 |
for ensemble in res: |
51 |
ensembleId = ensemble.get("id") |
|
52 |
||
53 |
for decoupageNode in ensemble.getchildren(): |
|
54 |
# ldt.utils.log.debug("Indexing project decoupage : "+ repr(decoupageNode.nodeType) + " in " + repr(self.decoupage_blacklist)) |
|
55 |
if decoupageNode.tag != "decoupage" or decoupageNode.get("id") in self.decoupage_blacklist: |
|
56 |
continue |
|
57 |
||
58 |
decoupId = decoupageNode.get("id") |
|
59 |
res = decoupageNode.xpath("elements/element") |
|
60 |
||
61 |
for elementNode in res: |
|
62 |
doc = lucene.Document() |
|
63 |
elementId = elementNode.get("id") |
|
64 |
tags = elementNode.get("tags") |
|
65 |
||
66 |
if tags is not None: |
|
67 |
tags.replace(",", ";") |
|
68 |
||
69 |
if tags is None or len(tags) == 0: |
|
70 |
tags = "" |
|
71 |
restagnode = elementNode.xpath("tag/text()") |
|
72 |
for tagnode in restagnode: |
|
73 |
tags = tags + " ; " + tagnode.text() |
|
74 |
||
75 |
if tags is None or len(tags) == 0: |
|
76 |
tags = "" |
|
77 |
restagnode = elementNode.xpath("tags/tag/text()") |
|
78 |
for tagnode in restagnode: |
|
79 |
tags = tags + " ; " + tagnode.text() |
|
80 |
||
81 |
title = "" |
|
82 |
for txtRes in elementNode.xpath("title/text()"): |
|
83 |
title = title + txtRes.text() |
|
84 |
||
85 |
abstract = "" |
|
86 |
for txtRes in elementNode.xpath("abstract/text()"): |
|
87 |
abstract = abstract + txtRes.text() |
|
88 |
||
89 |
doc.add(lucene.Field("ldt_id", project.ldt_id, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
90 |
doc.add(lucene.Field("iri_id", contentId, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
91 |
doc.add(lucene.Field("ensemble_id", ensembleId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
92 |
doc.add(lucene.Field("decoupage_id", decoupId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
93 |
doc.add(lucene.Field("element_id", elementId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
94 |
doc.add(lucene.Field("tags", tags, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
95 |
doc.add(lucene.Field("title", title, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
96 |
doc.add(lucene.Field("abstract", abstract, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
97 |
doc.add(lucene.Field("all", " ".join([tags, title, abstract]), lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
98 |
||
99 |
self.__writer.addDocument(doc) |
|
100 |
||
101 |
self.__writer.commit() |