|
0
|
1 |
import tempfile |
|
|
2 |
import os |
|
|
3 |
import os.path |
|
|
4 |
import shutil |
|
|
5 |
import ldt.utils.xml |
|
|
6 |
from ldt import settings |
|
|
7 |
import lucene |
|
|
8 |
from ldt.ldt_utils import STORE |
|
|
9 |
from ldt.ldt_utils import ANALYZER |
|
94
|
10 |
import lxml.etree |
|
91
|
11 |
|
|
0
|
12 |
def Property(func): |
|
|
13 |
return property(**func()) |
|
|
14 |
|
|
|
15 |
class ProjectIndexer(object): |
|
|
16 |
def __init__(self, projectList, writer, decoupage_blackList = settings.DECOUPAGE_BLACKLIST): |
|
|
17 |
self.__projectList = projectList |
|
|
18 |
self.__decoupage_blacklist = decoupage_blackList |
|
|
19 |
self.__writer = writer |
|
|
20 |
|
|
|
21 |
@Property |
|
|
22 |
def decoupage_blacklist(): #@NoSelf |
|
|
23 |
doc = """get blacklist""" #@UnusedVariable |
|
|
24 |
|
|
|
25 |
def fget(self): |
|
|
26 |
if self.__decoupage_blacklist is None: |
|
|
27 |
self.__decoupage_blacklist = () |
|
|
28 |
return self.__decoupage_blacklist |
|
|
29 |
|
|
|
30 |
def fset(self, value): |
|
|
31 |
self.__decoupage_blacklist = value |
|
|
32 |
|
|
|
33 |
def fdel(self): |
|
|
34 |
del self.__decoupage_blacklist |
|
|
35 |
|
|
|
36 |
return locals() |
|
|
37 |
|
|
|
38 |
def index_all(self): |
|
|
39 |
for project in self.__projectList: |
|
|
40 |
self.index_project(project) |
|
|
41 |
|
|
|
42 |
def index_project(self, project): |
|
91
|
43 |
# ldt.utils.log.debug("Indexing project : "+str(project.ldt_id)) |
|
|
44 |
|
|
94
|
45 |
ldt=project.ldt |
|
|
46 |
doc = lxml.etree.fromstring(ldt.encode( "utf-8" )) |
|
91
|
47 |
|
|
0
|
48 |
self.__writer.deleteDocuments(lucene.Term("ldt_id", project.ldt_id)) |
|
|
49 |
|
|
94
|
50 |
res = doc.xpath("/iri/annotations/content") |
|
91
|
51 |
project.ldt.encode( "utf-8 " ) |
|
0
|
52 |
|
|
|
53 |
for content in res: |
|
94
|
54 |
contentId = content.get("id") |
|
91
|
55 |
|
|
94
|
56 |
res =content.xpath("ensemble") |
|
0
|
57 |
for ensemble in res: |
|
94
|
58 |
ensembleId = ensemble.get("id") |
|
91
|
59 |
|
|
94
|
60 |
for decoupageNode in ensemble.getchildren(): |
|
91
|
61 |
# ldt.utils.log.debug("Indexing project decoupage : "+ repr(decoupageNode.nodeType) + " in " + repr(self.decoupage_blacklist)) |
|
94
|
62 |
if decoupageNode.tag != "decoupage" or decoupageNode.get("id") in self.decoupage_blacklist: |
|
0
|
63 |
continue |
|
|
64 |
|
|
94
|
65 |
decoupId = decoupageNode.get("id") |
|
|
66 |
res = decoupageNode.xpath("elements/element") |
|
91
|
67 |
|
|
0
|
68 |
for elementNode in res: |
|
|
69 |
doc = lucene.Document() |
|
94
|
70 |
elementId = elementNode.get("id") |
|
|
71 |
tags = elementNode.get("tags") |
|
0
|
72 |
|
|
|
73 |
if tags is not None: |
|
|
74 |
tags.replace(",", ";") |
|
|
75 |
|
|
|
76 |
if tags is None or len(tags) == 0: |
|
|
77 |
tags = "" |
|
94
|
78 |
restagnode = elementNode.xpath("tag/text()") |
|
0
|
79 |
for tagnode in restagnode: |
|
94
|
80 |
tags = tags + " ; " + tagnode.text() |
|
0
|
81 |
|
|
|
82 |
if tags is None or len(tags) == 0: |
|
|
83 |
tags = "" |
|
94
|
84 |
restagnode = elementNode.xpath("tags/tag/text()") |
|
0
|
85 |
for tagnode in restagnode: |
|
94
|
86 |
tags = tags + " ; " + tagnode.text() |
|
0
|
87 |
|
|
|
88 |
title = "" |
|
94
|
89 |
for txtRes in elementNode.xpath("title/text()"): |
|
|
90 |
title = title + txtRes.text() |
|
0
|
91 |
|
|
|
92 |
abstract = "" |
|
94
|
93 |
for txtRes in elementNode.xpath("abstract/text()"): |
|
|
94 |
abstract = abstract + txtRes.text() |
|
0
|
95 |
|
|
94
|
96 |
doc.add(lucene.Field("ldt_id", project.ldt_id, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
|
97 |
doc.add(lucene.Field("iri_id", contentId, lucene.Field.Store.YES, lucene.Field.Index.NOT_ANALYZED)) |
|
0
|
98 |
doc.add(lucene.Field("ensemble_id", ensembleId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
|
99 |
doc.add(lucene.Field("decoupage_id", decoupId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
|
100 |
doc.add(lucene.Field("element_id", elementId, lucene.Field.Store.YES, lucene.Field.Index.NO)) |
|
94
|
101 |
doc.add(lucene.Field("tags", tags, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
|
102 |
doc.add(lucene.Field("title", title, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
|
103 |
doc.add(lucene.Field("abstract", abstract, lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
|
104 |
doc.add(lucene.Field("all", " ".join([tags, title, abstract]), lucene.Field.Store.NO, lucene.Field.Index.ANALYZED)) |
|
0
|
105 |
|
|
|
106 |
self.__writer.addDocument(doc) |
|
|
107 |
|
|
94
|
108 |
self.__writer.commit() |