src/core/wp_utils.py
author ymh <ymh.work@gmail.com>
Fri, 19 Jul 2013 10:39:17 +0200
changeset 71 3fde7d26ad08
parent 67 5d9223bb3aab
child 80 3851909cb730
permissions -rw-r--r--
Add link_semantic_level to filter (bug #17542)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
from .models import Term, TermLabel, TERM_URL_STATUS_DICT
63
a7f4a418d5af can directly paste wikipedia urls
ymh <ymh.work@gmail.com>
parents: 61
diff changeset
     3
from core.models.term import TERM_WK_LINK_SEMANTIC_LEVEL_DICT
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
from django.conf import settings
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
from django.utils.http import urlquote
63
a7f4a418d5af can directly paste wikipedia urls
ymh <ymh.work@gmail.com>
parents: 61
diff changeset
     6
from urlparse import urlparse
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
from wikitools import api, wiki
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
import logging
63
a7f4a418d5af can directly paste wikipedia urls
ymh <ymh.work@gmail.com>
parents: 61
diff changeset
     9
import urllib2
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
logger = logging.getLogger(__name__)
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    12
sites = {}
33
61c3ffd94f11 - correct imports.
ymh <ymh.work@gmail.com>
parents: 4
diff changeset
    13
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    14
def __get_site(lang):
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    15
    site = sites.get(lang, None)
33
61c3ffd94f11 - correct imports.
ymh <ymh.work@gmail.com>
parents: 4
diff changeset
    16
    if site is None:
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    17
        site = wiki.Wiki(settings.WIKIPEDIA_URLS[lang]['api_url'])  # @UndefinedVariable
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    18
        sites[lang] = site        
33
61c3ffd94f11 - correct imports.
ymh <ymh.work@gmail.com>
parents: 4
diff changeset
    19
    return site
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
def normalize_term(term):
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    if len(term) == 0:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
        return term
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    term = term.strip()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
    term = term.replace("_", " ")
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    term = " ".join(term.split())
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
    term = term[0].upper() + term[1:]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
    return term
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
def switch_case_group(term):
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
    seg_group = term.split()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
    uc_group = []
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
    lc_group = []
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
    for seg in seg_group:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
        is_all_upper = all(c.isupper() or not c.isalpha() for c in seg) 
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
        if is_all_upper and not lc_group:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
            uc_group.append(seg)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
        elif not is_all_upper and uc_group:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
            lc_group.append(seg)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
        else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
            return term
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
            
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    if uc_group and lc_group and len(uc_group)+len(lc_group) == len(seg_group):        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
        return " ".join(lc_group + [normalize_term(t.lower()) for t in uc_group])
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
    elif uc_group and not lc_group and len(uc_group) == len(seg_group):
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
        return " ".join([normalize_term(t.lower()) for t in uc_group])
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
    else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
        return term
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
def urlize_for_wikipedia(label):
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
    return urlquote(label.replace(" ", "_"))
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    56
def __is_homonymie(page_dict, lang):
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
    for cat in page_dict.get(u"categories", []):
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    58
        if settings.WIKIPEDIA_URLS[lang]['disambiguation_cat'] in cat.get(u"title", u""):
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
            return True
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
    return False
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
    63
def query_wikipedia_title(site, lang, label=None, pageid=None):
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
    params = {'action':'query', 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500'}
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
    if label:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
        params['titles'] = label
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
    else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
        params['pageids'] = pageid
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
    response = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
    def return_null_result():
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
        return { 'new_label': None, 'alternative_label': None, 'status': TERM_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'alternative_wikipedia_url': None, 'alternative_pageid': None, 'dbpedia_uri': None, 'revision_id': None, 'response': response }
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
    try:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
        wpquery = api.APIRequest(site, params) #@UndefinedVariable
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        response = wpquery.query()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
    except:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
        logger.exception("Exception when querying wikipedia")
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
        return return_null_result()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
    original_response = response
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
    query_dict = response['query']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
    # get page if multiple pages or none -> return Tag.null_result
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
    pages = query_dict.get("pages", {})
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
    if len(pages) > 1 or len(pages) == 0:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
        return return_null_result()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
    page = pages.values()[0]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
    if u"invalid" in page or u"missing" in page:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
        return return_null_result()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
    url = page.get(u'fullurl', None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
    pageid = page.get(u'pageid', None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
    new_label = page[u'title']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
    alternative_label = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
    alternative_url = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
    alternative_pageid = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
    
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   105
    if __is_homonymie(page, lang):
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
        status = TERM_URL_STATUS_DICT["homonyme"]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
    elif u"redirect" in page:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
        status = TERM_URL_STATUS_DICT["redirection"]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
    else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        status = TERM_URL_STATUS_DICT["match"]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
    if status == TERM_URL_STATUS_DICT["redirection"]:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
        params['redirects'] = True
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
        try:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
            wpquery = api.APIRequest(site, params) #@UndefinedVariable    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
            response = wpquery.query()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
        except:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
            logger.exception("Exception when querying wikipedia for redirects")
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
            return return_null_result()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
        query_dict = response['query']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
        pages = query_dict.get("pages", {})
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
        #we know that we have at least one answer        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
        if len(pages) > 1 or len(pages) == 0:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
            return return_null_result()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
        page = pages.values()[0]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
        alternative_label = page.get('title', None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
        alternative_url = page.get('fullurl', None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
        alternative_pageid = page.get('pageid',None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
    revision_id = page.get('lastrevid', None)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
    if status == TERM_URL_STATUS_DICT['match'] or status == TERM_URL_STATUS_DICT['redirection']:
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   135
        dbpedia_uri = settings.WIKIPEDIA_URLS[lang]['dbpedia_uri'] % (urlize_for_wikipedia(new_label))
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
    else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
        dbpedia_uri = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
            
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
    return { 'new_label': new_label, 'alternative_label': alternative_label, 'status': status, 'wikipedia_url': url, 'pageid': pageid, 'alternative_wikipedia_url': alternative_url, 'alternative_pageid': alternative_pageid, 'dbpedia_uri': dbpedia_uri, 'revision_id': revision_id, 'response': original_response }
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   144
def get_or_create_term(term_label, term_uri, term_lang, thesaurus, lang, wp_label_transform=(lambda l:l), skip_wp_query=False):
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
    term_label_normalized = normalize_term(term_label)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
    # We get the wikipedia references for the tag_label
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
    # We get or create the tag object
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
    
33
61c3ffd94f11 - correct imports.
ymh <ymh.work@gmail.com>
parents: 4
diff changeset
   150
    
61
0048668779c0 change model for thesaurus tree. show level and ancestor
ymh <ymh.work@gmail.com>
parents: 35
diff changeset
   151
    term, created = Term.objects.get_or_create(uri=term_uri, defaults = {'label':term_label, 'thesaurus':thesaurus, 'normalized_label':term_label_normalized, 'lang' : term_lang})  # @UndefinedVariable
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
 
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
    if created:
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   154
        wikipedia_revision_id = process_term(__get_site(lang), term, lang, label=wp_label_transform(term_label_normalized))
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
        term_label_obj = TermLabel(label=term_label, term=term, lang=term_lang)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
        term_label_obj.save()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        
4
ae066d797a66 add link to joconde search. Add option to import
ymh <ymh.work@gmail.com>
parents: 3
diff changeset
   158
    elif term.wikipedia_pageid and not skip_wp_query:
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   159
        wp_res = query_wikipedia_title(__get_site(lang), lang, pageid=term.wikipedia_pageid)
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
        wikipedia_revision_id = wp_res['revision_id']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
        term.wikipedia_revision_id = wikipedia_revision_id
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        term.save()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
    else:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        wikipedia_revision_id = None
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
    return term, wikipedia_revision_id, created
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   170
def process_term(site, term, lang, label=None, verbosity=0):
71
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   171
    
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   172
    label_is_url = False
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   173
    fragment = ""
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
    if not label:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
        label = term.label
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   176
    else:
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   177
        for lang_code, urls in settings.WIKIPEDIA_URLS.iteritems():
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   178
            if label.startswith(urls['page_url']):
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   179
                # lang is overrided when an url is passed as a label.
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   180
                lang = lang_code
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   181
                url_parts = urlparse(label)
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   182
                label = urllib2.unquote(str(url_parts.path.split('/')[-1])).decode("utf-8")
71
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   183
                if url_parts.fragment:
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   184
                    label_is_url = True
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   185
                    fragment = url_parts.fragment
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   186
                break
63
a7f4a418d5af can directly paste wikipedia urls
ymh <ymh.work@gmail.com>
parents: 61
diff changeset
   187
67
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   188
    if site == None:
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   189
        site = __get_site(lang)
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   190
5d9223bb3aab Add other wikipedia.
ymh <ymh.work@gmail.com>
parents: 63
diff changeset
   191
    wp_res = query_wikipedia_title(site, lang, label=label)
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   192
    new_label = wp_res['new_label']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   193
    alternative_label= wp_res['alternative_label']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   194
    status =  wp_res['status']
71
3fde7d26ad08 Add link_semantic_level to filter (bug #17542)
ymh <ymh.work@gmail.com>
parents: 67
diff changeset
   195
    url = wp_res['wikipedia_url'] + ("#"+fragment if label_is_url else "")    
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   196
    alternative_url = wp_res['alternative_wikipedia_url']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   197
    pageid = wp_res['pageid']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   198
    alternative_pageid = wp_res['alternative_pageid']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   199
    response = wp_res['response']
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   200
    dbpedia_uri =  wp_res["dbpedia_uri"]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   201
    revision_id = wp_res["revision_id"]
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   202
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   203
    if verbosity >= 2 :
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   204
        print "response from query to %s with parameters %s :" % (site.apibase, repr(new_label))
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   205
        print repr(response)
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   206
    
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   207
    if new_label is not None:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   208
        term.wp_label = new_label
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   209
    if status is not None:
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   210
        term.url_status = status
35
859862939996 add qualifier on the wikipedia link
ymh <ymh.work@gmail.com>
parents: 33
diff changeset
   211
        term.link_semantic_level = TERM_WK_LINK_SEMANTIC_LEVEL_DICT['--']
0
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   212
    term.wikipedia_url = url
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   213
    term.wikipedia_pageid = pageid
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   214
    term.dbpedia_uri = dbpedia_uri
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   215
    term.alternative_label = alternative_label
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   216
    term.alternative_wikipedia_url = alternative_url
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   217
    term.alternative_wikipedia_pageid = alternative_pageid
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   218
    term.wikipedia_revision_id=revision_id
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   219
        
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   220
    term.save()
4095911a7830 Jocondelab first commit before design
ymh <ymh.work@gmail.com>
parents:
diff changeset
   221
    
3
221af1052ec4 Correct process wikipedia
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   222
    return revision_id
221af1052ec4 Correct process wikipedia
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
   223