web/hdabo/management/commands/querywikipedia.py
author ymh <ymh.work@gmail.com>
Tue, 14 Jun 2011 01:51:26 +0200
changeset 24 8f84b9bbd22d
parent 21 20d3375b6d28
child 25 e5f8cb1020c8
permissions -rw-r--r--
add wikipedia link edition
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jun 7, 2011
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
     8
from django.conf import settings
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
from django.core.management.base import NoArgsCommand
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.core.management.color import no_style
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from hdabo.models import Tag
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
    12
from hdabo.wp_utils import query_wikipedia_title
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
from optparse import make_option
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
    14
from wikitools import wiki
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
import math
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
import sys
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
class Command(NoArgsCommand):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
    query and update wikipedia for tag title.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
    options = ''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    help = """query and update wikipedia for tag title."""
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    option_list = NoArgsCommand.option_list + (
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
        make_option('--force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
            dest='force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
            help='force all tags to be updated, not only those not yet processed'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
        make_option('--random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
            dest='random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
            help='randomize query on tags'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
        make_option('--site',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
            type='string',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
            dest='site_url',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
            default="http://fr.wikipedia.org/w/api.php",
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
            help='the url for the wikipedia site'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
        make_option('--limit',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
            dest='limit',
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    46
            default= -1,
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
            help='number of tag to process'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
        make_option('--start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
            dest='start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
            default=0,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
            help='number of tag to ignore'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
        )
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
    def __is_homonymie(self, page_dict):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
        for cat in page_dict.get(u"categories", []):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
            if u'Catégorie:Homonymie' in cat.get(u"title", u"") or u'Category:Disambiguation pages' in cat.get(u"title", u""):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
                return True
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
        return False
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    63
    def process_wp_response(self, label, response):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        query_dict = response['query']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
        # get page if multiple pages or none -> return Tag.null_result
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
        pages = query_dict.get("pages", {})
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    69
        if len(pages) > 1 or len(pages) == 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
        page = pages.values()[0]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
        if u"invalid" in page or u"missing" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
        url = page.get(u'fullurl', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
        pageid = page.get(u'pageid', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        new_label = page[u'title']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
        if self.__is_homonymie(page):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
            status = Tag.TAG_URL_STATUS_DICT["homonyme"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
        elif u"redirect" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
            status = Tag.TAG_URL_STATUS_DICT["redirection"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
            status = Tag.TAG_URL_STATUS_DICT["match"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
        return new_label, status, url, pageid 
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
    def show_progress(self, current_line, total_line, label, width):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
        percent = (float(current_line) / float(total_line)) * 100.0
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
        marks = math.floor(width * (percent / 100.0))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
        spaces = math.floor(width - marks)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
        loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
        sys.stdout.write(u"%s %d%% %d/%d - %s\r" % (loader, percent, current_line - 1, total_line - 1, repr(label))) #takes the header into account
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
        if percent >= 100:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
            sys.stdout.write("\n")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
        sys.stdout.flush()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
    def handle_noargs(self, **options):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
        self.style = no_style()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
        interactive = options.get('interactive', True)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        verbosity = int(options.get('verbosity', '1'))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
        force = options.get('force', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
        limit = options.get("limit", -1)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
        start = options.get("start", 0)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
        
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   117
        site_url = options.get('site_url', settings.WIKIPEDIA_API_URL)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
        random = options.get('random', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
        if verbosity > 2:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
            print "option passed : " + repr(options)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
        if force and interactive:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
            confirm = raw_input("""You have requested to query and replace the wikipedia information for all datasheets.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
Are you sure you want to do this ?
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
    Type 'yes' to continue, or 'no' to cancel: """)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
            confirm = 'yes'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
        if confirm != "yes":
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
            print "wikipedia query cancelled"
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
            return
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
        if force:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
            queryset = Tag.objects.all()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
            queryset = Tag.objects.filter(url_status=None)                    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
        if random:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
            queryset = queryset.order_by("?")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
            queryset = queryset.order_by("label")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   145
        if limit >= 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
            queryset = queryset[start:limit]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
            queryset = queryset[start:]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
        if verbosity > 2 :
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
            print "Tag Query is %s" % (queryset.query)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   154
        site = wiki.Wiki(site_url) #@UndefinedVariable
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        count = queryset.count()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
        if verbosity > 1:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
            print "Processing %d tags" % (count)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   163
        for i, tag in enumerate(queryset):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
            if verbosity > 1:
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   166
                print "processing tag %s (%d/%d)" % (tag.label, i + 1, count)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
            else:
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   168
                self.show_progress(i + 1, count, tag.label, 60)                            
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   169
                
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   170
            new_label, status, url, pageid, response = query_wikipedia_title(site, tag.label)
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   171
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   172
            if verbosity >= 2 :
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   173
                print "response from query to %s with parameters %s :" % (site_url, repr(new_label))
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
                print repr(response)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
            if new_label is not None:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
                tag.label = new_label
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
            if status is not None:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
                tag.url_status = status
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
            if url is not None:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   181
                tag.wikipedia_url = url
17
89782c9e96cf update wikipedia_activated to true
ymh <ymh.work@gmail.com>
parents: 15
diff changeset
   182
                tag.wikipedia_activated = True
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   183
            if pageid is not None:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   184
                tag.wikipedia_pageid = pageid
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   185
                
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   186
            tag.save()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   187
            
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   188