web/hdabo/management/commands/query_wikipedia.py
author ymh <ymh.work@gmail.com>
Wed, 22 Jun 2011 01:00:47 +0200
changeset 47 08b008c5a07d
parent 43 web/hdabo/management/commands/querywikipedia.py@e0812bc3ef44
child 108 4b73a767a6c0
permissions -rw-r--r--
- add popularity - calculate dbpedia_uri - display dbpedia uri - add manual_order - various corrections
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jun 7, 2011
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
     8
from django.conf import settings
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
from django.core.management.base import NoArgsCommand
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.core.management.color import no_style
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from hdabo.models import Tag
47
08b008c5a07d - add popularity
ymh <ymh.work@gmail.com>
parents: 43
diff changeset
    12
from hdabo.wp_utils import process_tag
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
from optparse import make_option
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
    14
from wikitools import wiki
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
import math
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
import sys
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
25
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    18
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    19
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
class Command(NoArgsCommand):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
    query and update wikipedia for tag title.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
    options = ''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    help = """query and update wikipedia for tag title."""
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    option_list = NoArgsCommand.option_list + (
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
        make_option('--force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
            dest='force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
            help='force all tags to be updated, not only those not yet processed'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
        make_option('--random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
            dest='random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
            help='randomize query on tags'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
        make_option('--site',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
            type='string',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
            dest='site_url',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
            default="http://fr.wikipedia.org/w/api.php",
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
            help='the url for the wikipedia site'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
        make_option('--limit',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
            dest='limit',
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    48
            default= -1,
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
            help='number of tag to process'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
        make_option('--start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
            dest='start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
            default=0,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
            help='number of tag to ignore'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
        )
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    def __is_homonymie(self, page_dict):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
        for cat in page_dict.get(u"categories", []):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
            if u'Catégorie:Homonymie' in cat.get(u"title", u"") or u'Category:Disambiguation pages' in cat.get(u"title", u""):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
                return True
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
        return False
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    65
    def process_wp_response(self, label, response):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
        query_dict = response['query']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
        # get page if multiple pages or none -> return Tag.null_result
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
        pages = query_dict.get("pages", {})
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    71
        if len(pages) > 1 or len(pages) == 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
        page = pages.values()[0]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
        if u"invalid" in page or u"missing" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        url = page.get(u'fullurl', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
        pageid = page.get(u'pageid', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
        new_label = page[u'title']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
        if self.__is_homonymie(page):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
            status = Tag.TAG_URL_STATUS_DICT["homonyme"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
        elif u"redirect" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
            status = Tag.TAG_URL_STATUS_DICT["redirection"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
            status = Tag.TAG_URL_STATUS_DICT["match"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
        return new_label, status, url, pageid 
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
    def show_progress(self, current_line, total_line, label, width):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
        percent = (float(current_line) / float(total_line)) * 100.0
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
        marks = math.floor(width * (percent / 100.0))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
        spaces = math.floor(width - marks)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
        loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
        sys.stdout.write(u"%s %d%% %d/%d - %s\r" % (loader, percent, current_line - 1, total_line - 1, repr(label))) #takes the header into account
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
        if percent >= 100:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
            sys.stdout.write("\n")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
        sys.stdout.flush()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
    def handle_noargs(self, **options):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
        self.style = no_style()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        interactive = options.get('interactive', True)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
        verbosity = int(options.get('verbosity', '1'))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
        force = options.get('force', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
        limit = options.get("limit", -1)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
        start = options.get("start", 0)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
        
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   119
        site_url = options.get('site_url', settings.WIKIPEDIA_API_URL)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
        random = options.get('random', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
        if verbosity > 2:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
            print "option passed : " + repr(options)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
        if force and interactive:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
            confirm = raw_input("""You have requested to query and replace the wikipedia information for all datasheets.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
Are you sure you want to do this ?
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
    Type 'yes' to continue, or 'no' to cancel: """)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
            confirm = 'yes'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
        if confirm != "yes":
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
            print "wikipedia query cancelled"
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
            return
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
        if force:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
            queryset = Tag.objects.all()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
            queryset = Tag.objects.filter(url_status=None)                    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
        if random:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
            queryset = queryset.order_by("?")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
            queryset = queryset.order_by("label")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   147
        if limit >= 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
            queryset = queryset[start:limit]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
            queryset = queryset[start:]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
        if verbosity > 2 :
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   154
            print "Tag Query is %s" % (queryset.query)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   156
        site = wiki.Wiki(site_url) #@UndefinedVariable
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
        count = queryset.count()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
        if verbosity > 1:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
            print "Processing %d tags" % (count)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   165
        for i, tag in enumerate(queryset):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
            if verbosity > 1:
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   168
                print "processing tag %s (%d/%d)" % (tag.label, i + 1, count)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
            else:
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   170
                self.show_progress(i + 1, count, tag.label, 60)                            
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   171
            
25
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
   172
            process_tag(site, tag, verbosity)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
            
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   174