web/hdabo/management/commands/querywikipedia.py
author ymh <ymh.work@gmail.com>
Tue, 14 Jun 2011 12:56:48 +0200
changeset 25 e5f8cb1020c8
parent 24 8f84b9bbd22d
child 43 e0812bc3ef44
permissions -rw-r--r--
add command to reset wikipedia info on a tag
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jun 7, 2011
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
     8
from django.conf import settings
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
from django.core.management.base import NoArgsCommand
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.core.management.color import no_style
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from hdabo.models import Tag
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
    12
from hdabo.wp_utils import query_wikipedia_title
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
from optparse import make_option
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
    14
from wikitools import wiki
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
import math
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
import sys
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
25
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    18
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    19
def process_tag(site, tag, verbosity):
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    20
    new_label, status, url, pageid, response = query_wikipedia_title(site, tag.label)
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    21
    
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    22
    if verbosity >= 2 :
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    23
        print "response from query to %s with parameters %s :" % (site.apibase, repr(new_label))
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    24
        print repr(response)
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    25
    
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    26
    if new_label is not None:
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    27
        tag.label = new_label
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    28
    if status is not None:
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    29
        tag.url_status = status
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    30
    if url is not None:
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    31
        tag.wikipedia_url = url
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    32
        tag.wikipedia_activated = True
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    33
    if pageid is not None:
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    34
        tag.wikipedia_pageid = pageid
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    35
        
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    36
    tag.save()
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    37
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    38
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
    39
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
class Command(NoArgsCommand):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
    query and update wikipedia for tag title.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
    '''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    options = ''
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
    help = """query and update wikipedia for tag title."""
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
    option_list = NoArgsCommand.option_list + (
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
        make_option('--force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
            dest='force',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
            help='force all tags to be updated, not only those not yet processed'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
        make_option('--random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
            action='store_true',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
            dest='random',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
            default=False,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
            help='randomize query on tags'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
        make_option('--site',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
            type='string',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
            dest='site_url',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
            default="http://fr.wikipedia.org/w/api.php",
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
            help='the url for the wikipedia site'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
        make_option('--limit',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
            dest='limit',
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    68
            default= -1,
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
            help='number of tag to process'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
        make_option('--start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
            action='store',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
            type='int',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
            dest='start',
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
            default=0,
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
            help='number of tag to ignore'),
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
        )
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
    def __is_homonymie(self, page_dict):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        for cat in page_dict.get(u"categories", []):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
            if u'Catégorie:Homonymie' in cat.get(u"title", u"") or u'Category:Disambiguation pages' in cat.get(u"title", u""):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
                return True
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
        return False
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    85
    def process_wp_response(self, label, response):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
        query_dict = response['query']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
        # get page if multiple pages or none -> return Tag.null_result
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
        pages = query_dict.get("pages", {})
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
    91
        if len(pages) > 1 or len(pages) == 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
        page = pages.values()[0]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
        if u"invalid" in page or u"missing" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
            return None, Tag.TAG_URL_STATUS_DICT["null_result"], None, None
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
        url = page.get(u'fullurl', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
        pageid = page.get(u'pageid', None)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
        new_label = page[u'title']
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
        if self.__is_homonymie(page):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
            status = Tag.TAG_URL_STATUS_DICT["homonyme"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
        elif u"redirect" in page:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
            status = Tag.TAG_URL_STATUS_DICT["redirection"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
            status = Tag.TAG_URL_STATUS_DICT["match"]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        return new_label, status, url, pageid 
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
    def show_progress(self, current_line, total_line, label, width):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
        percent = (float(current_line) / float(total_line)) * 100.0
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
        marks = math.floor(width * (percent / 100.0))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
        spaces = math.floor(width - marks)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
        loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
        sys.stdout.write(u"%s %d%% %d/%d - %s\r" % (loader, percent, current_line - 1, total_line - 1, repr(label))) #takes the header into account
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
        if percent >= 100:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
            sys.stdout.write("\n")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
        sys.stdout.flush()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
    def handle_noargs(self, **options):
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
        self.style = no_style()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
        interactive = options.get('interactive', True)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
        verbosity = int(options.get('verbosity', '1'))
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
        force = options.get('force', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
        limit = options.get("limit", -1)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
        start = options.get("start", 0)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
        
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   139
        site_url = options.get('site_url', settings.WIKIPEDIA_API_URL)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
        random = options.get('random', False)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
        if verbosity > 2:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
            print "option passed : " + repr(options)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
        if force and interactive:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
            confirm = raw_input("""You have requested to query and replace the wikipedia information for all datasheets.
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
Are you sure you want to do this ?
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
    Type 'yes' to continue, or 'no' to cancel: """)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
            confirm = 'yes'
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
        if confirm != "yes":
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   154
            print "wikipedia query cancelled"
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
            return
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        if force:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
            queryset = Tag.objects.all()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
            queryset = Tag.objects.filter(url_status=None)                    
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        if random:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
            queryset = queryset.order_by("?")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
            queryset = queryset.order_by("label")
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   167
        if limit >= 0:
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
            queryset = queryset[start:limit]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
        else:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   170
            queryset = queryset[start:]
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   171
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   172
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
        if verbosity > 2 :
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
            print "Tag Query is %s" % (queryset.query)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   176
        site = wiki.Wiki(site_url) #@UndefinedVariable
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
        count = queryset.count()
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
        if verbosity > 1:
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   181
            print "Processing %d tags" % (count)
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   182
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   183
        
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   184
        
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   185
        for i, tag in enumerate(queryset):
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   186
            
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   187
            if verbosity > 1:
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   188
                print "processing tag %s (%d/%d)" % (tag.label, i + 1, count)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   189
            else:
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 21
diff changeset
   190
                self.show_progress(i + 1, count, tag.label, 60)                            
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   191
            
25
e5f8cb1020c8 add command to reset wikipedia info on a tag
ymh <ymh.work@gmail.com>
parents: 24
diff changeset
   192
            process_tag(site, tag, verbosity)
15
a9136d8f0b4a add commant to reorder tags and query wikipedia
ymh <ymh.work@gmail.com>
parents:
diff changeset
   193
            
21
20d3375b6d28 reformat code
ymh <ymh.work@gmail.com>
parents: 19
diff changeset
   194