| author | ymh <ymh.work@gmail.com> |
| Wed, 11 Apr 2018 12:19:47 +0200 | |
| branch | documentation |
| changeset 693 | 09e00f38d177 |
| parent 683 | 59d49ab04ded |
| permissions | -rw-r--r-- |
| 204 | 1 |
# -*- coding: utf-8 -*- |
2 |
''' |
|
| 693 | 3 |
Requête wikipedia pour reconstituer l'arbre des catégories. |
4 |
||
5 |
Cette commande utilise directement `l'api wikipedia <https://www.mediawiki.org/wiki/API:Main_page>`_ pour faire ses requêtes. |
|
6 |
||
7 |
**Usage**: ``django-admin query_category_inclusion [options]`` |
|
8 |
||
9 |
**Options spécifiques:** |
|
| 204 | 10 |
|
| 693 | 11 |
- *\-\-all* : force à traiter toutes les catégories |
12 |
- *\-\-force* : ne pose aucune question |
|
13 |
- *\-\-site=SITE_URL* : url du site wikipedia (défaut: https://fr.wikipedia.org/w/api.php) |
|
14 |
- *\-\-limit=LIMIT* : Nombre de catégories à traiter |
|
15 |
- *\-\-start=START* : Nombre de catégories à ignorer |
|
16 |
- *\-\-category=CATEGORY* : Limite le traitement à cette catégorie |
|
17 |
||
| 204 | 18 |
''' |
19 |
||
20 |
from django.conf import settings |
|
21 |
from django.core.management.base import NoArgsCommand |
|
22 |
from django.core.management.color import no_style |
|
23 |
from hdalab.models import WpCategory, WpCategoryInclusion |
|
24 |
from optparse import make_option |
|
25 |
from wikitools import api,wiki |
|
26 |
import sys |
|
27 |
import re |
|
28 |
import itertools |
|
29 |
from hdabo import utils |
|
30 |
from django.db.models import Count |
|
31 |
from django.db import transaction |
|
32 |
||
33 |
CATEGORY_PREFIX = u'Catégorie:' |
|
34 |
||
35 |
class Command(NoArgsCommand): |
|
36 |
''' |
|
37 |
query and update wikipedia for tag title. |
|
38 |
''' |
|
39 |
options = '' |
|
40 |
help = """query and update wikipedia for tag title.""" |
|
| 693 | 41 |
|
| 204 | 42 |
option_list = NoArgsCommand.option_list + ( |
43 |
make_option('--all', |
|
44 |
action='store_true', |
|
45 |
dest='all', |
|
46 |
default=False, |
|
47 |
help='force all categories to be updated, not only those not yet processed'), |
|
48 |
make_option('--force', |
|
49 |
action='store_true', |
|
50 |
dest='force', |
|
51 |
default=False, |
|
52 |
help='ask no questions'), |
|
53 |
make_option('--site', |
|
54 |
action='store', |
|
55 |
type='string', |
|
56 |
dest='site_url', |
|
|
683
59d49ab04ded
use https for wikipedia api endpoint
ymh <ymh.work@gmail.com>
parents:
571
diff
changeset
|
57 |
default="https://fr.wikipedia.org/w/api.php", |
| 204 | 58 |
help='the url for the wikipedia site'), |
59 |
make_option('--limit', |
|
60 |
action='store', |
|
61 |
type='int', |
|
62 |
dest='limit', |
|
63 |
default= -1, |
|
64 |
help='number of categories to process'), |
|
65 |
make_option('--start', |
|
66 |
action='store', |
|
67 |
type='int', |
|
68 |
dest='start', |
|
69 |
default=0, |
|
70 |
help='number of categories to ignore'), |
|
71 |
make_option('--category', |
|
72 |
action='append', |
|
73 |
dest='category', |
|
74 |
type='string', |
|
75 |
default=[], |
|
76 |
help='the categories to query'), |
|
77 |
||
78 |
) |
|
79 |
||
80 |
||
81 |
def query_all_categories(self, category_title, site): |
|
| 693 | 82 |
|
| 204 | 83 |
params = {'action':'query', 'cmtitle':category_title, 'list':'categorymembers', 'cmlimit': 'max'} |
| 693 | 84 |
|
| 204 | 85 |
res = [] |
| 693 | 86 |
|
| 204 | 87 |
wpquery = api.APIRequest(site, params) #@UndefinedVariable |
88 |
response = wpquery.query() |
|
| 693 | 89 |
|
| 204 | 90 |
if self.verbosity > 1: |
91 |
print "Query category : " + repr(wpquery.request.get_full_url()+"?"+wpquery.request.get_data()) |
|
92 |
print repr(response) |
|
| 693 | 93 |
|
| 204 | 94 |
members = response.get('query', {}).get('categorymembers', []) |
| 693 | 95 |
|
| 204 | 96 |
for member in members: |
97 |
title = member.get('title',"") |
|
98 |
if re.match(CATEGORY_PREFIX, title): |
|
99 |
res.append(re.sub(CATEGORY_PREFIX, "", title)) |
|
| 693 | 100 |
|
| 204 | 101 |
if self.verbosity > 1: |
102 |
print "Query categories result: " |
|
103 |
print repr(res) |
|
| 693 | 104 |
|
| 204 | 105 |
return res |
| 693 | 106 |
|
| 204 | 107 |
def process_categories(self, cat_list, parent_cat): |
108 |
for cat in cat_list: |
|
109 |
child_cat,created = WpCategory.objects.get_or_create(label=cat) #@UnusedVariable |
|
110 |
WpCategoryInclusion.objects.get_or_create(parent_category=parent_cat, child_category=child_cat) |
|
| 693 | 111 |
|
| 204 | 112 |
def handle_noargs(self, **options): |
| 693 | 113 |
|
| 204 | 114 |
self.style = no_style() |
| 693 | 115 |
|
| 204 | 116 |
interactive = options.get('interactive', True) |
| 693 | 117 |
|
| 204 | 118 |
self.verbosity = int(options.get('verbosity', '1')) |
| 693 | 119 |
|
| 204 | 120 |
force = options.get('force', False) |
| 693 | 121 |
|
| 204 | 122 |
limit = options.get("limit", -1) |
123 |
start = options.get("start", 0) |
|
| 693 | 124 |
|
| 204 | 125 |
site_url = options.get('site_url', settings.WIKIPEDIA_API_URL) |
| 693 | 126 |
|
| 204 | 127 |
types_mask = 0 |
| 693 | 128 |
|
| 204 | 129 |
if self.verbosity > 2: |
130 |
print "option passed : " + repr(options) |
|
131 |
||
| 206 | 132 |
queryset = WpCategory.objects.filter(tags__hidden = False).distinct() |
| 693 | 133 |
|
| 204 | 134 |
cat_list = options.get("category", []); |
| 693 | 135 |
|
| 204 | 136 |
if cat_list: |
137 |
queryset = queryset.filter(label__in=cat_list) |
|
| 693 | 138 |
elif options.get('all',False): |
139 |
queryset = queryset.annotate(wpc=Count('child_categories')).filter(wpc = 0) |
|
140 |
||
| 206 | 141 |
queryset = queryset.order_by("label") |
| 693 | 142 |
|
| 204 | 143 |
if limit >= 0: |
144 |
queryset = queryset[start:limit] |
|
145 |
elif start > 0: |
|
| 693 | 146 |
queryset = queryset[start:] |
147 |
||
| 204 | 148 |
if self.verbosity > 2 : |
149 |
print "Category Query is %s" % (queryset.query) |
|
| 693 | 150 |
|
| 204 | 151 |
site = wiki.Wiki(site_url) #@UndefinedVariable |
| 693 | 152 |
|
153 |
||
| 204 | 154 |
count = queryset.count() |
155 |
if self.verbosity > 1: |
|
156 |
print "Processing %d categories" % (count) |
|
| 693 | 157 |
|
| 204 | 158 |
if not force and interactive: |
159 |
confirm = raw_input("You have requested to query and replace the wikipedia information for %d categories.\n Are you sure you want to do this? \nType 'yes' to continue, or 'no' to cancel: " % (count)) |
|
160 |
else: |
|
161 |
confirm = 'yes' |
|
| 693 | 162 |
|
| 204 | 163 |
if confirm != "yes": |
164 |
print "wikipedia query cancelled" |
|
165 |
return |
|
| 693 | 166 |
|
| 204 | 167 |
for i, category in enumerate(queryset): |
| 693 | 168 |
|
| 204 | 169 |
if self.verbosity > 1: |
170 |
print "processing category %s (%d/%d)" % (category.label, i + 1, count) |
|
171 |
else: |
|
| 693 | 172 |
utils.show_progress(i + 1, count, category.label, 60) |
173 |
||
| 204 | 174 |
title = CATEGORY_PREFIX + category.label |
175 |
# query categories |
|
176 |
||
|
571
d9642be7c937
replace commit_on_success with atomic
ymh <ymh.work@gmail.com>
parents:
271
diff
changeset
|
177 |
with transaction.atomic(): |
| 204 | 178 |
res = self.query_all_categories(title, site) |
| 693 | 179 |
self.process_categories(res, category) |