web/hdabo/utils.py
author ymh <ymh.work@gmail.com>
Tue, 17 Jan 2012 00:18:49 +0100
changeset 111 ceb381f5b0c7
parent 74 85f7f99c044c
child 113 0d2bfd84b989
permissions -rw-r--r--
query wp for categories and infoboxes
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 11
diff changeset
     1
# -*- coding: utf-8 -*-
11
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
     2
import collections
56
e70cbbc093cc improve tag letter list
ymh <ymh.work@gmail.com>
parents: 49
diff changeset
     3
import unicodedata
111
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
     4
import sys
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
     5
import math
0
896db0083b76 first commit
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
896db0083b76 first commit
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
###
896db0083b76 first commit
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
# allow to declare a property as a decorator
896db0083b76 first commit
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
###
896db0083b76 first commit
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
def Property(func):
11
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    11
    return property(**func())
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    12
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    13
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    14
## {{{ http://code.activestate.com/recipes/576694/ (r7)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    15
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    16
KEY, PREV, NEXT = range(3)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    17
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    18
class OrderedSet(collections.MutableSet):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    19
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    20
    def __init__(self, iterable=None):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    21
        self.end = end = [] 
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    22
        end += [None, end, end]         # sentinel node for doubly linked list
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    23
        self.map = {}                   # key --> [key, prev, next]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    24
        if iterable is not None:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    25
            self |= iterable
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    26
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    27
    def __len__(self):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    28
        return len(self.map)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    29
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    30
    def __contains__(self, key):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    31
        return key in self.map
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    32
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    33
    def add(self, key):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    34
        if key not in self.map:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    35
            end = self.end
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    36
            curr = end[PREV]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    37
            curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    38
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    39
    def discard(self, key):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    40
        if key in self.map:        
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    41
            key, prev, next = self.map.pop(key)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    42
            prev[NEXT] = next
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    43
            next[PREV] = prev
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    44
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    45
    def __iter__(self):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    46
        end = self.end
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    47
        curr = end[NEXT]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    48
        while curr is not end:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    49
            yield curr[KEY]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    50
            curr = curr[NEXT]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    51
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    52
    def __reversed__(self):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    53
        end = self.end
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    54
        curr = end[PREV]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    55
        while curr is not end:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    56
            yield curr[KEY]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    57
            curr = curr[PREV]
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    58
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    59
    def pop(self, last=True):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    60
        if not self:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    61
            raise KeyError('set is empty')
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    62
        key = next(reversed(self)) if last else next(iter(self))
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    63
        self.discard(key)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    64
        return key
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    65
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    66
    def __repr__(self):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    67
        if not self:
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    68
            return '%s()' % (self.__class__.__name__,)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    69
        return '%s(%r)' % (self.__class__.__name__, list(self))
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    70
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    71
    def __eq__(self, other):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    72
        if isinstance(other, OrderedSet):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    73
            return len(self) == len(other) and list(self) == list(other)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    74
        return set(self) == set(other)
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    75
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    76
    def __del__(self):
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    77
        self.clear()                    # remove circular references
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    78
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    79
            
143ab88d17f8 add ordered manytomany fields and indexing
ymh <ymh.work@gmail.com>
parents: 0
diff changeset
    80
## end of http://code.activestate.com/recipes/576694/ }}}
24
8f84b9bbd22d add wikipedia link edition
ymh <ymh.work@gmail.com>
parents: 11
diff changeset
    81
46
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    82
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    83
## {{{ http://code.activestate.com/recipes/576693/ (r9)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    84
# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    85
# Passes Python2.7's test suite and incorporates all the latest updates.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    86
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    87
try:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    88
    from thread import get_ident as _get_ident
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    89
except ImportError:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    90
    from dummy_thread import get_ident as _get_ident
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    91
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    92
try:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    93
    from _abcoll import KeysView, ValuesView, ItemsView
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    94
except ImportError:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    95
    pass
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    96
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    97
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    98
class OrderedDict(dict):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
    99
    'Dictionary that remembers insertion order'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   100
    # An inherited dict maps keys to values.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   101
    # The inherited dict provides __getitem__, __len__, __contains__, and get.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   102
    # The remaining methods are order-aware.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   103
    # Big-O running times for all methods are the same as for regular dictionaries.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   104
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   105
    # The internal self.__map dictionary maps keys to links in a doubly linked list.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   106
    # The circular doubly linked list starts and ends with a sentinel element.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   107
    # The sentinel element never gets deleted (this simplifies the algorithm).
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   108
    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   109
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   110
    def __init__(self, *args, **kwds):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   111
        '''Initialize an ordered dictionary.  Signature is the same as for
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   112
        regular dictionaries, but keyword arguments are not recommended
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   113
        because their insertion order is arbitrary.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   114
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   115
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   116
        if len(args) > 1:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   117
            raise TypeError('expected at most 1 arguments, got %d' % len(args))
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   118
        try:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   119
            self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   120
        except AttributeError:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   121
            self.__root = root = []                     # sentinel node
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   122
            root[:] = [root, root, None]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   123
            self.__map = {}
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   124
        self.__update(*args, **kwds)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   125
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   126
    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   127
        'od.__setitem__(i, y) <==> od[i]=y'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   128
        # Setting a new item creates a new link which goes at the end of the linked
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   129
        # list, and the inherited dictionary is updated with the new key/value pair.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   130
        if key not in self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   131
            root = self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   132
            last = root[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   133
            last[1] = root[0] = self.__map[key] = [last, root, key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   134
        dict_setitem(self, key, value)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   135
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   136
    def __delitem__(self, key, dict_delitem=dict.__delitem__):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   137
        'od.__delitem__(y) <==> del od[y]'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   138
        # Deleting an existing item uses self.__map to find the link which is
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   139
        # then removed by updating the links in the predecessor and successor nodes.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   140
        dict_delitem(self, key)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   141
        link_prev, link_next, key = self.__map.pop(key)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   142
        link_prev[1] = link_next
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   143
        link_next[0] = link_prev
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   144
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   145
    def __iter__(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   146
        'od.__iter__() <==> iter(od)'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   147
        root = self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   148
        curr = root[1]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   149
        while curr is not root:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   150
            yield curr[2]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   151
            curr = curr[1]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   152
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   153
    def __reversed__(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   154
        'od.__reversed__() <==> reversed(od)'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   155
        root = self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   156
        curr = root[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   157
        while curr is not root:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   158
            yield curr[2]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   159
            curr = curr[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   160
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   161
    def clear(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   162
        'od.clear() -> None.  Remove all items from od.'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   163
        try:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   164
            for node in self.__map.itervalues():
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   165
                del node[:]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   166
            root = self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   167
            root[:] = [root, root, None]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   168
            self.__map.clear()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   169
        except AttributeError:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   170
            pass
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   171
        dict.clear(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   172
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   173
    def popitem(self, last=True):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   174
        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   175
        Pairs are returned in LIFO order if last is true or FIFO order if false.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   176
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   177
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   178
        if not self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   179
            raise KeyError('dictionary is empty')
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   180
        root = self.__root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   181
        if last:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   182
            link = root[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   183
            link_prev = link[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   184
            link_prev[1] = root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   185
            root[0] = link_prev
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   186
        else:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   187
            link = root[1]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   188
            link_next = link[1]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   189
            root[1] = link_next
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   190
            link_next[0] = root
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   191
        key = link[2]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   192
        del self.__map[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   193
        value = dict.pop(self, key)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   194
        return key, value
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   195
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   196
    # -- the following methods do not depend on the internal structure --
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   197
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   198
    def keys(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   199
        'od.keys() -> list of keys in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   200
        return list(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   201
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   202
    def values(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   203
        'od.values() -> list of values in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   204
        return [self[key] for key in self]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   205
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   206
    def items(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   207
        'od.items() -> list of (key, value) pairs in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   208
        return [(key, self[key]) for key in self]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   209
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   210
    def iterkeys(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   211
        'od.iterkeys() -> an iterator over the keys in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   212
        return iter(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   213
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   214
    def itervalues(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   215
        'od.itervalues -> an iterator over the values in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   216
        for k in self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   217
            yield self[k]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   218
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   219
    def iteritems(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   220
        'od.iteritems -> an iterator over the (key, value) items in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   221
        for k in self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   222
            yield (k, self[k])
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   223
49
55e5f8a878ae add the version context_processor and correct css to display it correctly
ymh <ymh.work@gmail.com>
parents: 46
diff changeset
   224
    def update(*args, **kwds): #@NoSelf
46
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   225
        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   226
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   227
        If E is a dict instance, does:           for k in E: od[k] = E[k]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   228
        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   229
        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   230
        In either case, this is followed by:     for k, v in F.items(): od[k] = v
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   231
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   232
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   233
        if len(args) > 2:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   234
            raise TypeError('update() takes at most 2 positional '
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   235
                            'arguments (%d given)' % (len(args),))
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   236
        elif not args:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   237
            raise TypeError('update() takes at least 1 argument (0 given)')
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   238
        self = args[0]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   239
        # Make progressively weaker assumptions about "other"
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   240
        other = ()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   241
        if len(args) == 2:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   242
            other = args[1]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   243
        if isinstance(other, dict):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   244
            for key in other:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   245
                self[key] = other[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   246
        elif hasattr(other, 'keys'):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   247
            for key in other.keys():
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   248
                self[key] = other[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   249
        else:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   250
            for key, value in other:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   251
                self[key] = value
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   252
        for key, value in kwds.items():
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   253
            self[key] = value
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   254
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   255
    __update = update  # let subclasses override update without breaking __init__
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   256
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   257
    __marker = object()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   258
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   259
    def pop(self, key, default=__marker):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   260
        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   261
        If key is not found, d is returned if given, otherwise KeyError is raised.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   262
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   263
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   264
        if key in self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   265
            result = self[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   266
            del self[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   267
            return result
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   268
        if default is self.__marker:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   269
            raise KeyError(key)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   270
        return default
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   271
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   272
    def setdefault(self, key, default=None):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   273
        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   274
        if key in self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   275
            return self[key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   276
        self[key] = default
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   277
        return default
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   278
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   279
    def __repr__(self, _repr_running={}):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   280
        'od.__repr__() <==> repr(od)'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   281
        call_key = id(self), _get_ident()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   282
        if call_key in _repr_running:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   283
            return '...'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   284
        _repr_running[call_key] = 1
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   285
        try:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   286
            if not self:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   287
                return '%s()' % (self.__class__.__name__,)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   288
            return '%s(%r)' % (self.__class__.__name__, self.items())
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   289
        finally:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   290
            del _repr_running[call_key]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   291
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   292
    def __reduce__(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   293
        'Return state information for pickling'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   294
        items = [[k, self[k]] for k in self]
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   295
        inst_dict = vars(self).copy()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   296
        for k in vars(OrderedDict()):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   297
            inst_dict.pop(k, None)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   298
        if inst_dict:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   299
            return (self.__class__, (items,), inst_dict)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   300
        return self.__class__, (items,)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   301
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   302
    def copy(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   303
        'od.copy() -> a shallow copy of od'
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   304
        return self.__class__(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   305
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   306
    @classmethod
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   307
    def fromkeys(cls, iterable, value=None):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   308
        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   309
        and values equal to v (which defaults to None).
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   310
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   311
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   312
        d = cls()
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   313
        for key in iterable:
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   314
            d[key] = value
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   315
        return d
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   316
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   317
    def __eq__(self, other):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   318
        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   319
        while comparison to a regular mapping is order-insensitive.
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   320
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   321
        '''
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   322
        if isinstance(other, OrderedDict):
56
e70cbbc093cc improve tag letter list
ymh <ymh.work@gmail.com>
parents: 49
diff changeset
   323
            return len(self) == len(other) and self.items() == other.items()
46
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   324
        return dict.__eq__(self, other)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   325
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   326
    def __ne__(self, other):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   327
        return not self == other
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   328
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   329
    # -- the following methods are only used in Python 2.7 --
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   330
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   331
    def viewkeys(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   332
        "od.viewkeys() -> a set-like object providing a view on od's keys"
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   333
        return KeysView(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   334
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   335
    def viewvalues(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   336
        "od.viewvalues() -> an object providing a view on od's values"
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   337
        return ValuesView(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   338
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   339
    def viewitems(self):
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   340
        "od.viewitems() -> a set-like object providing a view on od's items"
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   341
        return ItemsView(self)
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   342
## end of http://code.activestate.com/recipes/576693/ }}}
3ad571e54608 get categories enhancement.
cavaliet
parents: 24
diff changeset
   343
56
e70cbbc093cc improve tag letter list
ymh <ymh.work@gmail.com>
parents: 49
diff changeset
   344
def remove_accents(str):
e70cbbc093cc improve tag letter list
ymh <ymh.work@gmail.com>
parents: 49
diff changeset
   345
    nkfd_form = unicodedata.normalize('NFKD', unicode(str))
e70cbbc093cc improve tag letter list
ymh <ymh.work@gmail.com>
parents: 49
diff changeset
   346
    return u"".join([c for c in nkfd_form if not unicodedata.combining(c)])
72
ba8ebabbaece -correct css and display
ymh <ymh.work@gmail.com>
parents: 56
diff changeset
   347
ba8ebabbaece -correct css and display
ymh <ymh.work@gmail.com>
parents: 56
diff changeset
   348
def normalize(str):
74
85f7f99c044c improve normalization
ymh <ymh.work@gmail.com>
parents: 72
diff changeset
   349
    return remove_accents(str).lower().replace(u"œ",u"oe")
111
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   350
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   351
def show_progress(current_line, total_line, label, width):
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   352
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   353
    percent = (float(current_line) / float(total_line)) * 100.0
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   354
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   355
    marks = math.floor(width * (percent / 100.0))
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   356
    spaces = math.floor(width - marks)
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   357
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   358
    loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']'
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   359
    
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   360
    sys.stdout.write(u"%s %d%% %d/%d - %r\r" % (loader, percent, current_line, total_line, label[:50].rjust(50))) #takes the header into account
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   361
    if percent >= 100:
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   362
        sys.stdout.write("\n")
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   363
    sys.stdout.flush()
ceb381f5b0c7 query wp for categories and infoboxes
ymh <ymh.work@gmail.com>
parents: 74
diff changeset
   364