--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/web/hdalab/management/commands/query_category_inclusion.py	Mon Jul 02 19:19:02 2012 +0200
@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+'''
+Created on July 2, 2012
+
+@author: raphv
+'''
+
+from django.conf import settings
+from django.core.management.base import NoArgsCommand
+from django.core.management.color import no_style
+from hdalab.models import WpCategory, WpCategoryInclusion
+from optparse import make_option
+from wikitools import api,wiki
+import sys
+import re
+import itertools
+from hdabo import utils
+from django.db.models import Count
+from django.db import transaction
+
+CATEGORY_PREFIX = u'Catégorie:'
+
+class Command(NoArgsCommand):
+    '''
+    query and update wikipedia for tag title.
+    '''
+    options = ''
+    help = """query and update wikipedia for tag title."""
+    
+    option_list = NoArgsCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='force all categories to be updated, not only those not yet processed'),
+        make_option('--force',
+            action='store_true',
+            dest='force',
+            default=False,
+            help='ask no questions'),
+        make_option('--random',
+            action='store_true',
+            dest='random',
+            default=False,
+            help='randomize query on categories'),
+        make_option('--site',
+            action='store',
+            type='string',
+            dest='site_url',
+            default="http://fr.wikipedia.org/w/api.php",
+            help='the url for the wikipedia site'),
+        make_option('--limit',
+            action='store',
+            type='int',
+            dest='limit',
+            default= -1,
+            help='number of categories to process'),
+        make_option('--start',
+            action='store',
+            type='int',
+            dest='start',
+            default=0,
+            help='number of categories to ignore'),
+        make_option('--category',
+            action='append',
+            dest='category',
+            type='string',
+            default=[],
+            help='the categories to query'),
+
+    )
+
+
+    def query_all_categories(self, category_title, site):
+        
+        params = {'action':'query', 'cmtitle':category_title, 'list':'categorymembers', 'cmlimit': 'max'}
+        
+        res = []
+        
+        wpquery = api.APIRequest(site, params) #@UndefinedVariable
+        response = wpquery.query()
+        
+        if self.verbosity > 1:
+            print "Query category : " + repr(wpquery.request.get_full_url()+"?"+wpquery.request.get_data())
+            print repr(response)
+        
+        members = response.get('query', {}).get('categorymembers', [])
+                    
+        for member in members:
+            title = member.get('title',"")
+            if re.match(CATEGORY_PREFIX, title):
+                res.append(re.sub(CATEGORY_PREFIX, "", title))
+            
+        if self.verbosity > 1:
+            print "Query categories result: "
+            print repr(res)
+            
+        return res
+    
+    def process_categories(self, cat_list, parent_cat):
+        for cat in cat_list:
+            child_cat,created = WpCategory.objects.get_or_create(label=cat) #@UnusedVariable
+            WpCategoryInclusion.objects.get_or_create(parent_category=parent_cat, child_category=child_cat)
+        
+    def handle_noargs(self, **options):
+        
+        self.style = no_style()
+        
+        interactive = options.get('interactive', True)
+        
+        self.verbosity = int(options.get('verbosity', '1'))
+        
+        force = options.get('force', False)
+        
+        limit = options.get("limit", -1)
+        start = options.get("start", 0)
+        
+        site_url = options.get('site_url', settings.WIKIPEDIA_API_URL)
+        
+        random = options.get('random', False)
+        
+        types_mask = 0
+        
+        if self.verbosity > 2:
+            print "option passed : " + repr(options)
+
+        queryset = WpCategory.objects
+        
+        cat_list = options.get("category", []);
+        
+        if cat_list:
+            queryset = queryset.filter(label__in=cat_list)
+        elif not options.get('all',False):            
+            queryset = queryset.annotate(wpc=Count('child_categories')).filter(wpc = 0)
+        #else:
+        #    queryset = Tag.objects.filter(url_status=None)                    
+        
+        if random:
+            queryset = queryset.order_by("?")
+        else:
+            queryset = queryset.order_by("label")
+        
+        if limit >= 0:
+            queryset = queryset[start:limit]
+        elif start > 0:
+            queryset = queryset[start:]            
+        
+        if self.verbosity > 2 :
+            print "Category Query is %s" % (queryset.query)
+        
+        site = wiki.Wiki(site_url) #@UndefinedVariable
+        
+        
+        count = queryset.count()
+        if self.verbosity > 1:
+            print "Processing %d categories" % (count)
+        
+        if not force and interactive:
+            confirm = raw_input("You have requested to query and replace the wikipedia information for %d categories.\n Are you sure you want to do this? \nType 'yes' to continue, or 'no' to cancel: " % (count))
+        else:
+            confirm = 'yes'
+            
+        if confirm != "yes":
+            print "wikipedia query cancelled"
+            return
+        
+        for i, category in enumerate(queryset):
+            
+            if self.verbosity > 1:
+                print "processing category %s (%d/%d)" % (category.label, i + 1, count)
+            else:
+                utils.show_progress(i + 1, count, category.label, 60)                            
+                
+            title = CATEGORY_PREFIX + category.label
+            # query categories
+
+            with transaction.commit_on_success():
+                res = self.query_all_categories(title, site)
+                self.process_categories(res, category)
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/web/hdalab/migrations/0010_categoryinclusion.py	Mon Jul 02 19:19:02 2012 +0200
@@ -0,0 +1,312 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'WpCategoryInclusion'
+        db.create_table('hdalab_wpcategoryinclusion', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('parent_category', self.gf('django.db.models.fields.related.ForeignKey')(related_name='child_categories', to=orm['hdalab.WpCategory'])),
+            ('child_category', self.gf('django.db.models.fields.related.ForeignKey')(related_name='parent_categories', to=orm['hdalab.WpCategory'])),
+        ))
+        db.send_create_signal('hdalab', ['WpCategoryInclusion'])
+
+        # Adding unique constraint on 'WpCategoryInclusion', fields ['parent_category', 'child_category']
+        db.create_unique('hdalab_wpcategoryinclusion', ['parent_category_id', 'child_category_id'])
+
+
+    def backwards(self, orm):
+        
+        # Removing unique constraint on 'WpCategoryInclusion', fields ['parent_category', 'child_category']
+        db.delete_unique('hdalab_wpcategoryinclusion', ['parent_category_id', 'child_category_id'])
+
+        # Deleting model 'WpCategoryInclusion'
+        db.delete_table('hdalab_wpcategoryinclusion')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 2, 5, 14, 31, 895549)'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 2, 5, 14, 31, 895384)'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'hdabo.author': {
+            'Meta': {'object_name': 'Author'},
+            'firstname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+            'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lastname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'})
+        },
+        'hdabo.datasheet': {
+            'Meta': {'object_name': 'Datasheet'},
+            'author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Author']", 'null': 'True', 'blank': 'True'}),
+            'college_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+            'college_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_themes']", 'to': "orm['hdabo.Domain']"}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'domains': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_domains']", 'to': "orm['hdabo.Domain']"}),
+            'format': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.DocumentFormat']", 'null': 'True', 'blank': 'True'}),
+            'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+            'highschool_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+            'highschool_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_themes']", 'to': "orm['hdabo.Domain']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'manual_order': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'modification_datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'organisation': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Organisation']", 'null': 'True'}),
+            'original_creation_date': ('django.db.models.fields.DateField', [], {}),
+            'original_modification_date': ('django.db.models.fields.DateField', [], {}),
+            'primary_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+            'primary_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_themes']", 'to': "orm['hdabo.Domain']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['hdabo.Tag']", 'through': "orm['hdabo.TaggedSheet']", 'symmetrical': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+            'town': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Location']", 'null': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'validated': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'validation_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'validator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        'hdabo.datasheet_college_periods': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_periods'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+            'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+        },
+        'hdabo.datasheet_college_themes': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_themes'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdabo.datasheet_domains': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_domains'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdabo.datasheet_highschool_periods': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_periods'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+            'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+        },
+        'hdabo.datasheet_highschool_themes': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_themes'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdabo.datasheet_primary_periods': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_periods'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+            'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+        },
+        'hdabo.datasheet_primary_themes': {
+            'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_themes'},
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sort_value': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdabo.documentformat': {
+            'Meta': {'object_name': 'DocumentFormat'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'})
+        },
+        'hdabo.domain': {
+            'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'Domain'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'school_period': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdabo.location': {
+            'Meta': {'object_name': 'Location'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '5'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '512'})
+        },
+        'hdabo.organisation': {
+            'Meta': {'object_name': 'Organisation'},
+            'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'})
+        },
+        'hdabo.tag': {
+            'Meta': {'unique_together': "(('label', 'original_label', 'url_status'),)", 'object_name': 'Tag'},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'alternative_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'alternative_wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'alternative_wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TagCategory']", 'null': 'True', 'blank': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'dbpedia_uri': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}),
+            'normalized_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}),
+            'original_label': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
+            'popularity': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+            'url_status': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'})
+        },
+        'hdabo.tagcategory': {
+            'Meta': {'object_name': 'TagCategory'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'})
+        },
+        'hdabo.taggedsheet': {
+            'Meta': {'object_name': 'TaggedSheet'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index_note': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'db_index': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+            'original_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Tag']"}),
+            'wikipedia_revision_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'hdabo.timeperiod': {
+            'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'TimePeriod'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'school_period': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'hdalab.country': {
+            'Meta': {'object_name': 'Country'},
+            'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'hdalab.datasheetextras': {
+            'Meta': {'object_name': 'DatasheetExtras'},
+            'datasheet': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'extras'", 'unique': 'True', 'to': "orm['hdabo.Datasheet']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'insee': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.InseeCoords']", 'null': 'True', 'blank': 'True'})
+        },
+        'hdalab.dbpediafields': {
+            'Meta': {'object_name': 'DbpediaFields'},
+            'abstract': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'dbpedia_uri': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+            'tag': ('django.db.models.fields.related.OneToOneField', [], {'blank': 'True', 'related_name': "'dbpedia_fields'", 'unique': 'True', 'null': 'True', 'to': "orm['hdabo.Tag']"}),
+            'thumbnail': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'})
+        },
+        'hdalab.dbpediafieldstranslation': {
+            'Meta': {'unique_together': "(('master', 'language_code'),)", 'object_name': 'DbpediaFieldsTranslation'},
+            'abstract': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+            'language_code': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
+            'master': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'translations'", 'to': "orm['hdalab.DbpediaFields']"})
+        },
+        'hdalab.geoinclusion': {
+            'Meta': {'object_name': 'GeoInclusion'},
+            'country': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'includes'", 'to': "orm['hdalab.Country']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tag': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'locatedin'", 'unique': 'True', 'to': "orm['hdabo.Tag']"})
+        },
+        'hdalab.hdasession': {
+            'Meta': {'object_name': 'HdaSession'},
+            'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'sessionid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36', 'db_index': 'True'})
+        },
+        'hdalab.infoboxparameter': {
+            'Meta': {'unique_together': "(('tag_infobox', 'param_name'),)", 'object_name': 'InfoboxParameter'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'param_name': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+            'param_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'tag_infobox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.TagInfobox']"})
+        },
+        'hdalab.inseecoords': {
+            'Meta': {'object_name': 'InseeCoords'},
+            'city_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'insee': ('django.db.models.fields.IntegerField', [], {'primary_key': 'True'}),
+            'latitude': ('django.db.models.fields.FloatField', [], {}),
+            'longitude': ('django.db.models.fields.FloatField', [], {})
+        },
+        'hdalab.taginfobox': {
+            'Meta': {'unique_together': "(('tag', 'name', 'revision_id'),)", 'object_name': 'TagInfobox'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+            'revision_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'source': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'infoboxes'", 'to': "orm['hdabo.Tag']"})
+        },
+        'hdalab.taglinks': {
+            'Meta': {'object_name': 'TagLinks'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taglinks_objects'", 'to': "orm['hdabo.Tag']"}),
+            'subject': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taglinks_subjects'", 'to': "orm['hdabo.Tag']"})
+        },
+        'hdalab.tagwpcategory': {
+            'Meta': {'unique_together': "(('tag', 'wp_category', 'hidden'),)", 'object_name': 'TagWpCategory'},
+            'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'wp_categories'", 'to': "orm['hdabo.Tag']"}),
+            'wp_category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.WpCategory']"})
+        },
+        'hdalab.tagyears': {
+            'Meta': {'object_name': 'TagYears'},
+            'end_year': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'start_year': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'years'", 'to': "orm['hdabo.Tag']"})
+        },
+        'hdalab.wpcategory': {
+            'Meta': {'object_name': 'WpCategory'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048'})
+        },
+        'hdalab.wpcategoryinclusion': {
+            'Meta': {'unique_together': "(('parent_category', 'child_category'),)", 'object_name': 'WpCategoryInclusion'},
+            'child_category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parent_categories'", 'to': "orm['hdalab.WpCategory']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'parent_category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'child_categories'", 'to': "orm['hdalab.WpCategory']"})
+        }
+    }
+
+    complete_apps = ['hdalab']

--- a/web/hdalab/models/__init__.py	Fri Jun 22 19:16:46 2012 +0200
+++ b/web/hdalab/models/__init__.py	Mon Jul 02 19:19:02 2012 +0200
@@ -1,5 +1,5 @@
 from hdalab.models.categories import (WpCategory, InfoboxParameter, TagInfobox, 
-    TagWpCategory)
+    TagWpCategory, WpCategoryInclusion)
 from hdalab.models.dataviz import TagYears, Country, GeoInclusion, TagLinks, DbpediaFields, HdaSession, InseeCoords, DatasheetExtras
 
-__all__ = ['WpCategory', 'InfoboxParameter', 'TagInfobox', 'TagWpCategory', 'TagYears', 'Country', 'GeoInclusion', 'TagLinks', 'DbpediaFields', 'HdaSession', 'InseeCoords', 'DatasheetExtras']
\ No newline at end of file
+__all__ = ['WpCategory', 'InfoboxParameter', 'TagInfobox', 'TagWpCategory', 'TagYears', 'Country', 'GeoInclusion', 'TagLinks', 'DbpediaFields', 'HdaSession', 'InseeCoords', 'DatasheetExtras', 'WpCategoryInclusion']
\ No newline at end of file

--- a/web/hdalab/models/categories.py	Fri Jun 22 19:16:46 2012 +0200
+++ b/web/hdalab/models/categories.py	Mon Jul 02 19:19:02 2012 +0200
@@ -44,3 +44,13 @@
     class Meta:
         app_label = 'hdalab'
         unique_together = ('tag_infobox','param_name')
+
+# Added on 02/07/2012
+
+class WpCategoryInclusion(models.Model):
+    parent_category = models.ForeignKey(WpCategory, related_name="child_categories")
+    child_category = models.ForeignKey(WpCategory, related_name="parent_categories")
+    
+    class Meta:
+        app_label = 'hdalab'
+        unique_together = ('parent_category','child_category')
\ No newline at end of file

--- a/web/hdalab/urls.py	Fri Jun 22 19:16:46 2012 +0200
+++ b/web/hdalab/urls.py	Mon Jul 02 19:19:02 2012 +0200
@@ -33,4 +33,5 @@
     (r'^a/taginfo$', 'taginfo', {}, 'tag_info'),
     (r'^a/sessioninfo$', 'sessioninfo', {}, 'session_info'),
     (r'^a/tagsearch$', 'tagsearch', {}, 'tag_search'),
+    (r'^a/cattree$', 'cattree', {}, 'cattree'),
 )

--- a/web/hdalab/views/ajax.py	Fri Jun 22 19:16:46 2012 +0200
+++ b/web/hdalab/views/ajax.py	Mon Jul 02 19:19:02 2012 +0200
@@ -11,6 +11,7 @@
 from hdabo.models import Tag, Datasheet, TaggedSheet
 from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras
 from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields
+from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory
 from hdalab.utils import fix_cache_key
 import copy
 import django.utils.simplejson as json
@@ -80,8 +81,82 @@
     translations = dict([(t.master.label, t.label) for t in translationqs])
     
     return HttpResponse(content=json.dumps(translations), mimetype='application/json')
+
+def subcat(category, globtags, level, max_level ):
+    # recursive function used by cattree
+    tags = Tag.objects.select_related('wp_categories__wp_category').filter(wp_categories__wp_category = category)
+    taglabels = [k for k in dict([(t.label,t.label) for t in tags])]
+    catlabel = category.label
+    resobj = {
+          'category': category.label,
+          'tags': [],
+          'contents': []
+          }
+    for label in taglabels:
+        if label == catlabel:
+            globtags[label] = {'level': level, 'access': resobj }
+        else:
+            tag_in_list = {'label' : label, 'contents': []}
+            resobj['tags'].append(tag_in_list)
+            globtags[label] = {'level': (level + 1), 'access': tag_in_list }
+            
+    if level < max_level:
+        subcats = WpCategory.objects.select_related('parent_categories__parent_category').filter(parent_categories__parent_category = category)
+        resobj['sub_categories'] = [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))]
+    return resobj
+
+def cleantags(category):
+    if category.has_key('contents') and len(category['contents']) == 0:
+        del category['contents']
+    if category.has_key('tags'):
+        category['tags'] = [tag for tag in category['tags'] if len(tag['contents'])]
+        if len(category['tags']) == 0:
+            del category['tags']
+    if category.has_key('sub_categories'):
+        sub_cats = []
+        for sub_cat in category['sub_categories']:
+            cat = cleantags(sub_cat)
+            if cat.has_key('tags') or cat.has_key('sub_categories') or cat.has_key('contents'):
+                sub_cats.append(cat)
+        category['sub_categories'] = sub_cats
+        if len(category['sub_categories']) == 0:
+            del category['sub_categories']
+    return category
+
+def cattree(request):
+    # Gets the category tree from a label
+    MAX_TAG_ORDER = 5
+    MAX_LEVEL = 3
+    label = request.GET.get('label', None)
+    globtags = {}
+    resobj = None
+    master_category = WpCategory.objects.filter(label__iexact=label)[0:1]
+    if len(master_category):
+        resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL )
     
-
+    tag_list = [k for k in globtags]
+    
+    if len(tag_list):
+        contents = []
+        datasheets = Datasheet.objects.select_related('taggedsheet__tag').filter(taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct()
+        for datasheet in datasheets:
+            # Calculating where we add the datasheet in the tree
+            maintag = None
+            maintagscore = -5
+            for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER):
+                label = ts.tag.label
+                if globtags.has_key(label):
+                    score = 3 * globtags[label]['level'] - ts.order
+                    if score > maintagscore:
+                        maintagscore = score
+                        maintag = label
+            if maintag is not None:
+                globtags[maintag]['access']['contents'].append({'id': datasheet.id, 'title': datasheet.title})
+        cleantags(resobj)
+                    
+#        resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets]
+    
+    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
 
 def sessioninfo(request):