# HG changeset patch # User veltr # Date 1341249542 -7200 # Node ID 0a1744477bc1d7fc5e83647e6ef86c6c962c1b1f # Parent 00fc169cc6a9f037d3c72f6b505be7c99cfa7060 Added category tree Ajax API + Tables diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/management/commands/query_category_inclusion.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/web/hdalab/management/commands/query_category_inclusion.py Mon Jul 02 19:19:02 2012 +0200 @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +''' +Created on July 2, 2012 + +@author: raphv +''' + +from django.conf import settings +from django.core.management.base import NoArgsCommand +from django.core.management.color import no_style +from hdalab.models import WpCategory, WpCategoryInclusion +from optparse import make_option +from wikitools import api,wiki +import sys +import re +import itertools +from hdabo import utils +from django.db.models import Count +from django.db import transaction + +CATEGORY_PREFIX = u'Catégorie:' + +class Command(NoArgsCommand): + ''' + query and update wikipedia for tag title. + ''' + options = '' + help = """query and update wikipedia for tag title.""" + + option_list = NoArgsCommand.option_list + ( + make_option('--all', + action='store_true', + dest='all', + default=False, + help='force all categories to be updated, not only those not yet processed'), + make_option('--force', + action='store_true', + dest='force', + default=False, + help='ask no questions'), + make_option('--random', + action='store_true', + dest='random', + default=False, + help='randomize query on categories'), + make_option('--site', + action='store', + type='string', + dest='site_url', + default="http://fr.wikipedia.org/w/api.php", + help='the url for the wikipedia site'), + make_option('--limit', + action='store', + type='int', + dest='limit', + default= -1, + help='number of categories to process'), + make_option('--start', + action='store', + type='int', + dest='start', + default=0, + help='number of categories to ignore'), + make_option('--category', + action='append', + dest='category', + type='string', + default=[], + help='the categories to query'), + + ) + + + def query_all_categories(self, category_title, site): + + params = {'action':'query', 'cmtitle':category_title, 'list':'categorymembers', 'cmlimit': 'max'} + + res = [] + + wpquery = api.APIRequest(site, params) #@UndefinedVariable + response = wpquery.query() + + if self.verbosity > 1: + print "Query category : " + repr(wpquery.request.get_full_url()+"?"+wpquery.request.get_data()) + print repr(response) + + members = response.get('query', {}).get('categorymembers', []) + + for member in members: + title = member.get('title',"") + if re.match(CATEGORY_PREFIX, title): + res.append(re.sub(CATEGORY_PREFIX, "", title)) + + if self.verbosity > 1: + print "Query categories result: " + print repr(res) + + return res + + def process_categories(self, cat_list, parent_cat): + for cat in cat_list: + child_cat,created = WpCategory.objects.get_or_create(label=cat) #@UnusedVariable + WpCategoryInclusion.objects.get_or_create(parent_category=parent_cat, child_category=child_cat) + + def handle_noargs(self, **options): + + self.style = no_style() + + interactive = options.get('interactive', True) + + self.verbosity = int(options.get('verbosity', '1')) + + force = options.get('force', False) + + limit = options.get("limit", -1) + start = options.get("start", 0) + + site_url = options.get('site_url', settings.WIKIPEDIA_API_URL) + + random = options.get('random', False) + + types_mask = 0 + + if self.verbosity > 2: + print "option passed : " + repr(options) + + queryset = WpCategory.objects + + cat_list = options.get("category", []); + + if cat_list: + queryset = queryset.filter(label__in=cat_list) + elif not options.get('all',False): + queryset = queryset.annotate(wpc=Count('child_categories')).filter(wpc = 0) + #else: + # queryset = Tag.objects.filter(url_status=None) + + if random: + queryset = queryset.order_by("?") + else: + queryset = queryset.order_by("label") + + if limit >= 0: + queryset = queryset[start:limit] + elif start > 0: + queryset = queryset[start:] + + if self.verbosity > 2 : + print "Category Query is %s" % (queryset.query) + + site = wiki.Wiki(site_url) #@UndefinedVariable + + + count = queryset.count() + if self.verbosity > 1: + print "Processing %d categories" % (count) + + if not force and interactive: + confirm = raw_input("You have requested to query and replace the wikipedia information for %d categories.\n Are you sure you want to do this? \nType 'yes' to continue, or 'no' to cancel: " % (count)) + else: + confirm = 'yes' + + if confirm != "yes": + print "wikipedia query cancelled" + return + + for i, category in enumerate(queryset): + + if self.verbosity > 1: + print "processing category %s (%d/%d)" % (category.label, i + 1, count) + else: + utils.show_progress(i + 1, count, category.label, 60) + + title = CATEGORY_PREFIX + category.label + # query categories + + with transaction.commit_on_success(): + res = self.query_all_categories(title, site) + self.process_categories(res, category) \ No newline at end of file diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/migrations/0010_categoryinclusion.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/web/hdalab/migrations/0010_categoryinclusion.py Mon Jul 02 19:19:02 2012 +0200 @@ -0,0 +1,312 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'WpCategoryInclusion' + db.create_table('hdalab_wpcategoryinclusion', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('parent_category', self.gf('django.db.models.fields.related.ForeignKey')(related_name='child_categories', to=orm['hdalab.WpCategory'])), + ('child_category', self.gf('django.db.models.fields.related.ForeignKey')(related_name='parent_categories', to=orm['hdalab.WpCategory'])), + )) + db.send_create_signal('hdalab', ['WpCategoryInclusion']) + + # Adding unique constraint on 'WpCategoryInclusion', fields ['parent_category', 'child_category'] + db.create_unique('hdalab_wpcategoryinclusion', ['parent_category_id', 'child_category_id']) + + + def backwards(self, orm): + + # Removing unique constraint on 'WpCategoryInclusion', fields ['parent_category', 'child_category'] + db.delete_unique('hdalab_wpcategoryinclusion', ['parent_category_id', 'child_category_id']) + + # Deleting model 'WpCategoryInclusion' + db.delete_table('hdalab_wpcategoryinclusion') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 2, 5, 14, 31, 895549)'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 7, 2, 5, 14, 31, 895384)'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'hdabo.author': { + 'Meta': {'object_name': 'Author'}, + 'firstname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}), + 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lastname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}) + }, + 'hdabo.datasheet': { + 'Meta': {'object_name': 'Datasheet'}, + 'author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Author']", 'null': 'True', 'blank': 'True'}), + 'college_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_periods']", 'to': "orm['hdabo.TimePeriod']"}), + 'college_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_themes']", 'to': "orm['hdabo.Domain']"}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'domains': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_domains']", 'to': "orm['hdabo.Domain']"}), + 'format': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.DocumentFormat']", 'null': 'True', 'blank': 'True'}), + 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}), + 'highschool_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_periods']", 'to': "orm['hdabo.TimePeriod']"}), + 'highschool_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_themes']", 'to': "orm['hdabo.Domain']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'manual_order': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'modification_datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'organisation': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Organisation']", 'null': 'True'}), + 'original_creation_date': ('django.db.models.fields.DateField', [], {}), + 'original_modification_date': ('django.db.models.fields.DateField', [], {}), + 'primary_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_periods']", 'to': "orm['hdabo.TimePeriod']"}), + 'primary_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_themes']", 'to': "orm['hdabo.Domain']"}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['hdabo.Tag']", 'through': "orm['hdabo.TaggedSheet']", 'symmetrical': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '2048'}), + 'town': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Location']", 'null': 'True', 'blank': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'validated': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'validation_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'validator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) + }, + 'hdabo.datasheet_college_periods': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_periods'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}), + 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"}) + }, + 'hdabo.datasheet_college_themes': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_themes'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdabo.datasheet_domains': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_domains'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdabo.datasheet_highschool_periods': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_periods'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}), + 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"}) + }, + 'hdabo.datasheet_highschool_themes': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_themes'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdabo.datasheet_primary_periods': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_periods'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}), + 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"}) + }, + 'hdabo.datasheet_primary_themes': { + 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_themes'}, + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sort_value': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdabo.documentformat': { + 'Meta': {'object_name': 'DocumentFormat'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}) + }, + 'hdabo.domain': { + 'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'Domain'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'school_period': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdabo.location': { + 'Meta': {'object_name': 'Location'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '5'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'hdabo.organisation': { + 'Meta': {'object_name': 'Organisation'}, + 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}) + }, + 'hdabo.tag': { + 'Meta': {'unique_together': "(('label', 'original_label', 'url_status'),)", 'object_name': 'Tag'}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'alternative_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'alternative_wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'alternative_wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TagCategory']", 'null': 'True', 'blank': 'True'}), + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}), + 'normalized_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}), + 'original_label': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'popularity': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'url_status': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'null': 'True', 'db_index': 'True', 'blank': 'True'}), + 'wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}) + }, + 'hdabo.tagcategory': { + 'Meta': {'object_name': 'TagCategory'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}) + }, + 'hdabo.taggedsheet': { + 'Meta': {'object_name': 'TaggedSheet'}, + 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'index_note': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'db_index': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'original_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Tag']"}), + 'wikipedia_revision_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'hdabo.timeperiod': { + 'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'TimePeriod'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'school_period': ('django.db.models.fields.IntegerField', [], {}) + }, + 'hdalab.country': { + 'Meta': {'object_name': 'Country'}, + 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'hdalab.datasheetextras': { + 'Meta': {'object_name': 'DatasheetExtras'}, + 'datasheet': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'extras'", 'unique': 'True', 'to': "orm['hdabo.Datasheet']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'insee': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.InseeCoords']", 'null': 'True', 'blank': 'True'}) + }, + 'hdalab.dbpediafields': { + 'Meta': {'object_name': 'DbpediaFields'}, + 'abstract': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}), + 'tag': ('django.db.models.fields.related.OneToOneField', [], {'blank': 'True', 'related_name': "'dbpedia_fields'", 'unique': 'True', 'null': 'True', 'to': "orm['hdabo.Tag']"}), + 'thumbnail': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}) + }, + 'hdalab.dbpediafieldstranslation': { + 'Meta': {'unique_together': "(('master', 'language_code'),)", 'object_name': 'DbpediaFieldsTranslation'}, + 'abstract': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}), + 'language_code': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}), + 'master': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'translations'", 'to': "orm['hdalab.DbpediaFields']"}) + }, + 'hdalab.geoinclusion': { + 'Meta': {'object_name': 'GeoInclusion'}, + 'country': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'includes'", 'to': "orm['hdalab.Country']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'tag': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'locatedin'", 'unique': 'True', 'to': "orm['hdabo.Tag']"}) + }, + 'hdalab.hdasession': { + 'Meta': {'object_name': 'HdaSession'}, + 'data': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'sessionid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36', 'db_index': 'True'}) + }, + 'hdalab.infoboxparameter': { + 'Meta': {'unique_together': "(('tag_infobox', 'param_name'),)", 'object_name': 'InfoboxParameter'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'param_name': ('django.db.models.fields.CharField', [], {'max_length': '2048'}), + 'param_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'tag_infobox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.TagInfobox']"}) + }, + 'hdalab.inseecoords': { + 'Meta': {'object_name': 'InseeCoords'}, + 'city_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'insee': ('django.db.models.fields.IntegerField', [], {'primary_key': 'True'}), + 'latitude': ('django.db.models.fields.FloatField', [], {}), + 'longitude': ('django.db.models.fields.FloatField', [], {}) + }, + 'hdalab.taginfobox': { + 'Meta': {'unique_together': "(('tag', 'name', 'revision_id'),)", 'object_name': 'TagInfobox'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '2048'}), + 'revision_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'infoboxes'", 'to': "orm['hdabo.Tag']"}) + }, + 'hdalab.taglinks': { + 'Meta': {'object_name': 'TagLinks'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taglinks_objects'", 'to': "orm['hdabo.Tag']"}), + 'subject': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taglinks_subjects'", 'to': "orm['hdabo.Tag']"}) + }, + 'hdalab.tagwpcategory': { + 'Meta': {'unique_together': "(('tag', 'wp_category', 'hidden'),)", 'object_name': 'TagWpCategory'}, + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'wp_categories'", 'to': "orm['hdabo.Tag']"}), + 'wp_category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdalab.WpCategory']"}) + }, + 'hdalab.tagyears': { + 'Meta': {'object_name': 'TagYears'}, + 'end_year': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_year': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'years'", 'to': "orm['hdabo.Tag']"}) + }, + 'hdalab.wpcategory': { + 'Meta': {'object_name': 'WpCategory'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048'}) + }, + 'hdalab.wpcategoryinclusion': { + 'Meta': {'unique_together': "(('parent_category', 'child_category'),)", 'object_name': 'WpCategoryInclusion'}, + 'child_category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parent_categories'", 'to': "orm['hdalab.WpCategory']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parent_category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'child_categories'", 'to': "orm['hdalab.WpCategory']"}) + } + } + + complete_apps = ['hdalab'] diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/models/__init__.py --- a/web/hdalab/models/__init__.py Fri Jun 22 19:16:46 2012 +0200 +++ b/web/hdalab/models/__init__.py Mon Jul 02 19:19:02 2012 +0200 @@ -1,5 +1,5 @@ from hdalab.models.categories import (WpCategory, InfoboxParameter, TagInfobox, - TagWpCategory) + TagWpCategory, WpCategoryInclusion) from hdalab.models.dataviz import TagYears, Country, GeoInclusion, TagLinks, DbpediaFields, HdaSession, InseeCoords, DatasheetExtras -__all__ = ['WpCategory', 'InfoboxParameter', 'TagInfobox', 'TagWpCategory', 'TagYears', 'Country', 'GeoInclusion', 'TagLinks', 'DbpediaFields', 'HdaSession', 'InseeCoords', 'DatasheetExtras'] \ No newline at end of file +__all__ = ['WpCategory', 'InfoboxParameter', 'TagInfobox', 'TagWpCategory', 'TagYears', 'Country', 'GeoInclusion', 'TagLinks', 'DbpediaFields', 'HdaSession', 'InseeCoords', 'DatasheetExtras', 'WpCategoryInclusion'] \ No newline at end of file diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/models/categories.py --- a/web/hdalab/models/categories.py Fri Jun 22 19:16:46 2012 +0200 +++ b/web/hdalab/models/categories.py Mon Jul 02 19:19:02 2012 +0200 @@ -44,3 +44,13 @@ class Meta: app_label = 'hdalab' unique_together = ('tag_infobox','param_name') + +# Added on 02/07/2012 + +class WpCategoryInclusion(models.Model): + parent_category = models.ForeignKey(WpCategory, related_name="child_categories") + child_category = models.ForeignKey(WpCategory, related_name="parent_categories") + + class Meta: + app_label = 'hdalab' + unique_together = ('parent_category','child_category') \ No newline at end of file diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/urls.py --- a/web/hdalab/urls.py Fri Jun 22 19:16:46 2012 +0200 +++ b/web/hdalab/urls.py Mon Jul 02 19:19:02 2012 +0200 @@ -33,4 +33,5 @@ (r'^a/taginfo$', 'taginfo', {}, 'tag_info'), (r'^a/sessioninfo$', 'sessioninfo', {}, 'session_info'), (r'^a/tagsearch$', 'tagsearch', {}, 'tag_search'), + (r'^a/cattree$', 'cattree', {}, 'cattree'), ) diff -r 00fc169cc6a9 -r 0a1744477bc1 web/hdalab/views/ajax.py --- a/web/hdalab/views/ajax.py Fri Jun 22 19:16:46 2012 +0200 +++ b/web/hdalab/views/ajax.py Mon Jul 02 19:19:02 2012 +0200 @@ -11,6 +11,7 @@ from hdabo.models import Tag, Datasheet, TaggedSheet from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields +from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory from hdalab.utils import fix_cache_key import copy import django.utils.simplejson as json @@ -80,8 +81,82 @@ translations = dict([(t.master.label, t.label) for t in translationqs]) return HttpResponse(content=json.dumps(translations), mimetype='application/json') + +def subcat(category, globtags, level, max_level ): + # recursive function used by cattree + tags = Tag.objects.select_related('wp_categories__wp_category').filter(wp_categories__wp_category = category) + taglabels = [k for k in dict([(t.label,t.label) for t in tags])] + catlabel = category.label + resobj = { + 'category': category.label, + 'tags': [], + 'contents': [] + } + for label in taglabels: + if label == catlabel: + globtags[label] = {'level': level, 'access': resobj } + else: + tag_in_list = {'label' : label, 'contents': []} + resobj['tags'].append(tag_in_list) + globtags[label] = {'level': (level + 1), 'access': tag_in_list } + + if level < max_level: + subcats = WpCategory.objects.select_related('parent_categories__parent_category').filter(parent_categories__parent_category = category) + resobj['sub_categories'] = [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))] + return resobj + +def cleantags(category): + if category.has_key('contents') and len(category['contents']) == 0: + del category['contents'] + if category.has_key('tags'): + category['tags'] = [tag for tag in category['tags'] if len(tag['contents'])] + if len(category['tags']) == 0: + del category['tags'] + if category.has_key('sub_categories'): + sub_cats = [] + for sub_cat in category['sub_categories']: + cat = cleantags(sub_cat) + if cat.has_key('tags') or cat.has_key('sub_categories') or cat.has_key('contents'): + sub_cats.append(cat) + category['sub_categories'] = sub_cats + if len(category['sub_categories']) == 0: + del category['sub_categories'] + return category + +def cattree(request): + # Gets the category tree from a label + MAX_TAG_ORDER = 5 + MAX_LEVEL = 3 + label = request.GET.get('label', None) + globtags = {} + resobj = None + master_category = WpCategory.objects.filter(label__iexact=label)[0:1] + if len(master_category): + resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL ) - + tag_list = [k for k in globtags] + + if len(tag_list): + contents = [] + datasheets = Datasheet.objects.select_related('taggedsheet__tag').filter(taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct() + for datasheet in datasheets: + # Calculating where we add the datasheet in the tree + maintag = None + maintagscore = -5 + for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER): + label = ts.tag.label + if globtags.has_key(label): + score = 3 * globtags[label]['level'] - ts.order + if score > maintagscore: + maintagscore = score + maintag = label + if maintag is not None: + globtags[maintag]['access']['contents'].append({'id': datasheet.id, 'title': datasheet.title}) + cleantags(resobj) + +# resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets] + + return HttpResponse(content=json.dumps(resobj), mimetype='application/json') def sessioninfo(request):