--- a/.settings/org.eclipse.core.resources.prefs Wed Jul 10 18:47:57 2013 +0200
+++ b/.settings/org.eclipse.core.resources.prefs Thu Jul 11 14:25:41 2013 +0200
@@ -19,14 +19,18 @@
encoding//src/core/wp_utils.py=utf-8
encoding//src/jocondelab/config.py=utf-8
encoding//src/jocondelab/forms.py=utf-8
+encoding//src/jocondelab/management/commands/import_countries.py=utf-8
encoding//src/jocondelab/management/commands/import_csv.py=utf-8
encoding//src/jocondelab/management/commands/import_skos.py=utf-8
encoding//src/jocondelab/management/commands/import_term_labels.py=utf-8
encoding//src/jocondelab/management/commands/import_terms.py=utf-8
encoding//src/jocondelab/management/commands/populate_thesaurus_tree.py=utf-8
encoding//src/jocondelab/migrations/0001_initial.py=utf-8
+encoding//src/jocondelab/migrations/0002_add_country_table.py=utf-8
+encoding//src/jocondelab/migrations/0003_auto_add_indexes_on_countries.py=utf-8
encoding//src/jocondelab/models.py=utf-8
encoding//src/jocondelab/utils.py=utf-8
encoding//src/jocondelab/views.py=utf-8
+encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/SPARQLWrapper/__init__.py=utf8
encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/rdflib_sqlalchemy/__init__.py=utf-8
encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/wikitools/wiki.py=utf-8
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/jocondelab/management/commands/import_countries.py Thu Jul 11 14:25:41 2013 +0200
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+'''
+Created on Jun 11, 2013
+
+@author: ymh
+'''
+
+from SPARQLWrapper import SPARQLWrapper2
+from django.core.management.base import NoArgsCommand
+from django.db import transaction, reset_queries
+from jocondelab.models import Country
+from optparse import make_option
+
+
+class Command(NoArgsCommand):
+
+ help = "Import countries from dbpedia"
+
+ option_list = NoArgsCommand.option_list + (
+ make_option('-b', '--batch-size',
+ dest= 'batch_size',
+ type='int',
+ default= 50,
+ help= 'number of object to import in bulk operations'
+ ),
+ )
+
+
+ def handle_noargs(self, **options):
+
+ batch_size = options.get('batch_size', 50)
+
+ transaction.enter_transaction_management()
+ transaction.managed()
+
+ endpoint = SPARQLWrapper2("http://fr.dbpedia.org/sparql")
+ sparql = """
+ select distinct ?pays ?code where {
+ ?pays rdf:type dbpedia-owl:Country .
+ ?pays prop-fr:iso ?code
+ } LIMIT 300
+ """
+ endpoint.setQuery(sparql)
+ results = endpoint.query()
+
+ for i,binding in enumerate(results.bindings):
+ if binding[u"code"].type == 'literal':
+ dbpedia_uri = binding[u"pays"].value
+ iso_codes = binding[u"code"].value.split(", ")
+ Country.objects.create(
+ dbpedia_uri = dbpedia_uri,
+ iso_code_3 = iso_codes[0],
+ iso_code_2 = iso_codes[1])
+
+ if not ((i+1) % batch_size):
+ transaction.commit()
+ reset_queries()
+
+ # insert code for Maurice
+ Country.objects.create(dbpedia_uri=u'http://fr.dbpedia.org/resource/Maurice_(pays)', iso_code_3 = "MUS", iso_code_2 = "MU")
+
+ transaction.commit()
+ reset_queries()
+ transaction.leave_transaction_management()
+
\ No newline at end of file
--- a/src/jocondelab/migrations/0002_add_country_table.py Wed Jul 10 18:47:57 2013 +0200
+++ b/src/jocondelab/migrations/0002_add_country_table.py Thu Jul 11 14:25:41 2013 +0200
@@ -1,26 +1,24 @@
# -*- coding: utf-8 -*-
-import datetime
from south.db import db
from south.v2 import SchemaMigration
-from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'Country'
- db.create_table(u'jocondelab_country', (
+ db.create_table(u'jocondelab_country', ( # @UndefinedVariable
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('dbpedia_uri', self.gf('django.db.models.fields.URLField')(unique=True, max_length=2048, db_index=True)),
('iso_code_3', self.gf('django.db.models.fields.CharField')(max_length=3)),
('iso_code_2', self.gf('django.db.models.fields.CharField')(max_length=2)),
))
- db.send_create_signal(u'jocondelab', ['Country'])
+ db.send_create_signal(u'jocondelab', ['Country']) # @UndefinedVariable
def backwards(self, orm):
# Deleting model 'Country'
- db.delete_table(u'jocondelab_country')
+ db.delete_table(u'jocondelab_country') # @UndefinedVariable
models = {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/jocondelab/migrations/0003_auto_add_indexes_on_countries.py Thu Jul 11 14:25:41 2013 +0200
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding index on 'Country', fields ['iso_code_2']
+ db.create_index(u'jocondelab_country', ['iso_code_2']) # @UndefinedVariable
+
+ # Adding index on 'Country', fields ['iso_code_3']
+ db.create_index(u'jocondelab_country', ['iso_code_3']) # @UndefinedVariable
+
+
+ def backwards(self, orm):
+ # Removing index on 'Country', fields ['iso_code_3']
+ db.delete_index(u'jocondelab_country', ['iso_code_3']) # @UndefinedVariable
+
+ # Removing index on 'Country', fields ['iso_code_2']
+ db.delete_index(u'jocondelab_country', ['iso_code_2']) # @UndefinedVariable
+
+
+ models = {
+ u'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ u'auth.permission': {
+ 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ u'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ u'jocondelab.country': {
+ 'Meta': {'object_name': 'Country'},
+ 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'iso_code_2': ('django.db.models.fields.CharField', [], {'max_length': '2', 'db_index': 'True'}),
+ 'iso_code_3': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'})
+ },
+ u'jocondelab.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'language': ('django.db.models.fields.CharField', [], {'default': "'fr'", 'max_length': '2'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ }
+ }
+
+ complete_apps = ['jocondelab']
\ No newline at end of file
--- a/src/jocondelab/migrations/0003_populate_countries.py Wed Jul 10 18:47:57 2013 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,84 +0,0 @@
-# -*- coding: utf-8 -*-
-import datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
-import django.utils.simplejson as json
-from SPARQLWrapper import SPARQLWrapper, JSON
-
-
-class Migration(SchemaMigration):
-
- def forwards(self, orm):
- endpoint = SPARQLWrapper("http://fr.dbpedia.org/sparql")
- endpoint.setReturnFormat(JSON)
- sparql = """
- select distinct ?pays ?code where {
- ?pays rdf:type dbpedia-owl:Country .
- ?pays prop-fr:iso ?code
- } LIMIT 300
- """
- endpoint.setQuery(sparql)
- results = endpoint.query().convert()
-
- for r in results["results"]["bindings"]:
- if r["code"]["type"] == "literal":
- dbpedia_uri = r["pays"]["value"]
- iso_codes = r["code"]["value"].split(", ")
- orm.Country.objects.create(
- dbpedia_uri = dbpedia_uri,
- iso_code_3 = iso_codes[0],
- iso_code_2 = iso_codes[1])
-
-
- def backwards(self, orm):
- db.clear_table(u'jocondelab_country')
-
- models = {
- u'auth.group': {
- 'Meta': {'object_name': 'Group'},
- u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
- 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
- },
- u'auth.permission': {
- 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
- 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
- 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
- u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
- },
- u'contenttypes.contenttype': {
- 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
- 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
- u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
- 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
- },
- u'jocondelab.country': {
- 'Meta': {'object_name': 'Country'},
- 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}),
- u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'iso_code_3': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
- 'iso_code_2': ('django.db.models.fields.CharField', [], {'max_length': '2'})
- },
- u'jocondelab.user': {
- 'Meta': {'object_name': 'User'},
- 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
- 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
- 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
- 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
- u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
- 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
- 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
- 'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '2'}),
- 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
- 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
- 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
- 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
- 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
- }
- }
-
- complete_apps = ['jocondelab']
\ No newline at end of file
--- a/src/jocondelab/models.py Wed Jul 10 18:47:57 2013 +0200
+++ b/src/jocondelab/models.py Thu Jul 11 14:25:41 2013 +0200
@@ -14,6 +14,6 @@
class Country(models.Model):
dbpedia_uri = models.URLField(max_length=2048, unique=True, blank=False, null=False, db_index=True)
- iso_code_3 = models.CharField(max_length=3, unique=False, blank=False, null=False, db_index=False)
- iso_code_2 = models.CharField(max_length=2, unique=False, blank=False, null=False, db_index=False)
+ iso_code_3 = models.CharField(max_length=3, unique=False, blank=False, null=False, db_index=True)
+ iso_code_2 = models.CharField(max_length=2, unique=False, blank=False, null=False, db_index=True)
\ No newline at end of file