# HG changeset patch # User ymh # Date 1373545541 -7200 # Node ID 04d3dfd72e3b6034e4fa0a603aa96134b617b25d # Parent 4f97481fa28b7c24074f7142d38a047e26066540 add import countries command diff -r 4f97481fa28b -r 04d3dfd72e3b .settings/org.eclipse.core.resources.prefs --- a/.settings/org.eclipse.core.resources.prefs Wed Jul 10 18:47:57 2013 +0200 +++ b/.settings/org.eclipse.core.resources.prefs Thu Jul 11 14:25:41 2013 +0200 @@ -19,14 +19,18 @@ encoding//src/core/wp_utils.py=utf-8 encoding//src/jocondelab/config.py=utf-8 encoding//src/jocondelab/forms.py=utf-8 +encoding//src/jocondelab/management/commands/import_countries.py=utf-8 encoding//src/jocondelab/management/commands/import_csv.py=utf-8 encoding//src/jocondelab/management/commands/import_skos.py=utf-8 encoding//src/jocondelab/management/commands/import_term_labels.py=utf-8 encoding//src/jocondelab/management/commands/import_terms.py=utf-8 encoding//src/jocondelab/management/commands/populate_thesaurus_tree.py=utf-8 encoding//src/jocondelab/migrations/0001_initial.py=utf-8 +encoding//src/jocondelab/migrations/0002_add_country_table.py=utf-8 +encoding//src/jocondelab/migrations/0003_auto_add_indexes_on_countries.py=utf-8 encoding//src/jocondelab/models.py=utf-8 encoding//src/jocondelab/utils.py=utf-8 encoding//src/jocondelab/views.py=utf-8 +encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/SPARQLWrapper/__init__.py=utf8 encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/rdflib_sqlalchemy/__init__.py=utf-8 encoding//virtualenv/web/env/venv_jocondelab/lib/python2.7/site-packages/wikitools/wiki.py=utf-8 diff -r 4f97481fa28b -r 04d3dfd72e3b src/jocondelab/management/commands/import_countries.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/jocondelab/management/commands/import_countries.py Thu Jul 11 14:25:41 2013 +0200 @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +''' +Created on Jun 11, 2013 + +@author: ymh +''' + +from SPARQLWrapper import SPARQLWrapper2 +from django.core.management.base import NoArgsCommand +from django.db import transaction, reset_queries +from jocondelab.models import Country +from optparse import make_option + + +class Command(NoArgsCommand): + + help = "Import countries from dbpedia" + + option_list = NoArgsCommand.option_list + ( + make_option('-b', '--batch-size', + dest= 'batch_size', + type='int', + default= 50, + help= 'number of object to import in bulk operations' + ), + ) + + + def handle_noargs(self, **options): + + batch_size = options.get('batch_size', 50) + + transaction.enter_transaction_management() + transaction.managed() + + endpoint = SPARQLWrapper2("http://fr.dbpedia.org/sparql") + sparql = """ + select distinct ?pays ?code where { + ?pays rdf:type dbpedia-owl:Country . + ?pays prop-fr:iso ?code + } LIMIT 300 + """ + endpoint.setQuery(sparql) + results = endpoint.query() + + for i,binding in enumerate(results.bindings): + if binding[u"code"].type == 'literal': + dbpedia_uri = binding[u"pays"].value + iso_codes = binding[u"code"].value.split(", ") + Country.objects.create( + dbpedia_uri = dbpedia_uri, + iso_code_3 = iso_codes[0], + iso_code_2 = iso_codes[1]) + + if not ((i+1) % batch_size): + transaction.commit() + reset_queries() + + # insert code for Maurice + Country.objects.create(dbpedia_uri=u'http://fr.dbpedia.org/resource/Maurice_(pays)', iso_code_3 = "MUS", iso_code_2 = "MU") + + transaction.commit() + reset_queries() + transaction.leave_transaction_management() + \ No newline at end of file diff -r 4f97481fa28b -r 04d3dfd72e3b src/jocondelab/migrations/0002_add_country_table.py --- a/src/jocondelab/migrations/0002_add_country_table.py Wed Jul 10 18:47:57 2013 +0200 +++ b/src/jocondelab/migrations/0002_add_country_table.py Thu Jul 11 14:25:41 2013 +0200 @@ -1,26 +1,24 @@ # -*- coding: utf-8 -*- -import datetime from south.db import db from south.v2 import SchemaMigration -from django.db import models class Migration(SchemaMigration): def forwards(self, orm): # Adding model 'Country' - db.create_table(u'jocondelab_country', ( + db.create_table(u'jocondelab_country', ( # @UndefinedVariable (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('dbpedia_uri', self.gf('django.db.models.fields.URLField')(unique=True, max_length=2048, db_index=True)), ('iso_code_3', self.gf('django.db.models.fields.CharField')(max_length=3)), ('iso_code_2', self.gf('django.db.models.fields.CharField')(max_length=2)), )) - db.send_create_signal(u'jocondelab', ['Country']) + db.send_create_signal(u'jocondelab', ['Country']) # @UndefinedVariable def backwards(self, orm): # Deleting model 'Country' - db.delete_table(u'jocondelab_country') + db.delete_table(u'jocondelab_country') # @UndefinedVariable models = { diff -r 4f97481fa28b -r 04d3dfd72e3b src/jocondelab/migrations/0003_auto_add_indexes_on_countries.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/jocondelab/migrations/0003_auto_add_indexes_on_countries.py Thu Jul 11 14:25:41 2013 +0200 @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding index on 'Country', fields ['iso_code_2'] + db.create_index(u'jocondelab_country', ['iso_code_2']) # @UndefinedVariable + + # Adding index on 'Country', fields ['iso_code_3'] + db.create_index(u'jocondelab_country', ['iso_code_3']) # @UndefinedVariable + + + def backwards(self, orm): + # Removing index on 'Country', fields ['iso_code_3'] + db.delete_index(u'jocondelab_country', ['iso_code_3']) # @UndefinedVariable + + # Removing index on 'Country', fields ['iso_code_2'] + db.delete_index(u'jocondelab_country', ['iso_code_2']) # @UndefinedVariable + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'jocondelab.country': { + 'Meta': {'object_name': 'Country'}, + 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'iso_code_2': ('django.db.models.fields.CharField', [], {'max_length': '2', 'db_index': 'True'}), + 'iso_code_3': ('django.db.models.fields.CharField', [], {'max_length': '3', 'db_index': 'True'}) + }, + u'jocondelab.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'language': ('django.db.models.fields.CharField', [], {'default': "'fr'", 'max_length': '2'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + } + } + + complete_apps = ['jocondelab'] \ No newline at end of file diff -r 4f97481fa28b -r 04d3dfd72e3b src/jocondelab/migrations/0003_populate_countries.py --- a/src/jocondelab/migrations/0003_populate_countries.py Wed Jul 10 18:47:57 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models -import django.utils.simplejson as json -from SPARQLWrapper import SPARQLWrapper, JSON - - -class Migration(SchemaMigration): - - def forwards(self, orm): - endpoint = SPARQLWrapper("http://fr.dbpedia.org/sparql") - endpoint.setReturnFormat(JSON) - sparql = """ - select distinct ?pays ?code where { - ?pays rdf:type dbpedia-owl:Country . - ?pays prop-fr:iso ?code - } LIMIT 300 - """ - endpoint.setQuery(sparql) - results = endpoint.query().convert() - - for r in results["results"]["bindings"]: - if r["code"]["type"] == "literal": - dbpedia_uri = r["pays"]["value"] - iso_codes = r["code"]["value"].split(", ") - orm.Country.objects.create( - dbpedia_uri = dbpedia_uri, - iso_code_3 = iso_codes[0], - iso_code_2 = iso_codes[1]) - - - def backwards(self, orm): - db.clear_table(u'jocondelab_country') - - models = { - u'auth.group': { - 'Meta': {'object_name': 'Group'}, - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) - }, - u'auth.permission': { - 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - u'contenttypes.contenttype': { - 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - u'jocondelab.country': { - 'Meta': {'object_name': 'Country'}, - 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'iso_code_3': ('django.db.models.fields.CharField', [], {'max_length': '3'}), - 'iso_code_2': ('django.db.models.fields.CharField', [], {'max_length': '2'}) - }, - u'jocondelab.user': { - 'Meta': {'object_name': 'User'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '2'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - } - } - - complete_apps = ['jocondelab'] \ No newline at end of file diff -r 4f97481fa28b -r 04d3dfd72e3b src/jocondelab/models.py --- a/src/jocondelab/models.py Wed Jul 10 18:47:57 2013 +0200 +++ b/src/jocondelab/models.py Thu Jul 11 14:25:41 2013 +0200 @@ -14,6 +14,6 @@ class Country(models.Model): dbpedia_uri = models.URLField(max_length=2048, unique=True, blank=False, null=False, db_index=True) - iso_code_3 = models.CharField(max_length=3, unique=False, blank=False, null=False, db_index=False) - iso_code_2 = models.CharField(max_length=2, unique=False, blank=False, null=False, db_index=False) + iso_code_3 = models.CharField(max_length=3, unique=False, blank=False, null=False, db_index=True) + iso_code_2 = models.CharField(max_length=2, unique=False, blank=False, null=False, db_index=True) \ No newline at end of file