--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/hdabo/management/commands/diff_csv.py Fri Jun 22 14:13:23 2012 +0200
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+'''
+Created on May 25, 2011
+
+@author: ymh
+'''
+#Auteur,Chemin,Comment,Controle,Datcre,Datmaj,Desc,Domaine,Format,ID,Insee,Org,Org_Home,OrgID,Periode1,Periode2,Periode3,Satut,Sousdom,Tag,Theme2,Theme3,Titre,Url,Vignette,Ville
+#"Auteur","Chemin","Comment","Controle","Datcre","Datmaj","Desc","Domaine","Format","ID","Insee","Org","Org_Home","OrgID","Periode1","Periode2","Periode3","Satut","Sousdom","Tag","Theme2","Theme3","Titre","Url","Vignette","Ville",
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from hdabo.models import Datasheet
+from optparse import make_option
+import csv
+import math
+import sys
+
+class Command(BaseCommand):
+ '''
+ Command to diff datasheets content from csv content
+ '''
+ args = '<path_to_csv_file path_to_csv_file ...>'
+ options = '[--do-delete] [--encoding] [--delimiter] [--dialect]'
+ help = """Import of a csv file for hdabo
+Options:
+ --do-delete : ignore existing datasheets
+ --encoding : files encoding. default to latin-1
+ --delimiter : scv delimiter
+ --dialect : csv dialect
+ --fieldnames : csv columns
+ """
+
+ option_list = BaseCommand.option_list + (
+ make_option('--encoding',
+ action='store',
+ type='string',
+ dest='encoding',
+ default="latin-1",
+ help='fix the file encoding. default to latin-1'),
+ make_option('--delimiter',
+ action='store',
+ type='string',
+ dest='delimiter',
+ default=";",
+ help='csv file delimiter'),
+ make_option('--dialect',
+ action='store',
+ type='string',
+ dest='dialect',
+ default="excel",
+ help='csv dialect'),
+ make_option('--fieldnames',
+ action='store',
+ type='string',
+ dest='fieldnames',
+ default=None,
+ help='fields list (comma separated)'),
+ make_option('--do-delete',
+ action='store_true',
+ dest='do_delete',
+ default=False,
+ help='delete datasheets'),
+
+ )
+
+ def show_progress(self, current_line, total_line, width):
+
+ percent = (float(current_line) / float(total_line)) * 100.0
+
+ marks = math.floor(width * (percent / 100.0))
+ spaces = math.floor(width - marks)
+
+ loader = '[' + ('=' * int(marks)) + (' ' * int(spaces)) + ']'
+
+ sys.stdout.write("%s %d%% %d/%d\r" % (loader, percent, current_line - 1, total_line - 1)) #takes the header into account
+ if percent >= 100:
+ sys.stdout.write("\n")
+ sys.stdout.flush()
+
+
+
+ def handle(self, *args, **options):
+
+ if len(args) == 0:
+ raise CommandError("Gives at lat one csv file to import")
+
+ self.encoding = options.get('encoding', "latin-1")
+ self.do_delete = options.get('do_delete', False)
+ fieldnames = options.get('fieldnames', None)
+
+ for csv_path in args:
+ print "Processing %s " % (csv_path)
+ with open(csv_path, 'rU') as csv_file:
+
+ # get the number of lines if necessary
+ for i, l in enumerate(csv_file): #@UnusedVariable
+ pass
+ total_line = i + 1
+ if fieldnames:
+ total_line = total_line + 1
+ csv_file.seek(0)
+
+ delimiter = options.get('delimiter', ";")
+ if delimiter == "TAB" or delimiter == "\\t":
+ delimiter = '\t'
+
+ dr_kwargs = {'delimiter':delimiter}
+ if fieldnames is not None:
+ dr_kwargs['fieldnames'] = [f.strip() for f in fieldnames.split(",")]
+ dialect = options.get('dialect', "excel")
+ if dialect is not None:
+ dr_kwargs['dialect'] = dialect
+
+ reader = csv.DictReader(csv_file, **dr_kwargs)
+
+ ids = []
+
+ for row in reader:
+ line_num = reader.line_num if fieldnames is None else reader.line_num + 1
+ self.show_progress(line_num, total_line, 60)
+ def safe_decode(val, encoding):
+ if val:
+ return val.decode(encoding)
+ else:
+ return val
+
+ row = dict([(safe_decode(key, self.encoding), safe_decode(value, self.encoding)) for key, value in row.items()])
+
+ ids.append(row['ID'])
+
+ qs = Datasheet.objects.exclude(hda_id__in = ids).order_by("hda_id")
+
+ qs_count = qs.count()
+
+ if qs_count == 0:
+ print("No datasheet to delete : exit")
+ return
+
+ print("The following datasheets are in the database and not in the csv file")
+ for i,ds in enumerate(qs):
+ print("%*d- %4s : %s" % (len(str(qs_count+1)), i+1, ds.hda_id, ds.title.strip() if ds.title is not None else ""))
+
+
+ if self.do_delete:
+ print("deleting datasheets")
+ qs.delete()
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/hdabo/migrations/0005_auto__chg_field_datasheet_organisation.py Fri Jun 22 14:13:23 2012 +0200
@@ -0,0 +1,208 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Changing field 'Datasheet.organisation'
+ db.alter_column('hdabo_datasheet', 'organisation_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['hdabo.Organisation'], null=True))
+
+
+ def backwards(self, orm):
+
+ # User chose to not deal with backwards NULL issues for 'Datasheet.organisation'
+ raise RuntimeError("Cannot reverse this migration. 'Datasheet.organisation' and its values cannot be restored.")
+
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 6, 21, 18, 18, 22, 503161)'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 6, 21, 18, 18, 22, 503032)'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'hdabo.author': {
+ 'Meta': {'object_name': 'Author'},
+ 'firstname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+ 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'lastname': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'})
+ },
+ 'hdabo.datasheet': {
+ 'Meta': {'object_name': 'Datasheet'},
+ 'author': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Author']", 'null': 'True', 'blank': 'True'}),
+ 'college_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+ 'college_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'college_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_college_themes']", 'to': "orm['hdabo.Domain']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+ 'domains': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_domains']", 'to': "orm['hdabo.Domain']"}),
+ 'format': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.DocumentFormat']", 'null': 'True', 'blank': 'True'}),
+ 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+ 'highschool_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+ 'highschool_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'highschool_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_highschool_themes']", 'to': "orm['hdabo.Domain']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'manual_order': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'modification_datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'organisation': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Organisation']", 'null': 'True'}),
+ 'original_creation_date': ('django.db.models.fields.DateField', [], {}),
+ 'original_modification_date': ('django.db.models.fields.DateField', [], {}),
+ 'primary_periods': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_periods_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_periods']", 'to': "orm['hdabo.TimePeriod']"}),
+ 'primary_themes': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'primary_themes_datasheets'", 'symmetrical': 'False', 'through': "orm['hdabo.Datasheet_primary_themes']", 'to': "orm['hdabo.Domain']"}),
+ 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['hdabo.Tag']", 'through': "orm['hdabo.TaggedSheet']", 'symmetrical': 'False'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+ 'town': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Location']", 'null': 'True', 'blank': 'True'}),
+ 'url': ('django.db.models.fields.URLField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'validated': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'validation_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'validator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+ },
+ 'hdabo.datasheet_college_periods': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_periods'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+ 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+ },
+ 'hdabo.datasheet_college_themes': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_college_themes'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'hdabo.datasheet_domains': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_domains'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'hdabo.datasheet_highschool_periods': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_periods'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+ 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+ },
+ 'hdabo.datasheet_highschool_themes': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_highschool_themes'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'hdabo.datasheet_primary_periods': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_periods'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {}),
+ 'timeperiod': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TimePeriod']"})
+ },
+ 'hdabo.datasheet_primary_themes': {
+ 'Meta': {'ordering': "['sort_value']", 'object_name': 'Datasheet_primary_themes'},
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Domain']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'sort_value': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'hdabo.documentformat': {
+ 'Meta': {'object_name': 'DocumentFormat'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'})
+ },
+ 'hdabo.domain': {
+ 'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'Domain'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+ 'school_period': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'hdabo.location': {
+ 'Meta': {'object_name': 'Location'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '5'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'})
+ },
+ 'hdabo.organisation': {
+ 'Meta': {'object_name': 'Organisation'},
+ 'hda_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'location': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+ 'website': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'})
+ },
+ 'hdabo.tag': {
+ 'Meta': {'unique_together': "(('label', 'original_label', 'url_status'),)", 'object_name': 'Tag'},
+ 'alias': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'alternative_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'alternative_wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+ 'alternative_wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.TagCategory']", 'null': 'True', 'blank': 'True'}),
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'dbpedia_uri': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}),
+ 'normalized_label': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'db_index': 'True'}),
+ 'original_label': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
+ 'popularity': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+ 'url_status': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'null': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'wikipedia_pageid': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+ 'wikipedia_url': ('django.db.models.fields.URLField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'})
+ },
+ 'hdabo.tagcategory': {
+ 'Meta': {'object_name': 'TagCategory'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '512'})
+ },
+ 'hdabo.taggedsheet': {
+ 'Meta': {'object_name': 'TaggedSheet'},
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'datasheet': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Datasheet']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_note': ('django.db.models.fields.FloatField', [], {'default': '0.0', 'db_index': 'True'}),
+ 'order': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+ 'original_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'tag': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['hdabo.Tag']"}),
+ 'wikipedia_revision_id': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'})
+ },
+ 'hdabo.timeperiod': {
+ 'Meta': {'unique_together': "(('label', 'school_period'),)", 'object_name': 'TimePeriod'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'label': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+ 'school_period': ('django.db.models.fields.IntegerField', [], {})
+ }
+ }
+
+ complete_apps = ['hdabo']