web/hdalab/management/commands/import_insee_csv.py
author veltr
Fri, 16 Nov 2012 18:12:05 +0100
changeset 265 73f19fa4f997
parent 231 73123394bc53
permissions -rw-r--r--
Merge with 6a381e9fd3e7d7990ea7cc3af8c57b34aa5bcdb2
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     2
'''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     3
@author: raphv
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     4
'''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     5
from django.core.management.base import BaseCommand, CommandError
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     6
import django.utils.simplejson as json
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     7
from SPARQLWrapper import SPARQLWrapper, JSON
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     8
from hdalab.models import InseeCoords
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     9
import csv
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    10
import re
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    11
import sys
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    12
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    13
class Command(BaseCommand):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    14
    '''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    15
    Command to export tags
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    16
    '''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    17
    args = '<path_to_csv_file>'
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    18
    options = ''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    19
    help = """Imports Insee codes and geographic coordinates from a csv file"""
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    20
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    21
    def handle(self, *args, **options):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    22
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    23
        if len(args) == 0 or not args[0]:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    24
            raise CommandError("Give a CSV File to import")
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    25
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    26
        filename = args[0]
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    27
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    28
        csvfile = open(filename, "rb")
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    29
        dialect = csv.Sniffer().sniff(csvfile.read(1024))
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    30
        csvfile.seek(0)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    31
        reader = csv.reader(csvfile, dialect)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    32
        fieldstoget = [ 'ville', 'insee', 'latitude', 'longitude' ]
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    33
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    34
        for i,line in enumerate(reader):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    35
            if i == 0:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    36
                fields = {}
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    37
                minlength = 0
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    38
                for j,field in enumerate(line):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    39
                    for fieldname in fieldstoget:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    40
                        if re.search('(?i)%s' % fieldname, field):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    41
                            fields[fieldname] = j
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    42
                            minlength = max(j,minlength)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    43
            else:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    44
                if len(line) > minlength:
231
73123394bc53 correct import csv scripts
ymh <ymh.work@gmail.com>
parents: 124
diff changeset
    45
                    rawdata = dict([(k,line[v].strip()) for k,v in fields.iteritems()])
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    46
                    #print "Processing line %d" % i
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    47
                    #print rawdata
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    48
                    try:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    49
                        latitude = float(rawdata['latitude'].replace(',','.'))
124
f937ccc6c144 Correction import insee csv
veltr
parents: 123
diff changeset
    50
                        longitude = float(rawdata['longitude'].replace(',','.')) if rawdata['longitude'] != '-' else 0
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    51
                        insee = int(rawdata['insee'])
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    52
                        ville = unicode(rawdata['ville'], 'iso-8859-1')
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    53
                        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    54
                        InseeCoords.objects.get_or_create(insee=insee, city_name=ville, latitude=latitude, longitude=longitude)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    55
                        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    56
                    except:
124
f937ccc6c144 Correction import insee csv
veltr
parents: 123
diff changeset
    57
                        print line, "Error :", sys.exc_info()[1]
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    58
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    59
        csvfile.close()