src/hdalab/management/commands/import_hda_insee_csv.py
author ymh <ymh.work@gmail.com>
Fri, 19 Jul 2024 09:38:03 +0200
changeset 704 b5835dca2624
parent 443 27f71b0a772d
child 693 09e00f38d177
permissions -rw-r--r--
Adapt renkan preview to uses chrome headless/puppeteer
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     2
'''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     3
@author: raphv
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     4
'''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     5
import csv
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
     6
import re
443
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
     7
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
     8
from django.core.management.base import BaseCommand, CommandError
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
     9
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
    10
from hdabo.models import Datasheet
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
    11
from hdalab.models import InseeCoords, DatasheetExtras
27f71b0a772d next version of import_rdf
ymh <ymh.work@gmail.com>
parents: 272
diff changeset
    12
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    13
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    14
class Command(BaseCommand):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    15
    '''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    16
    Command to export tags
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    17
    '''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    18
    args = '<path_to_csv_file>'
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    19
    options = ''
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    20
    help = """Imports HDA Lab INSEE codes from a csv file"""
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    21
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    22
    def handle(self, *args, **options):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    23
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    24
        if len(args) == 0 or not args[0]:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    25
            raise CommandError("Give a CSV File to import")
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    26
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    27
        filename = args[0]
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    28
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    29
        csvfile = open(filename, "rb")
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    30
        dialect = csv.Sniffer().sniff(csvfile.read(1024))
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    31
        csvfile.seek(0)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    32
        reader = csv.reader(csvfile, dialect)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    33
        fieldstoget = [ 'id', 'insee' ]
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    34
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    35
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    36
        for i,line in enumerate(reader):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    37
            if i == 0:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    38
                fields = {}
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    39
                minlength = 0
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    40
                for j,field in enumerate(line):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    41
                    for fieldname in fieldstoget:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    42
                        if re.search('(?i)%s' % fieldname, field):
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    43
                            fields[fieldname] = j
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    44
                            minlength = max(j,minlength)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    45
            else:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    46
                if len(line) > minlength:
231
73123394bc53 correct import csv scripts
ymh <ymh.work@gmail.com>
parents: 124
diff changeset
    47
                    rawdata = dict([(k,line[v].strip()) for k,v in fields.iteritems()])
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    48
                    # Traitement special pour la Corse
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    49
                    insee = int(re.sub('^2(A|B)','20',rawdata['insee']))
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    50
                        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    51
                    hda_id = int(rawdata['id'])
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    52
                    
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    53
                    try:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    54
                        datasheet = Datasheet.objects.get(hda_id=hda_id)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    55
                    except Datasheet.DoesNotExist:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    56
                        #print "Datasheet for id %d does not exist" % hda_id
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    57
                        datasheet = None
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    58
                    try:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    59
                        inseecoord = InseeCoords.objects.get(insee=insee)
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    60
                    except InseeCoords.DoesNotExist:
124
f937ccc6c144 Correction import insee csv
veltr
parents: 123
diff changeset
    61
                        print "INSEE entry for Insee Code %d does not exist" % insee
f937ccc6c144 Correction import insee csv
veltr
parents: 123
diff changeset
    62
                        inseecoord = None
123
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    63
                    
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    64
                    if datasheet is not None and inseecoord is not None:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    65
                        dsextra, created = DatasheetExtras.objects.get_or_create(datasheet=datasheet, defaults={'insee':inseecoord})
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    66
                        if not created:
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    67
                            dsextra.insee = inseecoord
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    68
                            dsextra.save()
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    69
        
94fc5f5b5cfd Added Insee Codes
veltr
parents:
diff changeset
    70
        csvfile.close()