# HG changeset patch # User ymh # Date 1372237058 -7200 # Node ID 7ab0021faed9e799df92f9ce3f7ad7cf9ae750ac # Parent 769a3ccebdd6f9ef2ee6bbb6eb740e03523580c1 improve memory consumption diff -r 769a3ccebdd6 -r 7ab0021faed9 src/jocondelab/management/commands/import_csv.py --- a/src/jocondelab/management/commands/import_csv.py Tue Jun 25 22:14:14 2013 +0200 +++ b/src/jocondelab/management/commands/import_csv.py Wed Jun 26 10:57:38 2013 +0200 @@ -15,7 +15,7 @@ LIEUX_CONTEXT, PERI_CONTEXT, REPR_CONTEXT, SREP_CONTEXT) from core.utils import show_progress from django.core.management import BaseCommand -from django.db import transaction +from django.db import transaction, reset_queries from optparse import make_option import csv import datetime @@ -135,11 +135,7 @@ for i,_ in enumerate(reader): if i >= (max_lines-1): break - - - transaction.enter_transaction_management() - transaction.managed() - + objects_buffer = {} nb_lines = min(max_lines, i+1) @@ -180,8 +176,7 @@ for klass, obj_list in objects_buffer.iteritems(): klass.objects.bulk_create(obj_list) objects_buffer = {} - transaction.commit() - + reset_queries() except Exception as e: error_msg = "%s - Error treating line %d/%d: id %s : %s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),i+1, reader.line_num, row['REF'] if (row and 'REF' in row and row['REF']) else 'n/a', repr(e) ) logger.exception(error_msg) @@ -193,20 +188,23 @@ for klass, obj_list in objects_buffer.iteritems(): klass.objects.bulk_create(obj_list) objects_buffer = {} - transaction.commit() + reset_queries() except Exception as e: error_msg = "%s - Error treating line : %s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), repr(e) ) logger.exception(error_msg) if not cont_on_error: raise + + transaction.enter_transaction_management() + transaction.managed() notice_count = Notice.objects.count() self.stdout.write("Processing %d notices" % notice_count) writer = None - for i,notice_obj in enumerate(Notice.objects.iterator()): + for i,notice_obj in enumerate(Notice.objects.all().iterator()): writer = show_progress(i+1, notice_count, u"Processing notice %s" % notice_obj.ref, 50, writer) for field in POST_NOTICE_FIELDS: processor = NOTICE_FIELD_PROCESSORS.get(field, DEFAULT_FIELD_PROCESSOR_KLASS(field)) @@ -219,6 +217,7 @@ map(lambda o: o.save(), obj_list) objects_buffer = {} transaction.commit() + reset_queries() if objects_buffer: try: @@ -226,6 +225,7 @@ map(lambda o: o.save(), obj_list) objects_buffer = {} transaction.commit() + reset_queries() except Exception as e: error_msg = "%s - Error treating line: %s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), repr(e) ) logger.exception(error_msg) diff -r 769a3ccebdd6 -r 7ab0021faed9 src/jocondelab/management/commands/import_term_labels.py --- a/src/jocondelab/management/commands/import_term_labels.py Tue Jun 25 22:14:14 2013 +0200 +++ b/src/jocondelab/management/commands/import_term_labels.py Wed Jun 26 10:57:38 2013 +0200 @@ -10,7 +10,7 @@ from core.rdf_models import graph from core.wp_utils import normalize_term from django.core.management.base import NoArgsCommand -from django.db import transaction +from django.db import transaction, reset_queries from optparse import make_option from rdflib.term import URIRef @@ -56,7 +56,9 @@ self.stdout.write("Thesaurus %s term pref label %s for %s does not exists" % (thes.label, repr(o), repr(s))) if not ((i+1) % batch_size): transaction.commit() + reset_queries() transaction.commit() + reset_queries() for i,(s,_,o) in enumerate(graph.triples((None, URIRef("http://www.w3.org/2004/02/skos/core#altLabel"), None), context=context)): self.stdout.write("%d - Thesaurus %s term alt label %s for %s" % (i+1, thes.label, repr(o), repr(s))) @@ -70,7 +72,9 @@ self.stdout.write("Thesaurus %s term alt label %s for %s does not exists" % (thes.label, repr(o), repr(s))) if not ((i+1) % batch_size): transaction.commit() + reset_queries() transaction.commit() - transaction.leave_transaction_management() + reset_queries() + transaction.leave_transaction_management() \ No newline at end of file diff -r 769a3ccebdd6 -r 7ab0021faed9 src/jocondelab/management/commands/import_terms.py --- a/src/jocondelab/management/commands/import_terms.py Tue Jun 25 22:14:14 2013 +0200 +++ b/src/jocondelab/management/commands/import_terms.py Wed Jun 26 10:57:38 2013 +0200 @@ -12,7 +12,7 @@ LIEUX_CONTEXT, PERI_CONTEXT, REPR_CONTEXT, SREP_CONTEXT) from core.wp_utils import get_or_create_term, switch_case_group from django.core.management.base import NoArgsCommand -from django.db import transaction +from django.db import transaction, reset_queries from optparse import make_option from rdflib.term import URIRef @@ -74,6 +74,7 @@ if not Thesaurus.objects.filter(uri=t_def['uri']).exists(): Thesaurus.objects.create(**t_def) transaction.commit() + reset_queries() for thes in Thesaurus.objects.all(): self.stdout.write("Processing Thesaurus %s" % thes.label) @@ -84,6 +85,7 @@ thes.description = unicode(o) thes.save() transaction.commit() + reset_queries() context = graph.get_context(URIRef(thes.uri)) for i,(s,_,o) in enumerate(graph.triples((None, URIRef("http://www.w3.org/2004/02/skos/core#prefLabel"), None), context=context)): self.stdout.write("%d - Thesaurus %s term pref label %s" % (i+1,thes.label, repr(o))) @@ -99,7 +101,9 @@ if not ((i+1) % batch_size): transaction.commit() + reset_queries() transaction.commit() + reset_queries() for i,(s,_,o) in enumerate(graph.triples((None, URIRef("http://www.w3.org/2004/02/skos/core#altLabel"), None), context=context)): self.stdout.write("%d - Thesaurus %s term alt label %s for %s" % (i+1, thes.label, repr(o), repr(s))) @@ -113,7 +117,9 @@ self.stdout.write("Thesaurus %s term alt label %s for %s does not exists" % (thes.label, repr(o), repr(s))) if not ((i+1) % batch_size): transaction.commit() + reset_queries() transaction.commit() - transaction.leave_transaction_management() + reset_queries() + transaction.leave_transaction_management() \ No newline at end of file