diff -r b758351d191f -r cc9b7e14412b web/lib/django/contrib/gis/utils/layermapping.py --- a/web/lib/django/contrib/gis/utils/layermapping.py Wed May 19 17:43:59 2010 +0200 +++ b/web/lib/django/contrib/gis/utils/layermapping.py Tue May 25 02:43:45 2010 +0200 @@ -3,115 +3,15 @@ The LayerMapping class provides a way to map the contents of OGR vector files (e.g. SHP files) to Geographic-enabled Django models. - This grew out of my personal needs, specifically the code repetition - that went into pulling geometries and fields out of an OGR layer, - converting to another coordinate system (e.g. WGS84), and then inserting - into a GeoDjango model. - - Please report any bugs encountered using this utility. - - Requirements: OGR C Library (from GDAL) required. - - Usage: - lm = LayerMapping(model, source_file, mapping) where, - - model: - GeoDjango model (not an instance) - - data: - OGR-supported data source file (e.g. a shapefile) or - gdal.DataSource instance - - mapping: - A python dictionary, keys are strings corresponding - to the GeoDjango model field, and values correspond to - string field names for the OGR feature, or if the model field - is a geographic then it should correspond to the OGR - geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'. - - Keyword Args: - layer: - The index of the layer to use from the Data Source (defaults to 0) - - source_srs: - Use this to specify the source SRS manually (for example, - some shapefiles don't come with a '.prj' file). An integer SRID, - a string WKT, and SpatialReference objects are valid parameters. - - encoding: - Specifies the encoding of the string in the OGR data source. - For example, 'latin-1', 'utf-8', and 'cp437' are all valid - encoding parameters. - - transaction_mode: - May be 'commit_on_success' (default) or 'autocommit'. - - transform: - Setting this to False will disable all coordinate transformations. - - unique: - Setting this to the name, or a tuple of names, from the given - model will create models unique only to the given name(s). - Geometries will from each feature will be added into the collection - associated with the unique model. Forces transaction mode to - be 'autocommit'. - -Example: - - 1. You need a GDAL-supported data source, like a shapefile. - - Assume we're using the test_poly SHP file: - >>> from django.contrib.gis.gdal import DataSource - >>> ds = DataSource('test_poly.shp') - >>> layer = ds[0] - >>> print layer.fields # Exploring the fields in the layer, we only want the 'str' field. - ['float', 'int', 'str'] - >>> print len(layer) # getting the number of features in the layer (should be 3) - 3 - >>> print layer.geom_type # Should be 3 (a Polygon) - 3 - >>> print layer.srs # WGS84 - GEOGCS["GCS_WGS_1984", - DATUM["WGS_1984", - SPHEROID["WGS_1984",6378137,298.257223563]], - PRIMEM["Greenwich",0], - UNIT["Degree",0.017453292519943295]] - - 2. Now we define our corresponding Django model (make sure to use syncdb): - - from django.contrib.gis.db import models - class TestGeo(models.Model, models.GeoMixin): - name = models.CharField(maxlength=25) # corresponds to the 'str' field - poly = models.PolygonField(srid=4269) # we want our model in a different SRID - objects = models.GeoManager() - def __str__(self): - return 'Name: %s' % self.name - - 3. Use LayerMapping to extract all the features and place them in the database: - - >>> from django.contrib.gis.utils import LayerMapping - >>> from geoapp.models import TestGeo - >>> mapping = {'name' : 'str', # The 'name' model field maps to the 'str' layer field. - 'poly' : 'POLYGON', # For geometry fields use OGC name. - } # The mapping is a dictionary - >>> lm = LayerMapping(TestGeo, 'test_poly.shp', mapping) - >>> lm.save(verbose=True) # Save the layermap, imports the data. - Saved: Name: 1 - Saved: Name: 2 - Saved: Name: 3 - - LayerMapping just transformed the three geometries from the SHP file from their - source spatial reference system (WGS84) to the spatial reference system of - the GeoDjango model (NAD83). If no spatial reference system is defined for - the layer, use the `source_srs` keyword with a SpatialReference object to - specify one. + For more information, please consult the GeoDjango documentation: + http://geodjango.org/docs/layermapping.html """ import sys from datetime import date, datetime from decimal import Decimal from django.core.exceptions import ObjectDoesNotExist +from django.db import connections, DEFAULT_DB_ALIAS from django.contrib.gis.db.models import GeometryField -from django.contrib.gis.db.backend import SpatialBackend from django.contrib.gis.gdal import CoordTransform, DataSource, \ OGRException, OGRGeometry, OGRGeomType, SpatialReference from django.contrib.gis.gdal.field import \ @@ -128,11 +28,14 @@ class LayerMapping(object): "A class that maps OGR Layers to GeoDjango Models." - + # Acceptable 'base' types for a multi-geometry type. MULTI_TYPES = {1 : OGRGeomType('MultiPoint'), 2 : OGRGeomType('MultiLineString'), 3 : OGRGeomType('MultiPolygon'), + OGRGeomType('Point25D').num : OGRGeomType('MultiPoint25D'), + OGRGeomType('LineString25D').num : OGRGeomType('MultiLineString25D'), + OGRGeomType('Polygon25D').num : OGRGeomType('MultiPolygon25D'), } # Acceptable Django field types and corresponding acceptable OGR @@ -161,10 +64,10 @@ 'commit_on_success' : transaction.commit_on_success, } - def __init__(self, model, data, mapping, layer=0, + def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None, - transaction_mode='commit_on_success', - transform=True, unique=None): + transaction_mode='commit_on_success', + transform=True, unique=None, using=DEFAULT_DB_ALIAS): """ A LayerMapping object is initialized using the given Model (not an instance), a DataSource (or string path to an OGR-supported data file), and a mapping @@ -178,20 +81,23 @@ self.ds = data self.layer = self.ds[layer] + self.using = using + self.spatial_backend = connections[using].ops + # Setting the mapping & model attributes. self.mapping = mapping self.model = model - + # Checking the layer -- intitialization of the object will fail if # things don't check out before hand. self.check_layer() - # Getting the geometry column associated with the model (an + # Getting the geometry column associated with the model (an # exception will be raised if there is no geometry column). - if SpatialBackend.mysql: + if self.spatial_backend.mysql: transform = False else: - self.geo_col = self.geometry_column() + self.geo_field = self.geometry_field() # Checking the source spatial reference system, and getting # the coordinate transformation object (unless the `transform` @@ -219,14 +125,17 @@ else: self.unique = None - # Setting the transaction decorator with the function in the + # Setting the transaction decorator with the function in the # transaction modes dictionary. if transaction_mode in self.TRANSACTION_MODES: self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode] self.transaction_mode = transaction_mode else: raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) - + + if using is None: + pass + #### Checking routines used during initialization #### def check_fid_range(self, fid_range): "This checks the `fid_range` keyword." @@ -282,30 +191,40 @@ if self.geom_field: raise LayerMapError('LayerMapping does not support more than one GeometryField per model.') + # Getting the coordinate dimension of the geometry field. + coord_dim = model_field.dim + try: - gtype = OGRGeomType(ogr_name) + if coord_dim == 3: + gtype = OGRGeomType(ogr_name + '25D') + else: + gtype = OGRGeomType(ogr_name) except OGRException: raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name) # Making sure that the OGR Layer's Geometry is compatible. ltype = self.layer.geom_type - if not (gtype == ltype or self.make_multi(ltype, model_field)): - raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s.' % (fld_name, gtype)) + if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)): + raise LayerMapError('Invalid mapping geometry; model has %s%s, ' + 'layer geometry type is %s.' % + (fld_name, (coord_dim == 3 and '(dim=3)') or '', ltype)) # Setting the `geom_field` attribute w/the name of the model field - # that is a Geometry. + # that is a Geometry. Also setting the coordinate dimension + # attribute. self.geom_field = field_name + self.coord_dim = coord_dim fields_val = model_field elif isinstance(model_field, models.ForeignKey): if isinstance(ogr_name, dict): # Is every given related model mapping field in the Layer? rel_model = model_field.rel.to - for rel_name, ogr_field in ogr_name.items(): + for rel_name, ogr_field in ogr_name.items(): idx = check_ogr_fld(ogr_field) try: rel_field = rel_model._meta.get_field(rel_name) except models.fields.FieldDoesNotExist: - raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % + raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % (rel_name, rel_model.__class__.__name__)) fields_val = rel_model else: @@ -321,25 +240,25 @@ # Can the OGR field type be mapped to the Django field type? if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]): - raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % + raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % (ogr_field, ogr_field.__name__, fld_name)) fields_val = model_field - + self.fields[field_name] = fields_val def check_srs(self, source_srs): "Checks the compatibility of the given spatial reference object." - from django.contrib.gis.models import SpatialRefSys + if isinstance(source_srs, SpatialReference): sr = source_srs - elif isinstance(source_srs, SpatialRefSys): + elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()): sr = source_srs.srs elif isinstance(source_srs, (int, basestring)): sr = SpatialReference(source_srs) else: # Otherwise just pulling the SpatialReference from the layer sr = self.layer.srs - + if not sr: raise LayerMapError('No source reference system defined.') else: @@ -349,7 +268,7 @@ "Checks the `unique` keyword parameter -- may be a sequence or string." if isinstance(unique, (list, tuple)): # List of fields to determine uniqueness with - for attr in unique: + for attr in unique: if not attr in self.mapping: raise ValueError elif isinstance(unique, basestring): # Only a single field passed in. @@ -370,7 +289,7 @@ # dictionary mapping. for field_name, ogr_name in self.mapping.items(): model_field = self.fields[field_name] - + if isinstance(model_field, GeometryField): # Verify OGR geometry. val = self.verify_geom(feat.geom, model_field) @@ -385,7 +304,7 @@ # Setting the keyword arguments for the field name with the # value obtained above. kwargs[field_name] = val - + return kwargs def unique_kwargs(self, kwargs): @@ -403,11 +322,11 @@ def verify_ogr_field(self, ogr_field, model_field): """ Verifies if the OGR Field contents are acceptable to the Django - model field. If they are, the verified value is returned, + model field. If they are, the verified value is returned, otherwise the proper exception is raised. """ - if (isinstance(ogr_field, OFTString) and - isinstance(model_field, (models.CharField, models.TextField))): + if (isinstance(ogr_field, OFTString) and + isinstance(model_field, (models.CharField, models.TextField))): if self.encoding: # The encoding for OGR data sources may be specified here # (e.g., 'cp437' for Census Bureau boundary files). @@ -432,14 +351,14 @@ # Maximum amount of precision, or digits to the left of the decimal. max_prec = model_field.max_digits - model_field.decimal_places - # Getting the digits to the left of the decimal place for the + # Getting the digits to the left of the decimal place for the # given decimal. if d_idx < 0: n_prec = len(digits[:d_idx]) else: n_prec = len(digits) + d_idx - # If we have more than the maximum digits allowed, then throw an + # If we have more than the maximum digits allowed, then throw an # InvalidDecimal exception. if n_prec > max_prec: raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' % @@ -462,7 +381,7 @@ mapping. """ # TODO: It is expensive to retrieve a model for every record -- - # explore if an efficient mechanism exists for caching related + # explore if an efficient mechanism exists for caching related # ForeignKey models. # Constructing and verifying the related model keyword arguments. @@ -475,13 +394,17 @@ return rel_model.objects.get(**fk_kwargs) except ObjectDoesNotExist: raise MissingForeignKey('No ForeignKey %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) - + def verify_geom(self, geom, model_field): """ Verifies the geometry -- will construct and return a GeometryCollection if necessary (for example if the model field is MultiPolygonField while the mapped shapefile only contains Polygons). """ + # Downgrade a 3D geom to a 2D one, if necessary. + if self.coord_dim != geom.coord_dim: + geom.coord_dim = self.coord_dim + if self.make_multi(geom.geom_type, model_field): # Constructing a multi-geometry type to contain the single geometry multi_type = self.MULTI_TYPES[geom.geom_type.num] @@ -491,61 +414,51 @@ g = geom # Transforming the geometry with our Coordinate Transformation object, - # but only if the class variable `transform` is set w/a CoordTransform + # but only if the class variable `transform` is set w/a CoordTransform # object. if self.transform: g.transform(self.transform) - + # Returning the WKT of the geometry. return g.wkt #### Other model methods #### def coord_transform(self): "Returns the coordinate transformation object." - from django.contrib.gis.models import SpatialRefSys + SpatialRefSys = self.spatial_backend.spatial_ref_sys() try: # Getting the target spatial reference system - target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs + target_srs = SpatialRefSys.objects.get(srid=self.geo_field.srid).srs # Creating the CoordTransform object return CoordTransform(self.source_srs, target_srs) except Exception, msg: raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) - def geometry_column(self): - "Returns the GeometryColumn model associated with the geographic column." - from django.contrib.gis.models import GeometryColumns - # Getting the GeometryColumn object. - try: - db_table = self.model._meta.db_table - geo_col = self.geom_field - if SpatialBackend.oracle: - # Making upper case for Oracle. - db_table = db_table.upper() - geo_col = geo_col.upper() - gc_kwargs = {GeometryColumns.table_name_col() : db_table, - GeometryColumns.geom_col_name() : geo_col, - } - return GeometryColumns.objects.get(**gc_kwargs) - except Exception, msg: - raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) + def geometry_field(self): + "Returns the GeometryField instance associated with the geographic column." + # Use the `get_field_by_name` on the model's options so that we + # get the correct field instance if there's model inheritance. + opts = self.model._meta + fld, model, direct, m2m = opts.get_field_by_name(self.geom_field) + return fld def make_multi(self, geom_type, model_field): """ - Given the OGRGeomType for a geometry and its associated GeometryField, + Given the OGRGeomType for a geometry and its associated GeometryField, determine whether the geometry should be turned into a GeometryCollection. """ - return (geom_type.num in self.MULTI_TYPES and + return (geom_type.num in self.MULTI_TYPES and model_field.__class__.__name__ == 'Multi%s' % geom_type.django) - def save(self, verbose=False, fid_range=False, step=False, + def save(self, verbose=False, fid_range=False, step=False, progress=False, silent=False, stream=sys.stdout, strict=False): """ Saves the contents from the OGR DataSource Layer into the database - according to the mapping dictionary given at initialization. - + according to the mapping dictionary given at initialization. + Keyword Parameters: verbose: - If set, information will be printed subsequent to each model save + If set, information will be printed subsequent to each model save executed on the database. fid_range: @@ -555,32 +468,32 @@ data source. step: - If set with an integer, transactions will occur at every step - interval. For example, if step=1000, a commit would occur after + If set with an integer, transactions will occur at every step + interval. For example, if step=1000, a commit would occur after the 1,000th feature, the 2,000th feature etc. progress: - When this keyword is set, status information will be printed giving - the number of features processed and sucessfully saved. By default, - progress information will pe printed every 1000 features processed, - however, this default may be overridden by setting this keyword with an + When this keyword is set, status information will be printed giving + the number of features processed and sucessfully saved. By default, + progress information will pe printed every 1000 features processed, + however, this default may be overridden by setting this keyword with an integer for the desired interval. stream: - Status information will be written to this file handle. Defaults to + Status information will be written to this file handle. Defaults to using `sys.stdout`, but any object with a `write` method is supported. silent: - By default, non-fatal error notifications are printed to stdout, but + By default, non-fatal error notifications are printed to stdout, but this keyword may be set to disable these notifications. strict: - Execution of the model mapping will cease upon the first error + Execution of the model mapping will cease upon the first error encountered. The default behavior is to attempt to continue. """ # Getting the default Feature ID range. default_range = self.check_fid_range(fid_range) - + # Setting the progress interval, if requested. if progress: if progress is True or not isinstance(progress, int): @@ -588,7 +501,7 @@ else: progress_interval = progress - # Defining the 'real' save method, utilizing the transaction + # Defining the 'real' save method, utilizing the transaction # decorator created during initialization. @self.transaction_decorator def _save(feat_range=default_range, num_feat=0, num_saved=0): @@ -605,7 +518,7 @@ except LayerMapError, msg: # Something borked the validation if strict: raise - elif not silent: + elif not silent: stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) else: # Constructing the model using the keyword args @@ -617,16 +530,16 @@ # Getting the keyword arguments and retrieving # the unique model. u_kwargs = self.unique_kwargs(kwargs) - m = self.model.objects.get(**u_kwargs) + m = self.model.objects.using(self.using).get(**u_kwargs) is_update = True - - # Getting the geometry (in OGR form), creating - # one from the kwargs WKT, adding in additional - # geometries, and update the attribute with the + + # Getting the geometry (in OGR form), creating + # one from the kwargs WKT, adding in additional + # geometries, and update the attribute with the # just-updated geometry WKT. geom = getattr(m, self.geom_field).ogr new = OGRGeometry(kwargs[self.geom_field]) - for g in new: geom.add(g) + for g in new: geom.add(g) setattr(m, self.geom_field, geom.wkt) except ObjectDoesNotExist: # No unique model exists yet, create. @@ -636,7 +549,7 @@ try: # Attempting to save. - m.save() + m.save(using=self.using) num_saved += 1 if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m)) except SystemExit: @@ -646,7 +559,7 @@ # Rolling back the transaction so that other model saves # will work. transaction.rollback_unless_managed() - if strict: + if strict: # Bailing out if the `strict` keyword is set. if not silent: stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) @@ -658,15 +571,15 @@ # Printing progress information, if requested. if progress and num_feat % progress_interval == 0: stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) - + # Only used for status output purposes -- incremental saving uses the # values returned here. return num_saved, num_feat nfeat = self.layer.num_feat if step and isinstance(step, int) and step < nfeat: - # Incremental saving is requested at the given interval (step) - if default_range: + # Incremental saving is requested at the given interval (step) + if default_range: raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.') beg, num_feat, num_saved = (0, 0, 0) indices = range(step, nfeat, step) @@ -677,7 +590,7 @@ # special (e.g, [100:] instead of [90:100]). if i+1 == n_i: step_slice = slice(beg, None) else: step_slice = slice(beg, end) - + try: num_feat, num_saved = _save(step_slice, num_feat, num_saved) beg = end