diff options
Diffstat (limited to 'lib/python2.7/site-packages/django/contrib/gis/utils/layermapping.py')
| -rw-r--r-- | lib/python2.7/site-packages/django/contrib/gis/utils/layermapping.py | 596 |
1 files changed, 596 insertions, 0 deletions
diff --git a/lib/python2.7/site-packages/django/contrib/gis/utils/layermapping.py b/lib/python2.7/site-packages/django/contrib/gis/utils/layermapping.py new file mode 100644 index 0000000..a502aa2 --- /dev/null +++ b/lib/python2.7/site-packages/django/contrib/gis/utils/layermapping.py @@ -0,0 +1,596 @@ +# LayerMapping -- A Django Model/OGR Layer Mapping Utility +""" + The LayerMapping class provides a way to map the contents of OGR + vector files (e.g. SHP files) to Geographic-enabled Django models. + + For more information, please consult the GeoDjango documentation: + http://geodjango.org/docs/layermapping.html +""" +import sys +from decimal import Decimal +from django.core.exceptions import ObjectDoesNotExist +from django.db import connections, router +from django.contrib.gis.db.models import GeometryField +from django.contrib.gis.gdal import (CoordTransform, DataSource, + OGRException, OGRGeometry, OGRGeomType, SpatialReference) +from django.contrib.gis.gdal.field import ( + OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime) +from django.db import models, transaction +from django.utils import six +from django.utils.encoding import force_text + + +# LayerMapping exceptions. +class LayerMapError(Exception): pass +class InvalidString(LayerMapError): pass +class InvalidDecimal(LayerMapError): pass +class InvalidInteger(LayerMapError): pass +class MissingForeignKey(LayerMapError): pass + +class LayerMapping(object): + "A class that maps OGR Layers to GeoDjango Models." + + # Acceptable 'base' types for a multi-geometry type. + MULTI_TYPES = {1 : OGRGeomType('MultiPoint'), + 2 : OGRGeomType('MultiLineString'), + 3 : OGRGeomType('MultiPolygon'), + OGRGeomType('Point25D').num : OGRGeomType('MultiPoint25D'), + OGRGeomType('LineString25D').num : OGRGeomType('MultiLineString25D'), + OGRGeomType('Polygon25D').num : OGRGeomType('MultiPolygon25D'), + } + + # Acceptable Django field types and corresponding acceptable OGR + # counterparts. + FIELD_TYPES = { + models.AutoField : OFTInteger, + models.IntegerField : (OFTInteger, OFTReal, OFTString), + models.FloatField : (OFTInteger, OFTReal), + models.DateField : OFTDate, + models.DateTimeField : OFTDateTime, + models.EmailField : OFTString, + models.TimeField : OFTTime, + models.DecimalField : (OFTInteger, OFTReal), + models.CharField : OFTString, + models.SlugField : OFTString, + models.TextField : OFTString, + models.URLField : OFTString, + models.BigIntegerField : (OFTInteger, OFTReal, OFTString), + models.SmallIntegerField : (OFTInteger, OFTReal, OFTString), + models.PositiveSmallIntegerField : (OFTInteger, OFTReal, OFTString), + } + + def __init__(self, model, data, mapping, layer=0, + source_srs=None, encoding='utf-8', + transaction_mode='commit_on_success', + transform=True, unique=None, using=None): + """ + A LayerMapping object is initialized using the given Model (not an instance), + a DataSource (or string path to an OGR-supported data file), and a mapping + dictionary. See the module level docstring for more details and keyword + argument usage. + """ + # Getting the DataSource and the associated Layer. + if isinstance(data, six.string_types): + self.ds = DataSource(data, encoding=encoding) + else: + self.ds = data + self.layer = self.ds[layer] + + self.using = using if using is not None else router.db_for_write(model) + self.spatial_backend = connections[self.using].ops + + # Setting the mapping & model attributes. + self.mapping = mapping + self.model = model + + # Checking the layer -- intitialization of the object will fail if + # things don't check out before hand. + self.check_layer() + + # Getting the geometry column associated with the model (an + # exception will be raised if there is no geometry column). + if self.spatial_backend.mysql: + transform = False + else: + self.geo_field = self.geometry_field() + + # Checking the source spatial reference system, and getting + # the coordinate transformation object (unless the `transform` + # keyword is set to False) + if transform: + self.source_srs = self.check_srs(source_srs) + self.transform = self.coord_transform() + else: + self.transform = transform + + # Setting the encoding for OFTString fields, if specified. + if encoding: + # Making sure the encoding exists, if not a LookupError + # exception will be thrown. + from codecs import lookup + lookup(encoding) + self.encoding = encoding + else: + self.encoding = None + + if unique: + self.check_unique(unique) + transaction_mode = 'autocommit' # Has to be set to autocommit. + self.unique = unique + else: + self.unique = None + + # Setting the transaction decorator with the function in the + # transaction modes dictionary. + self.transaction_mode = transaction_mode + if transaction_mode == 'autocommit': + self.transaction_decorator = None + elif transaction_mode == 'commit_on_success': + self.transaction_decorator = transaction.atomic + else: + raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) + + #### Checking routines used during initialization #### + def check_fid_range(self, fid_range): + "This checks the `fid_range` keyword." + if fid_range: + if isinstance(fid_range, (tuple, list)): + return slice(*fid_range) + elif isinstance(fid_range, slice): + return fid_range + else: + raise TypeError + else: + return None + + def check_layer(self): + """ + This checks the Layer metadata, and ensures that it is compatible + with the mapping information and model. Unlike previous revisions, + there is no need to increment through each feature in the Layer. + """ + # The geometry field of the model is set here. + # TODO: Support more than one geometry field / model. However, this + # depends on the GDAL Driver in use. + self.geom_field = False + self.fields = {} + + # Getting lists of the field names and the field types available in + # the OGR Layer. + ogr_fields = self.layer.fields + ogr_field_types = self.layer.field_types + + # Function for determining if the OGR mapping field is in the Layer. + def check_ogr_fld(ogr_map_fld): + try: + idx = ogr_fields.index(ogr_map_fld) + except ValueError: + raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld) + return idx + + # No need to increment through each feature in the model, simply check + # the Layer metadata against what was given in the mapping dictionary. + for field_name, ogr_name in self.mapping.items(): + # Ensuring that a corresponding field exists in the model + # for the given field name in the mapping. + try: + model_field = self.model._meta.get_field(field_name) + except models.fields.FieldDoesNotExist: + raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name) + + # Getting the string name for the Django field class (e.g., 'PointField'). + fld_name = model_field.__class__.__name__ + + if isinstance(model_field, GeometryField): + if self.geom_field: + raise LayerMapError('LayerMapping does not support more than one GeometryField per model.') + + # Getting the coordinate dimension of the geometry field. + coord_dim = model_field.dim + + try: + if coord_dim == 3: + gtype = OGRGeomType(ogr_name + '25D') + else: + gtype = OGRGeomType(ogr_name) + except OGRException: + raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name) + + # Making sure that the OGR Layer's Geometry is compatible. + ltype = self.layer.geom_type + if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)): + raise LayerMapError('Invalid mapping geometry; model has %s%s, ' + 'layer geometry type is %s.' % + (fld_name, '(dim=3)' if coord_dim == 3 else '', ltype)) + + # Setting the `geom_field` attribute w/the name of the model field + # that is a Geometry. Also setting the coordinate dimension + # attribute. + self.geom_field = field_name + self.coord_dim = coord_dim + fields_val = model_field + elif isinstance(model_field, models.ForeignKey): + if isinstance(ogr_name, dict): + # Is every given related model mapping field in the Layer? + rel_model = model_field.rel.to + for rel_name, ogr_field in ogr_name.items(): + idx = check_ogr_fld(ogr_field) + try: + rel_field = rel_model._meta.get_field(rel_name) + except models.fields.FieldDoesNotExist: + raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % + (rel_name, rel_model.__class__.__name__)) + fields_val = rel_model + else: + raise TypeError('ForeignKey mapping must be of dictionary type.') + else: + # Is the model field type supported by LayerMapping? + if not model_field.__class__ in self.FIELD_TYPES: + raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name) + + # Is the OGR field in the Layer? + idx = check_ogr_fld(ogr_name) + ogr_field = ogr_field_types[idx] + + # Can the OGR field type be mapped to the Django field type? + if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]): + raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % + (ogr_field, ogr_field.__name__, fld_name)) + fields_val = model_field + + self.fields[field_name] = fields_val + + def check_srs(self, source_srs): + "Checks the compatibility of the given spatial reference object." + + if isinstance(source_srs, SpatialReference): + sr = source_srs + elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()): + sr = source_srs.srs + elif isinstance(source_srs, (int, six.string_types)): + sr = SpatialReference(source_srs) + else: + # Otherwise just pulling the SpatialReference from the layer + sr = self.layer.srs + + if not sr: + raise LayerMapError('No source reference system defined.') + else: + return sr + + def check_unique(self, unique): + "Checks the `unique` keyword parameter -- may be a sequence or string." + if isinstance(unique, (list, tuple)): + # List of fields to determine uniqueness with + for attr in unique: + if not attr in self.mapping: raise ValueError + elif isinstance(unique, six.string_types): + # Only a single field passed in. + if unique not in self.mapping: raise ValueError + else: + raise TypeError('Unique keyword argument must be set with a tuple, list, or string.') + + #### Keyword argument retrieval routines #### + def feature_kwargs(self, feat): + """ + Given an OGR Feature, this will return a dictionary of keyword arguments + for constructing the mapped model. + """ + # The keyword arguments for model construction. + kwargs = {} + + # Incrementing through each model field and OGR field in the + # dictionary mapping. + for field_name, ogr_name in self.mapping.items(): + model_field = self.fields[field_name] + + if isinstance(model_field, GeometryField): + # Verify OGR geometry. + try: + val = self.verify_geom(feat.geom, model_field) + except OGRException: + raise LayerMapError('Could not retrieve geometry from feature.') + elif isinstance(model_field, models.base.ModelBase): + # The related _model_, not a field was passed in -- indicating + # another mapping for the related Model. + val = self.verify_fk(feat, model_field, ogr_name) + else: + # Otherwise, verify OGR Field type. + val = self.verify_ogr_field(feat[ogr_name], model_field) + + # Setting the keyword arguments for the field name with the + # value obtained above. + kwargs[field_name] = val + + return kwargs + + def unique_kwargs(self, kwargs): + """ + Given the feature keyword arguments (from `feature_kwargs`) this routine + will construct and return the uniqueness keyword arguments -- a subset + of the feature kwargs. + """ + if isinstance(self.unique, six.string_types): + return {self.unique : kwargs[self.unique]} + else: + return dict((fld, kwargs[fld]) for fld in self.unique) + + #### Verification routines used in constructing model keyword arguments. #### + def verify_ogr_field(self, ogr_field, model_field): + """ + Verifies if the OGR Field contents are acceptable to the Django + model field. If they are, the verified value is returned, + otherwise the proper exception is raised. + """ + if (isinstance(ogr_field, OFTString) and + isinstance(model_field, (models.CharField, models.TextField))): + if self.encoding: + # The encoding for OGR data sources may be specified here + # (e.g., 'cp437' for Census Bureau boundary files). + val = force_text(ogr_field.value, self.encoding) + else: + val = ogr_field.value + if model_field.max_length and len(val) > model_field.max_length: + raise InvalidString('%s model field maximum string length is %s, given %s characters.' % + (model_field.name, model_field.max_length, len(val))) + elif isinstance(ogr_field, OFTReal) and isinstance(model_field, models.DecimalField): + try: + # Creating an instance of the Decimal value to use. + d = Decimal(str(ogr_field.value)) + except: + raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field.value) + + # Getting the decimal value as a tuple. + dtup = d.as_tuple() + digits = dtup[1] + d_idx = dtup[2] # index where the decimal is + + # Maximum amount of precision, or digits to the left of the decimal. + max_prec = model_field.max_digits - model_field.decimal_places + + # Getting the digits to the left of the decimal place for the + # given decimal. + if d_idx < 0: + n_prec = len(digits[:d_idx]) + else: + n_prec = len(digits) + d_idx + + # If we have more than the maximum digits allowed, then throw an + # InvalidDecimal exception. + if n_prec > max_prec: + raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' % + (model_field.max_digits, model_field.decimal_places, max_prec)) + val = d + elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(model_field, models.IntegerField): + # Attempt to convert any OFTReal and OFTString value to an OFTInteger. + try: + val = int(ogr_field.value) + except: + raise InvalidInteger('Could not construct integer from: %s' % ogr_field.value) + else: + val = ogr_field.value + return val + + def verify_fk(self, feat, rel_model, rel_mapping): + """ + Given an OGR Feature, the related model and its dictionary mapping, + this routine will retrieve the related model for the ForeignKey + mapping. + """ + # TODO: It is expensive to retrieve a model for every record -- + # explore if an efficient mechanism exists for caching related + # ForeignKey models. + + # Constructing and verifying the related model keyword arguments. + fk_kwargs = {} + for field_name, ogr_name in rel_mapping.items(): + fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name)) + + # Attempting to retrieve and return the related model. + try: + return rel_model.objects.using(self.using).get(**fk_kwargs) + except ObjectDoesNotExist: + raise MissingForeignKey('No ForeignKey %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) + + def verify_geom(self, geom, model_field): + """ + Verifies the geometry -- will construct and return a GeometryCollection + if necessary (for example if the model field is MultiPolygonField while + the mapped shapefile only contains Polygons). + """ + # Downgrade a 3D geom to a 2D one, if necessary. + if self.coord_dim != geom.coord_dim: + geom.coord_dim = self.coord_dim + + if self.make_multi(geom.geom_type, model_field): + # Constructing a multi-geometry type to contain the single geometry + multi_type = self.MULTI_TYPES[geom.geom_type.num] + g = OGRGeometry(multi_type) + g.add(geom) + else: + g = geom + + # Transforming the geometry with our Coordinate Transformation object, + # but only if the class variable `transform` is set w/a CoordTransform + # object. + if self.transform: g.transform(self.transform) + + # Returning the WKT of the geometry. + return g.wkt + + #### Other model methods #### + def coord_transform(self): + "Returns the coordinate transformation object." + SpatialRefSys = self.spatial_backend.spatial_ref_sys() + try: + # Getting the target spatial reference system + target_srs = SpatialRefSys.objects.using(self.using).get(srid=self.geo_field.srid).srs + + # Creating the CoordTransform object + return CoordTransform(self.source_srs, target_srs) + except Exception as msg: + new_msg = 'Could not translate between the data source and model geometry: %s' % msg + six.reraise(LayerMapError, LayerMapError(new_msg), sys.exc_info()[2]) + + def geometry_field(self): + "Returns the GeometryField instance associated with the geographic column." + # Use the `get_field_by_name` on the model's options so that we + # get the correct field instance if there's model inheritance. + opts = self.model._meta + fld, model, direct, m2m = opts.get_field_by_name(self.geom_field) + return fld + + def make_multi(self, geom_type, model_field): + """ + Given the OGRGeomType for a geometry and its associated GeometryField, + determine whether the geometry should be turned into a GeometryCollection. + """ + return (geom_type.num in self.MULTI_TYPES and + model_field.__class__.__name__ == 'Multi%s' % geom_type.django) + + def save(self, verbose=False, fid_range=False, step=False, + progress=False, silent=False, stream=sys.stdout, strict=False): + """ + Saves the contents from the OGR DataSource Layer into the database + according to the mapping dictionary given at initialization. + + Keyword Parameters: + verbose: + If set, information will be printed subsequent to each model save + executed on the database. + + fid_range: + May be set with a slice or tuple of (begin, end) feature ID's to map + from the data source. In other words, this keyword enables the user + to selectively import a subset range of features in the geographic + data source. + + step: + If set with an integer, transactions will occur at every step + interval. For example, if step=1000, a commit would occur after + the 1,000th feature, the 2,000th feature etc. + + progress: + When this keyword is set, status information will be printed giving + the number of features processed and sucessfully saved. By default, + progress information will pe printed every 1000 features processed, + however, this default may be overridden by setting this keyword with an + integer for the desired interval. + + stream: + Status information will be written to this file handle. Defaults to + using `sys.stdout`, but any object with a `write` method is supported. + + silent: + By default, non-fatal error notifications are printed to stdout, but + this keyword may be set to disable these notifications. + + strict: + Execution of the model mapping will cease upon the first error + encountered. The default behavior is to attempt to continue. + """ + # Getting the default Feature ID range. + default_range = self.check_fid_range(fid_range) + + # Setting the progress interval, if requested. + if progress: + if progress is True or not isinstance(progress, int): + progress_interval = 1000 + else: + progress_interval = progress + + def _save(feat_range=default_range, num_feat=0, num_saved=0): + if feat_range: + layer_iter = self.layer[feat_range] + else: + layer_iter = self.layer + + for feat in layer_iter: + num_feat += 1 + # Getting the keyword arguments + try: + kwargs = self.feature_kwargs(feat) + except LayerMapError as msg: + # Something borked the validation + if strict: raise + elif not silent: + stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) + else: + # Constructing the model using the keyword args + is_update = False + if self.unique: + # If we want unique models on a particular field, handle the + # geometry appropriately. + try: + # Getting the keyword arguments and retrieving + # the unique model. + u_kwargs = self.unique_kwargs(kwargs) + m = self.model.objects.using(self.using).get(**u_kwargs) + is_update = True + + # Getting the geometry (in OGR form), creating + # one from the kwargs WKT, adding in additional + # geometries, and update the attribute with the + # just-updated geometry WKT. + geom = getattr(m, self.geom_field).ogr + new = OGRGeometry(kwargs[self.geom_field]) + for g in new: geom.add(g) + setattr(m, self.geom_field, geom.wkt) + except ObjectDoesNotExist: + # No unique model exists yet, create. + m = self.model(**kwargs) + else: + m = self.model(**kwargs) + + try: + # Attempting to save. + m.save(using=self.using) + num_saved += 1 + if verbose: stream.write('%s: %s\n' % ('Updated' if is_update else 'Saved', m)) + except SystemExit: + raise + except Exception as msg: + if strict: + # Bailing out if the `strict` keyword is set. + if not silent: + stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) + stream.write('%s\n' % kwargs) + raise + elif not silent: + stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) + + # Printing progress information, if requested. + if progress and num_feat % progress_interval == 0: + stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) + + # Only used for status output purposes -- incremental saving uses the + # values returned here. + return num_saved, num_feat + + if self.transaction_decorator is not None: + _save = self.transaction_decorator(_save) + + nfeat = self.layer.num_feat + if step and isinstance(step, int) and step < nfeat: + # Incremental saving is requested at the given interval (step) + if default_range: + raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.') + beg, num_feat, num_saved = (0, 0, 0) + indices = range(step, nfeat, step) + n_i = len(indices) + + for i, end in enumerate(indices): + # Constructing the slice to use for this step; the last slice is + # special (e.g, [100:] instead of [90:100]). + if i+1 == n_i: step_slice = slice(beg, None) + else: step_slice = slice(beg, end) + + try: + num_feat, num_saved = _save(step_slice, num_feat, num_saved) + beg = end + except: + stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice)) + raise + else: + # Otherwise, just calling the previously defined _save() function. + _save() |