From 20a4040aa7b834348ff06f34e76f3b6dda5d524e Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 11 Feb 2021 12:51:59 +0000 Subject: [PATCH] improvements to loadgisdata --- core/management/commands/addgeo.py | 9 +- core/management/commands/loadgisdata.py | 141 +++++++++++++----------- datasets/gadm36_ESP.json | 2 +- 3 files changed, 81 insertions(+), 71 deletions(-) diff --git a/core/management/commands/addgeo.py b/core/management/commands/addgeo.py index 50cf6af..f4f095d 100644 --- a/core/management/commands/addgeo.py +++ b/core/management/commands/addgeo.py @@ -32,8 +32,6 @@ class Command(BaseCommand): for layer in ds: logging.info(f"Layer {layer.name}:\n\t- Layers: {len(layer)}\n\t- Type: {layer.geom_type.name}\n\t- Features: {layer.num_feat}") - # create country for spain - # country = Country.objects.create(name='España') # country instances logging.info("loading country instances") country_counter = 0 @@ -101,8 +99,8 @@ class Command(BaseCommand): name = location['fields']['name'] new_province = None for feature in ds[2]: - import ipdb; ipdb.set_trace() - if feature.get('NAME_1') == name: + # import ipdb; ipdb.set_trace() + if feature.get('NAME_2') == name: logging.debug(f"Province {name} found in GADM data") # calculate geometry data geom = GEOSGeometry(str(feature.geom)) @@ -111,7 +109,8 @@ class Command(BaseCommand): polygon_list.append(polygon) geom_geos = MultiPolygon(polygon_list) # create instance - parent_region = Region.objects.get(id=location['fields']['region']) + import ipdb; ipdb.set_trace() + parent_region = Region.objects.get(name=feature.get('NAME_1')) new_province = Province.objects.create( name=name, region=parent_region, geo=geom_geos, id=location['pk'] ) diff --git a/core/management/commands/loadgisdata.py b/core/management/commands/loadgisdata.py index 8a2caa5..1ca9836 100644 --- a/core/management/commands/loadgisdata.py +++ b/core/management/commands/loadgisdata.py @@ -17,9 +17,12 @@ class Command(BaseCommand): help = 'Load GIS data into database' def handle(self, *args, **kwargs): - - print(self.help) - + logging.info('Deleting all instances of Country, Region, Province, City') + City.objects.all().delete() + Province.objects.all().delete() + Region.objects.all().delete() + Country.objects.all().delete() + print('Deleted all instances of Country, Region, Province, City') ds = DataSource('datasets/gadm36_ESP.gpkg') @@ -31,86 +34,94 @@ class Command(BaseCommand): # country instances logging.info("loading country instances") + print('Creating Country instances') country_counter = 0 for feature in ds[0]: - geom = GEOSGeometry(str(feature['geometry'])) - if feature['geometry']['type'] == "MultiPolygon": - poly_list = [] - for poly in geom: - poly_list.append(poly) - print(poly_list) - else: - poly_list = geom - - geom_geos = MultiPolygon(poly_list) - - name = feature['properties']['NAME_1'] - if name == 'España': - SPAIN = Country.objects.create(name=name,poly=geom_geos) - Country.objects.create(name=name,poly=geom_geos) - country_counter += 1 - logging.info(f"Country instance created for: {name}") + try: + # calculate geometry data + geom = GEOSGeometry(str(feature.geom)) + polygon_list = [] + for polygon in geom: + polygon_list.append(polygon) + geom_geos = MultiPolygon(polygon_list) + # create instance + name = feature.get('NAME_0') + if name == 'Spain': + SPAIN = Country.objects.create(name='España',geo=geom_geos) + Country.objects.create(name=name,geo=geom_geos) + country_counter += 1 + logging.info(f"Country instance created for: {name}") + except Exception as e: + logging.error(f"[{type(e)}] {str(e)}") + import ipdb; ipdb.set_trace() # region instances logging.info("loading region instances") + print('Creating Region instances') region_counter = 0 for feature in ds[1]: - geom = GEOSGeometry(str(feature['geometry'])) - if feature['geometry']['type'] == "MultiPolygon": - poly_list = [] - for poly in geom: - poly_list.append(poly) - print(poly_list) - else: - poly_list = geom - - geom_geos = MultiPolygon(poly_list) - - name = feature['properties']['NAME_1'] - Region.objects.create(name=name,poly=geom_geos, country=SPAIN) - region_counter += 1 - logging.info(f"Region instance created for: {name}") + try: + # calculate geometry data + geom = GEOSGeometry(str(feature.geom)) + polygon_list = [] + for polygon in geom: + polygon_list.append(polygon) + geom_geos = MultiPolygon(polygon_list) + # get region name + name = feature.get('NAME_1') + # create instance + Region.objects.create(name=name, geo=geom_geos, country=SPAIN) + region_counter += 1 + logging.info(f"Region instance created for: {name}") + except Exception as e: + logging.error(f"[{type(e)}] {str(e)}") + import ipdb; ipdb.set_trace() # province instances logging.info("loading province instances") + print('Creating Province instances') province_counter = 0 for feature in ds[2]: - geom = GEOSGeometry(str(feature['geometry'])) - if feature['geometry']['type'] == "MultiPolygon": - poly_list = [] - for poly in geom: - poly_list.append(poly) - print(poly_list) - else: - poly_list = geom - - geom_geos = MultiPolygon(poly_list) - - name = feature['properties']['NAME_1'] - parent_region = Region.objects.get(id=location['fields']['region']) - Province.objects.create(name=name,poly=geom_geos, country=SPAIN) - province_counter += 1 - logging.info(f"Province instance created for: {name}") + try: + # calculate geometry data + geom = GEOSGeometry(str(feature.geom)) + polygon_list = [] + for polygon in geom: + polygon_list.append(polygon) + geom_geos = MultiPolygon(polygon_list) + # get name and parent + name = feature['properties']['NAME_2'] + parent_region = Region.objects.get(name=feature.get('NAME_1')) + # create instance + Province.objects.create(name=name, geo=geom_geos, region=parent_region) + province_counter += 1 + logging.info(f"Province instance created for: {name}") + except Exception as e: + logging.error(f"[{type(e)}] {str(e)}") + import ipdb; ipdb.set_trace() # city instances logging.info("loading city instances") + print('Creating City instances') city_counter = 0 for feature in ds[4]: - geom = GEOSGeometry(str(feature['geometry'])) - if feature['geometry']['type'] == "MultiPolygon": - poly_list = [] - for poly in geom: - poly_list.append(poly) - print(poly_list) - else: - poly_list = geom + try: + # calculate geometry data + geom = GEOSGeometry(str(feature.geom)) + polygon_list = [] + for polygon in geom: + polygon_list.append(polygon) + geom_geos = MultiPolygon(polygon_list) - geom_geos = MultiPolygon(poly_list) - - name = feature['properties']['NAME_1'] - City.objects.create(name=name,poly=geom_geos, country=SPAIN) - city_counter += 1 - logging.debug(f"City instance created for: {name}") + # get name and parent + name = feature['properties']['NAME_3'] + parent_province = Region.objects.get(name=feature.get('NAME_2')) + City.objects.create(name=name, geo=geom_geos, province=parent_province) + city_counter += 1 + logging.debug(f"City instance created for: {name}") + except Exception as e: + logging.error(f"[{type(e)}] {str(e)}") + import ipdb; ipdb.set_trace() logging.info(f"Country instances created: {country_counter}") logging.info(f"Region instances created: {region_counter}") diff --git a/datasets/gadm36_ESP.json b/datasets/gadm36_ESP.json index 80831c3..26eed13 100644 --- a/datasets/gadm36_ESP.json +++ b/datasets/gadm36_ESP.json @@ -3920,7 +3920,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.17_1", - "NAME_1": "Asturias", + "NAME_1": "Principado de Asturias", "VARNAME_1": "Ast\u00farias|Asturie|Asturien|Asturies|Ast\u00faries|Asturias", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma",