From 1a1ada4def94fef3ccd17bb42bc93a4107be10ee Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 22 Jan 2021 10:32:04 +0000 Subject: [PATCH] fixed to datasets and addgeo to correctly create geo instances --- core/management/commands/addgeo.py | 89 ++++++++++++++++++++---------- gadm36_ESP.json | 14 ++--- locations.json | 2 +- logs/.gitignore | 2 + 4 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 logs/.gitignore diff --git a/core/management/commands/addgeo.py b/core/management/commands/addgeo.py index c27ceb7..2e02848 100644 --- a/core/management/commands/addgeo.py +++ b/core/management/commands/addgeo.py @@ -6,16 +6,22 @@ from django.contrib.gis.geos import GEOSGeometry, MultiPolygon from geo.models import City, Region, Province, Country +logging.basicConfig( + filename='logs/addgeo.log', + filemode='w', + format='%(levelname)s:%(message)s', + level=logging.INFO, + ) class Command(BaseCommand): def handle(self, *args, **kwargs): - + print('Deleting all instances of Country, Region, Province, City') logging.info('Deleting all instances of Country, Region, Province, City') - City.objecs.all().delete() - Province.objecs.all().delete() - Region.objecs.all().delete() - Country.objecs.all().delete() + City.objects.all().delete() + Province.objects.all().delete() + Region.objects.all().delete() + Country.objects.all().delete() # create country for spain country = Country.objects.create(name='España') @@ -24,45 +30,64 @@ class Command(BaseCommand): locations = json.loads(open(locations_file).read()) # REGIONS - region_counter = 0 - geo_file='gadm36_ESP_1.json' + + geo_file='gadm36_ESP.json' # geo boundary data for regions geo_data = json.loads(open(geo_file).read()) - for feature in geo_data['features']: - geom = GEOSGeometry(str(feature['geometry'])) - if feature['geometry']['type'] == "MultiPolygon": - poly_list = [] - for poly in geom: - poly_list.append(poly) - print(poly_list) - else: - poly_list = geom - geom_geos = MultiPolygon(poly_list) - - name = feature['properties']['NAME_1'] - Region.objects.create(name=name, country=country, geo=geom_geos) - region_counter += 1 - - """ + logging.info("Starting Region creation") + print("Starting Region creation") + region_counter = 0 for location in locations: if location['model'] == 'locations.region': - logging.info(f"Creating Region Object {location['fields']['name']}...") + logging.info(f"Creating Region instance {location['fields']['name']}...") + new_region = None + name = location['fields']['name'] - Region.objects.create(name=name, country=country, id=location['pk']) - region_counter += 1 - """ + # Bypass for lack of geo data on ceuta and melilla + if name in ('Ceuta', 'Melilla'): + new_region = Region.objects.create(name=name, country=country, id=location['pk']) + logging.info(f"Region {name} created (without GADM data)") + else: + for feature in geo_data['features']: + if feature['properties']['NAME_1'] == name: + logging.debug(f"Region {name} found in GADM data") + # calculate geometry data + geom = GEOSGeometry(str(feature['geometry'])) + if feature['geometry']['type'] == "MultiPolygon": + poly_list = [] + for poly in geom: + poly_list.append(poly) + else: + poly_list = geom + geom_geos = MultiPolygon(poly_list) + # create instance + new_region = Region.objects.create( + name=name, country=country, geo=geom_geos, id=location['pk'] + ) + logging.info(f"Region {name} created") + region_counter += 1 + break + if new_region is None: + logging.warning(f"No region named {name} found in GADM data") + # PROVINCES + print("Starting Province creation") + logging.info("Starting Province creation") province_counter = 0 for location in locations: if location['model'] == 'locations.province': - logging.info(f"Creating Province Object {location['fields']['name']}...") + logging.info(f"Creating Province instance {location['fields']['name']}...") name = location['fields']['name'] - Province.objects.create(name=name, region=Region.objects.get(id=location['fields']['region']), id=location['pk']) + # get parent region + # import ipdb; ipdb.set_trace() + parent_region = Region.objects.get(id=location['fields']['region']) + Province.objects.create(name=name, region=parent_region, id=location['pk']) province_counter += 1 # CITIES + print("Starting City creation") + logging.info("Starting City creation") city_counter = 0 - print('Creating cities...') for location in locations: if location['model'] == 'locations.city': name = location['fields']['name'] @@ -73,3 +98,7 @@ class Command(BaseCommand): logging.info(f"Province instances created: {province_counter}") logging.info(f"City instances created: {city_counter}") + print(f"Region instances created: {region_counter}") + print(f"Province instances created: {province_counter}") + print(f"City instances created: {city_counter}") + diff --git a/gadm36_ESP.json b/gadm36_ESP.json index 75aef20..80831c3 100644 --- a/gadm36_ESP.json +++ b/gadm36_ESP.json @@ -2090,7 +2090,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.8_1", - "NAME_1": "Comunidad de Madrid", + "NAME_1": "Madrid, Comunidad de", "VARNAME_1": "Madrid|Communaut\u00e9 de Madrid| Community of Madrid|Comunidad de Madrid |Comunidade de Madrid|Comunitat de Madrid", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -2232,7 +2232,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.9_1", - "NAME_1": "Comunidad Foral de Navarra", + "NAME_1": "Navarra, Comunidad Foral de", "VARNAME_1": "Communaut\u00e9 forale de Navarre|Comunidade Foral de Navarra|Comunitat Foral|Navarra", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -3196,7 +3196,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.13_1", - "NAME_1": "Islas Baleares", + "NAME_1": "Baleares, Islas", "VARNAME_1": "Balearic Islands|Balearen|Balearene|Baleares|Islas Baleares|Baleari|\u00celes Bal\u00e9ares|Ilhas Baleares|Illes Balears", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -3492,7 +3492,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.14_1", - "NAME_1": "Islas Canarias", + "NAME_1": "Canarias", "VARNAME_1": "Canarias|Canary Islands|Can\u00e1rias|Ilhas Can\u00e1rias|Canarie|\u00celes Canaries|Illes Can\u00e0ries|Kanari\u00f8yene|Kanarische Inseln", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -3614,7 +3614,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.15_1", - "NAME_1": "La Rioja", + "NAME_1": "Rioja, La", "VARNAME_1": "Rioja", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -3920,7 +3920,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.17_1", - "NAME_1": "Principado de Asturias", + "NAME_1": "Asturias", "VARNAME_1": "Ast\u00farias|Asturie|Asturien|Asturies|Ast\u00faries|Asturias", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", @@ -4066,7 +4066,7 @@ "GID_0": "ESP", "NAME_0": "Spain", "GID_1": "ESP.18_1", - "NAME_1": "Regi\u00f3n de Murcia", + "NAME_1": "Murcia, Región de", "VARNAME_1": "Murcia|Regi\u00e3o de M\u00farcia|Regi\u00f3 de M\u00farcia|R\u00e9gion de Murcie|Region of Murcia", "NL_NAME_1": "", "TYPE_1": "Comunidad Aut\u00f3noma", diff --git a/locations.json b/locations.json index e2c88c5..c1390a5 100644 --- a/locations.json +++ b/locations.json @@ -75,7 +75,7 @@ "model": "locations.region", "pk": 67, "fields": { - "name": "Comunitat Valenciana", + "name": "Comunidad Valenciana", "code": "10" } }, diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file