loadgisdata working, removed old addgeo

README.md

@@ -37,7 +37,7 @@ python manage.py migrate
 
 ## Location data
 
-To load initial location data use: `python manage.py addgeo`
+To load initial location data use: `python manage.py loadgisdata`
 
 
 ## Endpoints

core/management/commands/addgeo.py

@@ -1,140 +0,0 @@
-import logging
-import json
-
-from django.core.management.base import BaseCommand
-from django.contrib.gis.geos import GEOSGeometry, MultiPolygon
-from django.contrib.gis.gdal import DataSource
-
-from geo.models import City, Region, Province, Country
-
-logging.basicConfig(
-    filename='logs/addgeo.log',
-    filemode='w',
-    format='%(levelname)s:%(message)s',
-    level=logging.INFO,
-    )
-
-class Command(BaseCommand):
-
-    help = 'Load geographic dataset'
-
-    def handle(self, *args, **kwargs):
-        print('Deleting all instances of Country, Region, Province, City')
-        logging.info('Deleting all instances of Country, Region, Province, City')
-        City.objects.all().delete()
-        Province.objects.all().delete()
-        Region.objects.all().delete()
-        Country.objects.all().delete()
-
-        # load GADM data
-        ds = DataSource('datasets/gadm36_ESP.gpkg')
-        logging.info('GeoPackage data:')
-        for layer in ds:
-            logging.info(f"Layer {layer.name}:\n\t- Layers: {len(layer)}\n\t- Type: {layer.geom_type.name}\n\t- Features: {layer.num_feat}")
-
-        # country instances
-        logging.info("loading country instances")
-        country_counter = 0
-        for feature in ds[0]:
-            # calculate geometry data
-            geom = GEOSGeometry(str(feature.geom))
-            polygon_list = []
-            for polygon in geom:
-                polygon_list.append(polygon)
-            geom_geos = MultiPolygon(polygon_list)
-            # create instance
-            name = feature.get('NAME_0')
-            if name == 'Spain':
-                SPAIN = Country.objects.create(name='España',geo=geom_geos)
-            Country.objects.create(name=name,geo=geom_geos)
-            country_counter += 1
-            logging.info(f"Country instance created for: {name}")
-
-        locations_file = 'datasets/locations.json'
-        locations = json.loads(open(locations_file).read())
-
-        # REGIONS
-
-        geo_file='datasets/gadm36_ESP.json'    # geo boundary data for regions
-        geo_data = json.loads(open(geo_file).read())
-        logging.info("Starting Region creation")
-        print("Starting Region creation")
-        region_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.region':
-                logging.info(f"Creating Region instance {location['fields']['name']}...")
-                new_region = None
-                name = location['fields']['name']
-                # Bypass for lack of geo data on ceuta and melilla
-                if name in ('Ceuta', 'Melilla'):
-                    new_region = Region.objects.create(name=name, country=SPAIN, id=location['pk'])
-                    logging.info(f"Region {name} created (without GADM data)")
-                else:
-                    for feature in ds[1]:
-                        if feature.get('NAME_1') == name:
-                            logging.debug(f"Region {name} found in GADM data")
-                            # calculate geometry data
-                            geom = GEOSGeometry(str(feature.geom))
-                            polygon_list = []
-                            for polygon in geom:
-                                polygon_list.append(polygon)
-                            geom_geos = MultiPolygon(polygon_list)
-                            # create instance
-                            new_region = Region.objects.create(
-                                name=name, country=SPAIN, geo=geom_geos, id=location['pk']
-                                )
-                            logging.info(f"Region {name} created")
-                            region_counter += 1
-                            break
-                if new_region is None:
-                    logging.warning(f"No region named {name} found in  GADM data")
-
-        # PROVINCES
-        print("Starting Province creation")
-        logging.info("Starting Province creation")
-        province_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.province':
-                logging.info(f"Creating Province instance {location['fields']['name']}...")
-                name = location['fields']['name']
-                new_province = None
-                for feature in ds[2]:
-                    # import ipdb; ipdb.set_trace()
-                    if feature.get('NAME_2') == name:
-                        logging.debug(f"Province {name} found in GADM data")
-                        # calculate geometry data
-                        geom = GEOSGeometry(str(feature.geom))
-                        polygon_list = []
-                        for polygon in geom:
-                            polygon_list.append(polygon)
-                        geom_geos = MultiPolygon(polygon_list)
-                        # create instance
-                        import ipdb; ipdb.set_trace()
-                        parent_region = Region.objects.get(name=feature.get('NAME_1'))
-                        new_province = Province.objects.create(
-                            name=name, region=parent_region, geo=geom_geos, id=location['pk']
-                            )
-                        logging.info(f"Province {name} created")
-                        region_counter += 1
-                        break
-                if new_province is None:
-                    logging.warning(f"No province named {name} found in  GADM data")
-
-        # CITIES
-        print("Starting City creation")
-        logging.info("Starting City creation")
-        city_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.city':
-                name = location['fields']['name']
-                City.objects.create(name=name, province=Province.objects.get(id=location['fields']['province']), id=location['pk'])
-                city_counter += 1
-
-        logging.info(f"Region instances created: {region_counter}")
-        logging.info(f"Province instances created: {province_counter}")
-        logging.info(f"City instances created: {city_counter}")
-
-        print(f"Region instances created: {region_counter}")
-        print(f"Province instances created: {province_counter}")
-        print(f"City instances created: {city_counter}")
-

core/management/commands/loadgisdata.py

@@ -12,6 +12,14 @@ from django.contrib.gis.gdal import DataSource
 from geo.models import Country, Region, Province, City
 
 
+logging.basicConfig(
+    filename='logs/loadgisdata.log',
+    filemode='w',
+    format='%(levelname)s:%(message)s',
+    level=logging.INFO,
+    )
+
+
 class Command(BaseCommand):
 
     help = 'Load GIS data into database'
@@ -52,8 +60,8 @@ class Command(BaseCommand):
                 country_counter += 1
                 logging.info(f"Country instance created for: {name}")
             except Exception as e:
-                logging.error(f"[{type(e)}] {str(e)}")
-                import ipdb; ipdb.set_trace()
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
+                # import ipdb; ipdb.set_trace()
 
         # region instances
         logging.info("loading region instances")
@@ -74,8 +82,8 @@ class Command(BaseCommand):
                 region_counter += 1
                 logging.info(f"Region instance created for: {name}")
             except Exception as e:
-                logging.error(f"[{type(e)}] {str(e)}")
-                import ipdb; ipdb.set_trace()
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
+                # import ipdb; ipdb.set_trace()
 
         # province instances
         logging.info("loading province instances")
@@ -90,14 +98,14 @@ class Command(BaseCommand):
                     polygon_list.append(polygon)
                 geom_geos = MultiPolygon(polygon_list)
                 # get name and parent
-                name = feature['properties']['NAME_2']
+                name = feature.get('NAME_2')
                 parent_region = Region.objects.get(name=feature.get('NAME_1'))
                 # create instance
                 Province.objects.create(name=name, geo=geom_geos, region=parent_region)
                 province_counter += 1
                 logging.info(f"Province instance created for: {name}")
             except Exception as e:
-                logging.error(f"[{type(e)}] {str(e)}")
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
                 import ipdb; ipdb.set_trace()
 
         # city instances
@@ -114,16 +122,16 @@ class Command(BaseCommand):
                 geom_geos = MultiPolygon(polygon_list)
 
                 # get name and parent
-                name = feature['properties']['NAME_3']
-                parent_province = Region.objects.get(name=feature.get('NAME_2'))
+                name = feature.get('NAME_3')
+                parent_province = Province.objects.get(name=feature.get('NAME_2'))
                 City.objects.create(name=name, geo=geom_geos, province=parent_province)
                 city_counter += 1
                 logging.debug(f"City instance created for: {name}")
             except Exception as e:
                 logging.error(f"[{type(e)}] {str(e)}")
-                import ipdb; ipdb.set_trace()
+                # import ipdb; ipdb.set_trace()
 
         logging.info(f"Country instances created: {country_counter}")
         logging.info(f"Region instances created: {region_counter}")
-        logging.info(f"Province instances created: {[province_counter]}")
+        logging.info(f"Province instances created: {province_counter}")
         logging.info(f"City instances created: {city_counter}")