Merge branch 'development' of https://bitbucket.org/enreda/back-latienda into diego

README.md

@@ -37,7 +37,7 @@ python manage.py migrate
 
 ## Location data
 
-To load initial location data use: `python manage.py addgeo`
+To load initial location data use: `python manage.py loadgisdata`
 
 
 ## Endpoints

core/management/commands/addgeo.py

@@ -1,105 +0,0 @@
-import logging
-import json
-
-from django.core.management.base import BaseCommand
-from django.contrib.gis.geos import GEOSGeometry, MultiPolygon
-
-from geo.models import City, Region, Province, Country
-
-logging.basicConfig(
-    filename='logs/addgeo.log',
-    filemode='w',
-    format='%(levelname)s:%(message)s',
-    level=logging.INFO,
-    )
-
-class Command(BaseCommand):
-
-    'help' = 'Load geographic dataset'
-
-    def handle(self, *args, **kwargs):
-        print('Deleting all instances of Country, Region, Province, City')
-        logging.info('Deleting all instances of Country, Region, Province, City')
-        City.objects.all().delete()
-        Province.objects.all().delete()
-        Region.objects.all().delete()
-        Country.objects.all().delete()
-
-        # create country for spain
-        country = Country.objects.create(name='España')
-
-        locations_file = 'datasets/locations.json'
-        locations = json.loads(open(locations_file).read())
-
-        # REGIONS
-
-        geo_file='datasets/gadm36_ESP.json'    # geo boundary data for regions
-        geo_data = json.loads(open(geo_file).read())
-
-        logging.info("Starting Region creation")
-        print("Starting Region creation")
-        region_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.region':
-                logging.info(f"Creating Region instance {location['fields']['name']}...")
-                new_region = None
-
-                name = location['fields']['name']
-                # Bypass for lack of geo data on ceuta and melilla
-                if name in ('Ceuta', 'Melilla'):
-                    new_region = Region.objects.create(name=name, country=country, id=location['pk'])
-                    logging.info(f"Region {name} created (without GADM data)")
-                else:
-                    for feature in geo_data['features']:
-                        if feature['properties']['NAME_1'] == name:
-                            logging.debug(f"Region {name} found in GADM data")
-                            # calculate geometry data
-                            geom = GEOSGeometry(str(feature['geometry']))
-                            if feature['geometry']['type'] == "MultiPolygon":
-                                poly_list = []
-                                for poly in geom:
-                                    poly_list.append(poly)
-                            else:
-                                poly_list = geom
-                            geom_geos = MultiPolygon(poly_list)
-                            # create instance
-                            new_region = Region.objects.create(
-                                name=name, country=country, geo=geom_geos, id=location['pk']
-                                )
-                            logging.info(f"Region {name} created")
-                            region_counter += 1
-                            break
-                if new_region is None:
-                    logging.warning(f"No region named {name} found in  GADM data")
-
-        # PROVINCES
-        print("Starting Province creation")
-        logging.info("Starting Province creation")
-        province_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.province':
-                logging.info(f"Creating Province instance {location['fields']['name']}...")
-                name = location['fields']['name']
-                # get parent region
-                parent_region = Region.objects.get(id=location['fields']['region'])
-                Province.objects.create(name=name, region=parent_region, id=location['pk'])
-                province_counter += 1
-
-        # CITIES
-        print("Starting City creation")
-        logging.info("Starting City creation")
-        city_counter = 0
-        for location in locations:
-            if location['model'] == 'locations.city':
-                name = location['fields']['name']
-                City.objects.create(name=name, province=Province.objects.get(id=location['fields']['province']), id=location['pk'])
-                city_counter += 1
-
-        logging.info(f"Region instances created: {region_counter}")
-        logging.info(f"Province instances created: {province_counter}")
-        logging.info(f"City instances created: {city_counter}")
-
-        print(f"Region instances created: {region_counter}")
-        print(f"Province instances created: {province_counter}")
-        print(f"City instances created: {city_counter}")
-

core/management/commands/loadgisdata.py

@@ -0,0 +1,137 @@
+import logging
+import json
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from django.contrib.gis.geos import GeometryCollection, GEOSGeometry
+from django.contrib.gis.geos import MultiPolygon
+from django.contrib.gis.gdal import DataSource
+
+
+from geo.models import Country, Region, Province, City
+
+
+logging.basicConfig(
+    filename='logs/loadgisdata.log',
+    filemode='w',
+    format='%(levelname)s:%(message)s',
+    level=logging.INFO,
+    )
+
+
+class Command(BaseCommand):
+
+    help = 'Load GIS data into database'
+
+    def handle(self, *args, **kwargs):
+        logging.info('Deleting all instances of Country, Region, Province, City')
+        City.objects.all().delete()
+        Province.objects.all().delete()
+        Region.objects.all().delete()
+        Country.objects.all().delete()
+        print('Deleted all instances of Country, Region, Province, City')
+
+        ds = DataSource('datasets/gadm36_ESP.gpkg')
+
+        for layer in ds:
+            print(f"Layer {layer.name}:\n \
+            \t- Layers: {len(layer)}\n \
+            \t- Type: {layer.geom_type.name}\n\
+            \t- Features: {layer.num_feat}")
+
+        # country instances
+        logging.info("loading country instances")
+        print('Creating Country instances')
+        country_counter = 0
+        for feature in ds[0]:
+            try:
+                # calculate geometry data
+                geom = GEOSGeometry(str(feature.geom))
+                polygon_list = []
+                for polygon in geom:
+                    polygon_list.append(polygon)
+                geom_geos = MultiPolygon(polygon_list)
+                # create instance
+                name = feature.get('NAME_0')
+                if name == 'Spain':
+                    SPAIN = Country.objects.create(name='España',geo=geom_geos)
+                Country.objects.create(name=name,geo=geom_geos)
+                country_counter += 1
+                logging.info(f"Country instance created for: {name}")
+            except Exception as e:
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
+                # import ipdb; ipdb.set_trace()
+
+        # region instances
+        logging.info("loading region instances")
+        print('Creating Region instances')
+        region_counter = 0
+        for feature in ds[1]:
+            try:
+                # calculate geometry data
+                geom = GEOSGeometry(str(feature.geom))
+                polygon_list = []
+                for polygon in geom:
+                    polygon_list.append(polygon)
+                geom_geos = MultiPolygon(polygon_list)
+                # get region name
+                name = feature.get('NAME_1')
+                # create instance
+                Region.objects.create(name=name, geo=geom_geos, country=SPAIN)
+                region_counter += 1
+                logging.info(f"Region instance created for: {name}")
+            except Exception as e:
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
+                # import ipdb; ipdb.set_trace()
+
+        # province instances
+        logging.info("loading province instances")
+        print('Creating Province instances')
+        province_counter = 0
+        for feature in ds[2]:
+            try:
+                # calculate geometry data
+                geom = GEOSGeometry(str(feature.geom))
+                polygon_list = []
+                for polygon in geom:
+                    polygon_list.append(polygon)
+                geom_geos = MultiPolygon(polygon_list)
+                # get name and parent
+                name = feature.get('NAME_2')
+                parent_region = Region.objects.get(name=feature.get('NAME_1'))
+                # create instance
+                Province.objects.create(name=name, geo=geom_geos, region=parent_region)
+                province_counter += 1
+                logging.info(f"Province instance created for: {name}")
+            except Exception as e:
+                logging.error(f"[{name}][{type(e)}] {str(e)}")
+                import ipdb; ipdb.set_trace()
+
+        # city instances
+        logging.info("loading city instances")
+        print('Creating City instances')
+        city_counter = 0
+        for feature in ds[4]:
+            try:
+                # calculate geometry data
+                geom = GEOSGeometry(str(feature.geom))
+                polygon_list = []
+                for polygon in geom:
+                    polygon_list.append(polygon)
+                geom_geos = MultiPolygon(polygon_list)
+
+                # get name and parent
+                name = feature.get('NAME_3')
+                parent_province = Province.objects.get(name=feature.get('NAME_2'))
+                City.objects.create(name=name, geo=geom_geos, province=parent_province)
+                city_counter += 1
+                logging.debug(f"City instance created for: {name}")
+            except Exception as e:
+                logging.error(f"[{type(e)}] {str(e)}")
+                # import ipdb; ipdb.set_trace()
+
+        logging.info(f"Country instances created: {country_counter}")
+        logging.info(f"Region instances created: {region_counter}")
+        logging.info(f"Province instances created: {province_counter}")
+        logging.info(f"City instances created: {city_counter}")

geo/models.py

@@ -6,6 +6,7 @@ class Country(models.Model):
     Country model
     """
     name = models.CharField(max_length = 100)
+    geo = models.MultiPolygonField(null=True)
 
     # internal
     created = models.DateTimeField('date of creation', auto_now_add=True)
@@ -45,6 +46,7 @@ class Province(models.Model):
     """
     name = models.CharField(max_length = 100)
     region = models.ForeignKey(Region, on_delete=models.DO_NOTHING, related_name='province')
+    geo = models.MultiPolygonField(null=True)
 
     # internal
     created = models.DateTimeField('date of creation', auto_now_add=True)
@@ -64,6 +66,7 @@ class City(models.Model):
     """
     name = models.CharField(max_length = 250)
     province = models.ForeignKey(Province, on_delete=models.DO_NOTHING, related_name='city', null=True)
+    geo = models.MultiPolygonField(null=True)
 
     # internal
     created = models.DateTimeField('date of creation', auto_now_add=True)

products/models.py

@@ -32,9 +32,9 @@ class Product(models.Model):
     update_date = models.DateTimeField('Fecha de actualización de producto', null=True, blank=True)
     discount = models.DecimalField('Descuento', max_digits=5, decimal_places=2, null=True, blank=True)
     stock = models.PositiveIntegerField('Stock', null=True)
-    tags = TagField(force_lowercase=True,max_count=5, tree=True)
+    tags = TagField(force_lowercase=True, max_count=20, tree=True)
     category = SingleTagField(null=True) # main tag category
-    attributes = TagField(force_lowercase=True,max_count=5, tree=True)
+    attributes = TagField(force_lowercase=True, max_count=20, tree=True)
     identifiers = models.TextField('Identificador único de producto', null=True, blank=True)
 
     # internal

products/tests.py

@@ -368,20 +368,24 @@ class ProductSearchTest(TestCase):
         self.model = Product
         self.factory = ProductFactory
         # create admin user
-        self.admin_email = f"admin_user@mail.com"
+        self.admin_email = "admin_user@mail.com"
         self.password = ''.join(random.choices(string.ascii_uppercase, k = 10))
         self.admin_user = CustomUserFactory(email=self.admin_email, password=self.password, is_staff=True, is_active=True)
         # create regular user
-        self.reg_email = f"user@mail.com"
+        self.reg_email = "user@mail.com"
         self.user = CustomUserFactory(email=self.reg_email, is_active=True)
         self.user.set_password(self.password)
         self.user.save()
 
     def test_anon_user_can_search(self):
         expected_instances = [
-            self.factory(description="zapatos"),
+            self.factory(description="zapatos verdes"),
             self.factory(tags="rojos"),
         ]
+        unexpected_instances = [
+            self.factory(description="chanclas"),
+            self.factory(tags="azules"),
+        ]
 
         self.factory(tags="azul")