improvements to csv loading code

core/tests.py

@@ -452,8 +452,8 @@ class LoadCoopManagerTestCase(APITestCase):
 
         # send in request
         response = self.client.post(self.endpoint, files)
-
+        import ipdb; ipdb.set_trace()
-        # check re sponse
+        # check response
         self.assertEqual(response.status_code, 200)
         # check for object creation
         self.assertEquals(company_count + 5, self.company_model.objects.count())

core/utils.py

@@ -8,6 +8,7 @@ from django.template.loader import render_to_string
 from django.core.mail import EmailMessage
 from django.contrib.auth.tokens import PasswordResetTokenGenerator
 from django.conf import settings
+from django.core.validators import validate_email, EmailValidator, URLValidator, ValidationError
 
 from rest_framework_simplejwt.tokens import RefreshToken
 
@@ -101,7 +102,7 @@ def reformat_google_taxonomy(file_name):
         destination_file.write(line)
 
 
-def coop_loader(csv_reader, request):
+def coop_loader(csv_reader, request=None):
     """
     Parse csv data and extract:
 
@@ -113,16 +114,45 @@ def coop_loader(csv_reader, request):
     coop_counter = 0
     user_counter = 0
     for row in csv_reader:
+        # trim strings
+        for key in row:
+            if row[key]: row[key] = row[key].strip()
+        # import ipdb; ipdb.set_trace()
         if '' in (row['cif'], row['nombre-coop'], row['email']):
             logging.error(f"Required data missing: {row}")
             continue
+        # validate email
+        try:
+            validate_email(row['email'])
+        except ValidationError:
+            logging.warning(f"Invalid email value '{row['email']}', skipped")
+            continue
+        # validate URLs
+        if row['url'].startswith('http') is not True:
+            row['url'] = 'http://' + row['url']
+        if row['logo-url'].startswith('http') is not True:
+            row['logo-url'] = 'http://' + row['logo-url']
+        validator = URLValidator()
+        try:
+            validator(row['url'])
+        except ValidationError:
+            logging.warning(f"Invalid url value '{row['url']}', skipped")
+            row['url'] = None
+        try:
+            validator(row['logo-url'])
+        except ValidationError:
+            logging.warning(f"Invalid url value '{row['logo-url']}', skipped")
+            row['logo-url'] = None
+        # validate boolean
+
+        # create instances
         try:
             coop_data = {
-                'cif': row['cif'].strip(),
+                'cif': row['cif'],
-                'company_name': row['nombre-coop'].strip(),
+                'company_name': row['nombre-coop'],
-                'short_name': row['nombre-corto'].strip(),
+                'short_name': row['nombre-corto'],
-                'shop': bool(row['es-tienda'].strip()),
+                'shop': bool(row['es-tienda']),
-                'shop_link': row['url'].strip(),
+                'shop_link': row['url'],
             }
             coop = Company.objects.create(**coop_data)
             logging.info(f"Created Coop: {coop_data}")
@@ -130,10 +160,12 @@ def coop_loader(csv_reader, request):
 
             coop_user = User.objects.create_user(email=row['email'], company=coop, role='COOP_MANAGER', is_active=False)
             # send confirmation email
-            send_verification_email(request, coop_user)
+            if request is not None:
+                send_verification_email(request, coop_user)
             logging.info(f"Created User: {coop_user}")
             user_counter += 1
         except Exception as e:
+            import ipdb; ipdb.set_trace()
             logging.error(f"Could not parse {row}")
     return coop_counter, user_counter
 

datasets/test_coop.csv

@@ -1,9 +1,9 @@
-email,cif,nombre-coop,nombre-corto,url,es-tienda
+email,cif,nombre-coop,nombre-corto,url,es-tienda,logo-url,telefono,provincia,localidad,cp,direccion
-, 1223432214L, FEWQ4FEWQ COOP, fc, tienda1.com, True
+, 1223432214L, FEWQ4FEWQ COOP, fc, tienda1.com, True,,,,,,
-dsfds@mail.com,, FEW2QFEWQ COOP, fc, tienda2.com, True
+dsfds@mail.com,, FEW2QFEWQ COOP, fc, tienda2.com, True,,,,,,
-ghjhg@mail.com, 122343214L,, fc, tienda3.com, True
+ghjhg@mail.com, 122343214L,, fc, tienda3.com, True,,,,,,
-xcv@mail.com, 12343214L, FEWQ2FEWQ COOP,, tienda4.com, True
+xcv@mail.com, 12343214L, FEWQ2FEWQ COOP,, tienda4.com, True,,,,,,
-cvc@mail.com, 1879783214L, 2FEWQFEWQ COOP, fc,, True
+cvc@mail.com, 1879783214L, 2FEWQFEWQ COOP, fc,, True,,,,,,
-bvbc@mail.com, 5653214L, FEW2QFEWQ COOP, fc, tienda6.com,
+bvbc@mail.com, 5653214L, FEW2QFEWQ COOP, fc, tienda6.com,,,,,,
-kjk@mail.com, 54326543H, FE2WQF2EWQ COOP, fc, tienda7.com, True
+kjk@mail.com, 54326543H, FE2WQF2EWQ COOP, fc, tienda7.com, True,,,,,,
-yuyu@mail.com, 12343214L, F2EWQFEWQ COOP, fc, tienda8.com, True
+yuyu@mail.com, 12343214L, F2EWQFEWQ COOP, fc, tienda8.com, True,,,,,,