From 874068881de82edeed1fcb6869cd23ac69c8ccb4 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 26 Feb 2021 12:47:23 +0000 Subject: [PATCH] improvements to csv loading code --- core/tests.py | 4 ++-- core/utils.py | 46 +++++++++++++++++++++++++++++++++++------- datasets/test_coop.csv | 18 ++++++++--------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/core/tests.py b/core/tests.py index 10312d3..6a249ae 100644 --- a/core/tests.py +++ b/core/tests.py @@ -452,8 +452,8 @@ class LoadCoopManagerTestCase(APITestCase): # send in request response = self.client.post(self.endpoint, files) - - # check re sponse + import ipdb; ipdb.set_trace() + # check response self.assertEqual(response.status_code, 200) # check for object creation self.assertEquals(company_count + 5, self.company_model.objects.count()) diff --git a/core/utils.py b/core/utils.py index b94035d..14a363f 100644 --- a/core/utils.py +++ b/core/utils.py @@ -8,6 +8,7 @@ from django.template.loader import render_to_string from django.core.mail import EmailMessage from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.conf import settings +from django.core.validators import validate_email, EmailValidator, URLValidator, ValidationError from rest_framework_simplejwt.tokens import RefreshToken @@ -101,7 +102,7 @@ def reformat_google_taxonomy(file_name): destination_file.write(line) -def coop_loader(csv_reader, request): +def coop_loader(csv_reader, request=None): """ Parse csv data and extract: @@ -113,16 +114,45 @@ def coop_loader(csv_reader, request): coop_counter = 0 user_counter = 0 for row in csv_reader: + # trim strings + for key in row: + if row[key]: row[key] = row[key].strip() + # import ipdb; ipdb.set_trace() if '' in (row['cif'], row['nombre-coop'], row['email']): logging.error(f"Required data missing: {row}") continue + # validate email + try: + validate_email(row['email']) + except ValidationError: + logging.warning(f"Invalid email value '{row['email']}', skipped") + continue + # validate URLs + if row['url'].startswith('http') is not True: + row['url'] = 'http://' + row['url'] + if row['logo-url'].startswith('http') is not True: + row['logo-url'] = 'http://' + row['logo-url'] + validator = URLValidator() + try: + validator(row['url']) + except ValidationError: + logging.warning(f"Invalid url value '{row['url']}', skipped") + row['url'] = None + try: + validator(row['logo-url']) + except ValidationError: + logging.warning(f"Invalid url value '{row['logo-url']}', skipped") + row['logo-url'] = None + # validate boolean + + # create instances try: coop_data = { - 'cif': row['cif'].strip(), - 'company_name': row['nombre-coop'].strip(), - 'short_name': row['nombre-corto'].strip(), - 'shop': bool(row['es-tienda'].strip()), - 'shop_link': row['url'].strip(), + 'cif': row['cif'], + 'company_name': row['nombre-coop'], + 'short_name': row['nombre-corto'], + 'shop': bool(row['es-tienda']), + 'shop_link': row['url'], } coop = Company.objects.create(**coop_data) logging.info(f"Created Coop: {coop_data}") @@ -130,10 +160,12 @@ def coop_loader(csv_reader, request): coop_user = User.objects.create_user(email=row['email'], company=coop, role='COOP_MANAGER', is_active=False) # send confirmation email - send_verification_email(request, coop_user) + if request is not None: + send_verification_email(request, coop_user) logging.info(f"Created User: {coop_user}") user_counter += 1 except Exception as e: + import ipdb; ipdb.set_trace() logging.error(f"Could not parse {row}") return coop_counter, user_counter diff --git a/datasets/test_coop.csv b/datasets/test_coop.csv index 6080b1e..87cacbd 100644 --- a/datasets/test_coop.csv +++ b/datasets/test_coop.csv @@ -1,9 +1,9 @@ -email,cif,nombre-coop,nombre-corto,url,es-tienda -, 1223432214L, FEWQ4FEWQ COOP, fc, tienda1.com, True -dsfds@mail.com,, FEW2QFEWQ COOP, fc, tienda2.com, True -ghjhg@mail.com, 122343214L,, fc, tienda3.com, True -xcv@mail.com, 12343214L, FEWQ2FEWQ COOP,, tienda4.com, True -cvc@mail.com, 1879783214L, 2FEWQFEWQ COOP, fc,, True -bvbc@mail.com, 5653214L, FEW2QFEWQ COOP, fc, tienda6.com, -kjk@mail.com, 54326543H, FE2WQF2EWQ COOP, fc, tienda7.com, True -yuyu@mail.com, 12343214L, F2EWQFEWQ COOP, fc, tienda8.com, True \ No newline at end of file +email,cif,nombre-coop,nombre-corto,url,es-tienda,logo-url,telefono,provincia,localidad,cp,direccion +, 1223432214L, FEWQ4FEWQ COOP, fc, tienda1.com, True,,,,,, +dsfds@mail.com,, FEW2QFEWQ COOP, fc, tienda2.com, True,,,,,, +ghjhg@mail.com, 122343214L,, fc, tienda3.com, True,,,,,, +xcv@mail.com, 12343214L, FEWQ2FEWQ COOP,, tienda4.com, True,,,,,, +cvc@mail.com, 1879783214L, 2FEWQFEWQ COOP, fc,, True,,,,,, +bvbc@mail.com, 5653214L, FEW2QFEWQ COOP, fc, tienda6.com,,,,,, +kjk@mail.com, 54326543H, FE2WQF2EWQ COOP, fc, tienda7.com, True,,,,,, +yuyu@mail.com, 12343214L, F2EWQFEWQ COOP, fc, tienda8.com, True,,,,,, \ No newline at end of file