improvements in product loader

products/utils.py

@@ -1,11 +1,16 @@
 import logging
+import datetime
 
 from django.db.models import Q
 from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity
 from django.db.models import Max, Min
+from django.conf import settings
+
+import requests
 
 from products.models import Product
-
+from companies.models import Company
+from history.models import HistorySync
 
 
 def extract_search_filters(result_set):
@@ -210,3 +215,76 @@ def find_related_products_v4(keyword):
     ).order_by('-similarity')
 
     return set(products_qs)
+
+
+def product_loader(csv_reader, user):
+    """
+    Parse csv data and extract:
+
+    - product data
+
+    Return counts
+    """
+    counter = 0
+    # create historysync instance
+    history = HistorySync.objects.create(company=user.company, sync_date=datetime.datetime.now())
+
+    for row in csv_reader:
+        # trim strings
+        for key in row:
+            if row[key]: row[key] = row[key].strip().lower()
+
+        # check required data
+        if '' in (row['nombre-producto'], row['descripcion'], row['precio'], row['categoria']):
+            logging.error(f"Required data missing: {row}")
+            continue
+
+        try:
+            # TODO: if tags is empty, auto-generate tags
+
+            # assemble instance data
+            product_data = {
+                    'company': user.company,
+                    'name': row['nombre-producto'].strip(),
+                    'description': row['descripcion'].strip(),
+                    #'url': row['url'].strip(),
+                    #'price': row['precio'].strip(),
+                    #'shipping_cost': row['gastos-envio'].strip(),
+                    #'shipping_terms': row['cond-envio'].strip(),
+                    #'discount': row['descuento'].strip(),
+                    #'stock': row['stock'].strip(),
+                    #'tags': row['tags'].strip(),
+                    #'category': row['categoria'].strip(),
+                    #'identifiers': row['identificadores'].strip(),
+                    #'history': history,
+                    'creator': user,
+            }
+
+            product = Product.objects.create(**product_data)
+            # image logo data
+            if row['imagen'] is not None:
+                try:
+                    # get image
+                    headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
+                    response = requests.get(row['imagen'], stream=True, headers=headers)
+                    assert(response.status_code==200)
+                    response.raw.decode_content = True
+                    image = Image.open(response.raw)
+                    # save using File object
+                    img_io = BytesIO()
+                    image.save(img_io, format=image.format)
+                    product.image.save(f"{product.company.company_name}{product.name}.{image.format.lower()}", File(img_io), save=False)
+                    product.save()
+                except AssertionError as e:
+                    logging.error(f"Source image [{row['imagen']}] not reachable: {response.status_code}")
+                except Exception as e:
+                    logging.error(f"Could not add image to product from [{row['imagen']}]: {str(e)}")
+            logging.info(f"Created Product {product.id}")
+            counter += 1
+        except Exception as e:
+            import ipdb; ipdb.set_trace()
+            logging.error(f"Could not parse {row}")
+
+    history.quantity = counter
+    history.save()
+    return counter