adde pagination support to woocommerce product migration

products/models.py

@@ -25,7 +25,7 @@ class Product(models.Model):
     name = models.CharField('Nombre del producto', max_length=500, null=True, blank=True)
     description = models.TextField('Descripción', null=True, blank=True)
     image = models.ImageField('Imagen del producto', upload_to='products/', null=True, blank=True)
-    url = models.URLField('URL del producto', null=True, blank=True)
+    url = models.URLField('URL del producto', null=True, blank=True, max_length=1000)
     price = models.DecimalField('Precio', max_digits=10, decimal_places=2, null=True, blank=True)
     shipping_cost = models.DecimalField('Gastos de envío', max_digits=10, decimal_places=2, null=True, blank=True)
     shipping_terms = models.TextField('Condiciones de envío', null=True, blank=True)

utils/woocommerce.py

@@ -32,6 +32,59 @@ def get_wcapi_instance(url, key, secret, version="wc/v3"):
     return wcapi
 
 
+PRODUCT_FIELDS = [f.name for f in Product._meta.get_fields()]
+
+
+def create_imported_product(info, company, history, user):
+    # extract m2m field data
+    tags = [t.get('name') for t in info.pop('tags')]
+    attributes = [t.get('name') for t in info.pop('attributes')]
+    # prepare instance data
+    instance_data = {
+        'company':company.id,
+        'creator': user.id if user is not None else None,
+        'history': history.id,
+        'url': info['permalink'],
+        }
+    # parse the product info
+    for key in info:
+        if key in PRODUCT_FIELDS:
+            instance_data[key] = info[key]
+    # remove unwanted fields
+    instance_data.pop('id')
+
+    # alternative method
+    serializer = ProductSerializer(data=instance_data)
+    if serializer.is_valid():
+        try:
+            new = Product.objects.create(**serializer.validated_data)
+            new.tags = tags
+            new.attributes = attributes
+            new.save()
+        except Exception as e:
+            logging.error(f"Could not create product instance: {str(e)}")
+            return None
+        try:
+            # get image
+            headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
+            image_url = info['images'][0]['src']
+            response = requests.get(image_url, stream=True, headers=headers)
+            response.raw.decode_content = True
+            image = Image.open(response.raw)
+            # save using File object
+            img_io = BytesIO()
+            image.save(img_io, format='JPEG')
+            new.image.save(f"{new.name}-{new.sku}.jpg", File(img_io), save=False)
+            new.save()
+        except Exception as e:
+            logging.error(f"Could not add image to product: {str(e)}")
+        return new
+    else:
+        logging.error(f"{serializer.errors}")
+        return []
+
+
+
 def migrate_shop_products(url, key, secret, user=None, version="wc/v3"):
     """Tries to connect to WooCommerce site @ url with given credentials
 
@@ -52,18 +105,28 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"):
         print(f"Could not find Company with URL: {url}")
         return None
 
+    new_products = []
+    page = 1
     # list products
-    response = wcapi.get('products/')
+    while True:
+        try:
+            response = wcapi.get('products/', params={'per_page': 10, 'page': page})
+            page += 1
             if response.status_code == 200:
                 products = response.json()
             elif response.status_code == 401:
                 logging.error(f"{response.status_code} [{response.url}]: {response.json()}")
                 return None
             else:
-        import ipdb; ipdb.set_trace()
                 logging.error(f"Could not load products from {response.url}: [{response.status_code}]")
                 print(f"Could not load products fom {response.url}: [{response.status_code}]")
                 return None
+        except requests.exceptions.ReadTimeout as e:
+            logging.error(f"Timeout reading backend: {str(e)}")
+            return None
+        # exit loop if no more products
+        if not products:
+            break
 
         # create HistorySync instance and link to every product created
         history = HistorySync.objects.create(
@@ -71,69 +134,20 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"):
             sync_date=datetime.datetime.now(),
         )
 
-    product_fields = [f.name for f in Product._meta.get_fields()]
         counter = 0
-    products_created = []
         for product in products:
-        # extract m2m field data
-        tags = [t.get('name') for t in product.pop('tags')]
-        attributes = [t.get('name') for t in product.pop('attributes')]
-        # prepare instance data
-        instance_data = {
-            'company':company.id,
-            'creator': user.id if user is not None else None,
-            'history': history.id,
-            'url': product['permalink'],
-            }
-        # parse the product info
-        for key in product:
-            if key in product_fields:
-                instance_data[key] = product[key]
-        # remove unwanted fields
-        instance_data.pop('id')
-
-        # alternative method
-        serializer = ProductSerializer(data=instance_data)
-        if serializer.is_valid():
-            try:
-                new = Product.objects.create(**serializer.validated_data)
-                new.tags = tags
-                new.attributes = attributes
-                new.save()
-            except Exception as e:
-                import ipdb; ipdb.set_trace()
-                logging.error(f"Could not create product instance: {str(e)}")
-                continue
-            try:
-                # get image
-                headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
-                image_url = product['images'][0]['src']
-                response = requests.get(image_url, stream=True, headers=headers)
-                response.raw.decode_content = True
-                image = Image.open(response.raw)
-                # save using File object
-                img_io = BytesIO()
-                image.save(img_io, format='JPEG')
-                new.image.save(f"{new.name}-{new.sku}.jpg", File(img_io), save=False)
-                new.save()
-            except Exception as e:
-                logging.error(f"Could not add image to product: {str(e)}")
-        else:
-            import ipdb; ipdb.set_trace()
-            logging.error(f"{serializer.errors}")
-            continue
-
-        products_created.append(new)
+            new = create_imported_product(product, company, history, user)
+            new_products.append(new)
             counter += 1
-        logging.info(f"Product '{instance_data.get('name')}' created")
-        print(f"Product '{instance_data.get('name')}' created")
+            logging.info(f"Product '{new.name}' created")
+            print(f"Product '{new.name}' created")
 
         # update history.quantity
         history.quantity = counter
         history.save()
 
-    print(f"Products created: {counter}")
-    return products_created
+    print(f"Products created: {len(new_products)}")
+    return new_products