From 74cdd0ca10e7800b65987869a2a7a5609a3a662a Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 24 Feb 2021 12:36:32 +0000 Subject: [PATCH] adde pagination support to woocommerce product migration --- products/models.py | 2 +- utils/woocommerce.py | 168 +++++++++++++++++++++++-------------------- 2 files changed, 92 insertions(+), 78 deletions(-) diff --git a/products/models.py b/products/models.py index cd739dd..84c7060 100644 --- a/products/models.py +++ b/products/models.py @@ -25,7 +25,7 @@ class Product(models.Model): name = models.CharField('Nombre del producto', max_length=500, null=True, blank=True) description = models.TextField('Descripción', null=True, blank=True) image = models.ImageField('Imagen del producto', upload_to='products/', null=True, blank=True) - url = models.URLField('URL del producto', null=True, blank=True) + url = models.URLField('URL del producto', null=True, blank=True, max_length=1000) price = models.DecimalField('Precio', max_digits=10, decimal_places=2, null=True, blank=True) shipping_cost = models.DecimalField('Gastos de envío', max_digits=10, decimal_places=2, null=True, blank=True) shipping_terms = models.TextField('Condiciones de envío', null=True, blank=True) diff --git a/utils/woocommerce.py b/utils/woocommerce.py index 9645210..f6b60a0 100644 --- a/utils/woocommerce.py +++ b/utils/woocommerce.py @@ -32,6 +32,59 @@ def get_wcapi_instance(url, key, secret, version="wc/v3"): return wcapi +PRODUCT_FIELDS = [f.name for f in Product._meta.get_fields()] + + +def create_imported_product(info, company, history, user): + # extract m2m field data + tags = [t.get('name') for t in info.pop('tags')] + attributes = [t.get('name') for t in info.pop('attributes')] + # prepare instance data + instance_data = { + 'company':company.id, + 'creator': user.id if user is not None else None, + 'history': history.id, + 'url': info['permalink'], + } + # parse the product info + for key in info: + if key in PRODUCT_FIELDS: + instance_data[key] = info[key] + # remove unwanted fields + instance_data.pop('id') + + # alternative method + serializer = ProductSerializer(data=instance_data) + if serializer.is_valid(): + try: + new = Product.objects.create(**serializer.validated_data) + new.tags = tags + new.attributes = attributes + new.save() + except Exception as e: + logging.error(f"Could not create product instance: {str(e)}") + return None + try: + # get image + headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"} + image_url = info['images'][0]['src'] + response = requests.get(image_url, stream=True, headers=headers) + response.raw.decode_content = True + image = Image.open(response.raw) + # save using File object + img_io = BytesIO() + image.save(img_io, format='JPEG') + new.image.save(f"{new.name}-{new.sku}.jpg", File(img_io), save=False) + new.save() + except Exception as e: + logging.error(f"Could not add image to product: {str(e)}") + return new + else: + logging.error(f"{serializer.errors}") + return [] + + + def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): """Tries to connect to WooCommerce site @ url with given credentials @@ -52,88 +105,49 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): print(f"Could not find Company with URL: {url}") return None + new_products = [] + page = 1 # list products - response = wcapi.get('products/') - if response.status_code == 200: - products = response.json() - elif response.status_code == 401: - logging.error(f"{response.status_code} [{response.url}]: {response.json()}") - return None - else: - import ipdb; ipdb.set_trace() - logging.error(f"Could not load products from {response.url}: [{response.status_code}]") - print(f"Could not load products fom {response.url}: [{response.status_code}]") - return None + while True: + try: + response = wcapi.get('products/', params={'per_page': 10, 'page': page}) + page += 1 + if response.status_code == 200: + products = response.json() + elif response.status_code == 401: + logging.error(f"{response.status_code} [{response.url}]: {response.json()}") + return None + else: + logging.error(f"Could not load products from {response.url}: [{response.status_code}]") + print(f"Could not load products fom {response.url}: [{response.status_code}]") + return None + except requests.exceptions.ReadTimeout as e: + logging.error(f"Timeout reading backend: {str(e)}") + return None + # exit loop if no more products + if not products: + break - # create HistorySync instance and link to every product created - history = HistorySync.objects.create( - company=company, - sync_date=datetime.datetime.now(), - ) + # create HistorySync instance and link to every product created + history = HistorySync.objects.create( + company=company, + sync_date=datetime.datetime.now(), + ) - product_fields = [f.name for f in Product._meta.get_fields()] - counter = 0 - products_created = [] - for product in products: - # extract m2m field data - tags = [t.get('name') for t in product.pop('tags')] - attributes = [t.get('name') for t in product.pop('attributes')] - # prepare instance data - instance_data = { - 'company':company.id, - 'creator': user.id if user is not None else None, - 'history': history.id, - 'url': product['permalink'], - } - # parse the product info - for key in product: - if key in product_fields: - instance_data[key] = product[key] - # remove unwanted fields - instance_data.pop('id') + counter = 0 + for product in products: + new = create_imported_product(product, company, history, user) + new_products.append(new) + counter += 1 + logging.info(f"Product '{new.name}' created") + print(f"Product '{new.name}' created") - # alternative method - serializer = ProductSerializer(data=instance_data) - if serializer.is_valid(): - try: - new = Product.objects.create(**serializer.validated_data) - new.tags = tags - new.attributes = attributes - new.save() - except Exception as e: - import ipdb; ipdb.set_trace() - logging.error(f"Could not create product instance: {str(e)}") - continue - try: - # get image - headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"} - image_url = product['images'][0]['src'] - response = requests.get(image_url, stream=True, headers=headers) - response.raw.decode_content = True - image = Image.open(response.raw) - # save using File object - img_io = BytesIO() - image.save(img_io, format='JPEG') - new.image.save(f"{new.name}-{new.sku}.jpg", File(img_io), save=False) - new.save() - except Exception as e: - logging.error(f"Could not add image to product: {str(e)}") - else: - import ipdb; ipdb.set_trace() - logging.error(f"{serializer.errors}") - continue + # update history.quantity + history.quantity = counter + history.save() - products_created.append(new) - counter += 1 - logging.info(f"Product '{instance_data.get('name')}' created") - print(f"Product '{instance_data.get('name')}' created") - - # update history.quantity - history.quantity = counter - history.save() - - print(f"Products created: {counter}") - return products_created + print(f"Products created: {len(new_products)}") + return new_products