From 873ec25a92fd05f81eee59459cdd165ad21468fe Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 24 Feb 2021 11:26:07 +0000 Subject: [PATCH 1/2] improvement to WC migration --- companies/views.py | 4 +++- products/serializers.py | 2 +- utils/woocommerce.py | 36 ++++++++++++++++++++---------------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/companies/views.py b/companies/views.py index e784a6c..477b4b2 100644 --- a/companies/views.py +++ b/companies/views.py @@ -144,7 +144,9 @@ class CompanyViewSet(viewsets.ModelViewSet): products = woocommerce.migrate_shop_products( instance.web_link, credentials['key'], - credentials['secret']) + credentials['secret'], + request.user, + ) message = {'details': f'{len(products)} products added for {instance.company_name}'} else: message = {"error": 'Credentials have wrong format'} diff --git a/products/serializers.py b/products/serializers.py index 6bd7497..f262d90 100644 --- a/products/serializers.py +++ b/products/serializers.py @@ -15,7 +15,7 @@ class ProductSerializer(TaggitSerializer, serializers.ModelSerializer): class Meta: model = Product - exclude = ['created', 'updated', 'creator'] + fields = '__all__' class SearchResultSerializer(TaggitSerializer, serializers.ModelSerializer): diff --git a/utils/woocommerce.py b/utils/woocommerce.py index 349da8e..90ddbea 100644 --- a/utils/woocommerce.py +++ b/utils/woocommerce.py @@ -8,6 +8,7 @@ This file holds the functions necesary to: """ import logging from io import BytesIO +import datetime from woocommerce import API import requests @@ -17,6 +18,7 @@ from django.core.files import File from companies.models import Company from products.models import Product from products.serializers import ProductSerializer +from history.models import HistorySync def get_wcapi_instance(url, key, secret, version="wc/v3"): @@ -30,7 +32,7 @@ def get_wcapi_instance(url, key, secret, version="wc/v3"): return wcapi -def migrate_shop_products(url, key, secret, version="wc/v3"): +def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): """Tries to connect to WooCommerce site @ url with given credentials If succesful, returns list of Product instances created @@ -62,11 +64,23 @@ def migrate_shop_products(url, key, secret, version="wc/v3"): print(f"Could not load products fom {url}: [{response.status_code}]") return None + # create HistorySync instance and link to every product created + history = HistorySync.objects.create( + company=company, + sync_date=datetime.datetime.now(), + ) + product_fields = [f.name for f in Product._meta.get_fields()] counter = 0 products_created = [] for product in products: - instance_data = {'company':company.id} + # prepare instance data + instance_data = { + 'company':company.id, + 'creator': user.id if user is not None else None, + 'history': history.id, + 'url': product['permalink'], + } # parse the product info for key in product: if key in product_fields: @@ -77,20 +91,6 @@ def migrate_shop_products(url, key, secret, version="wc/v3"): tags = instance_data.pop('tags') attributes = instance_data.pop('attributes') - # create instance with serializer - ''' - serializer = ProductSerializer(data=instance_data) - if serializer.is_valid(): - new = serializer.save() - if tags: - new.tags.set(tags) - if attributes: - new.attributes.set(attributes) - new.save() - else: - logging.error(f"{serializer.errors}") - continue - ''' # alternative method serializer = ProductSerializer(data=instance_data) if serializer.is_valid(): @@ -129,6 +129,10 @@ def migrate_shop_products(url, key, secret, version="wc/v3"): logging.info(f"Product {instance_data.get('name')} created") print(f"Product {instance_data.get('name')} created") + # update history.quantity + history.quantity = counter + history.save() + print(f"Products created: {counter}") return products_created From 62eb7399782fa1257f5d422ed52c950bc05aa056 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 24 Feb 2021 11:54:11 +0000 Subject: [PATCH 2/2] more improvements to WC migration --- requirements.txt | 1 + utils/woocommerce.py | 29 ++++++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7b21041..fd96d61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ drf-extra-fields==3.0.4 django-ipware==3.0.2 geoip2==4.1.0 woocommerce==2.1.1 +# manually install `pip install --default-timeout=100 future` to avoid wcapi to timeout # required for production django-anymail[amazon_ses]==8.2 boto3==1.17.11 diff --git a/utils/woocommerce.py b/utils/woocommerce.py index 90ddbea..9645210 100644 --- a/utils/woocommerce.py +++ b/utils/woocommerce.py @@ -53,15 +53,16 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): return None # list products - response = wcapi.get('/products/') + response = wcapi.get('products/') if response.status_code == 200: products = response.json() elif response.status_code == 401: logging.error(f"{response.status_code} [{response.url}]: {response.json()}") return None else: - logging.error(f"Could not load products from {url}: [{response.status_code}]") - print(f"Could not load products fom {url}: [{response.status_code}]") + import ipdb; ipdb.set_trace() + logging.error(f"Could not load products from {response.url}: [{response.status_code}]") + print(f"Could not load products fom {response.url}: [{response.status_code}]") return None # create HistorySync instance and link to every product created @@ -74,6 +75,9 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): counter = 0 products_created = [] for product in products: + # extract m2m field data + tags = [t.get('name') for t in product.pop('tags')] + attributes = [t.get('name') for t in product.pop('attributes')] # prepare instance data instance_data = { 'company':company.id, @@ -87,28 +91,22 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): instance_data[key] = product[key] # remove unwanted fields instance_data.pop('id') - # extract m2m field data - tags = instance_data.pop('tags') - attributes = instance_data.pop('attributes') # alternative method serializer = ProductSerializer(data=instance_data) if serializer.is_valid(): try: new = Product.objects.create(**serializer.validated_data) - if tags: - new.tags.set(tags) - if attributes: - new.attributes.set(attributes) + new.tags = tags + new.attributes = attributes new.save() except Exception as e: + import ipdb; ipdb.set_trace() logging.error(f"Could not create product instance: {str(e)}") continue try: # get image - headers={ -"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" -} + headers={"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"} image_url = product['images'][0]['src'] response = requests.get(image_url, stream=True, headers=headers) response.raw.decode_content = True @@ -121,13 +119,14 @@ def migrate_shop_products(url, key, secret, user=None, version="wc/v3"): except Exception as e: logging.error(f"Could not add image to product: {str(e)}") else: + import ipdb; ipdb.set_trace() logging.error(f"{serializer.errors}") continue products_created.append(new) counter += 1 - logging.info(f"Product {instance_data.get('name')} created") - print(f"Product {instance_data.get('name')} created") + logging.info(f"Product '{instance_data.get('name')}' created") + print(f"Product '{instance_data.get('name')}' created") # update history.quantity history.quantity = counter