we have a working version with some bugs

todo: fix url for combination
todo: fix price for some combination products (e.g. id:1637)
todo: remove name from combination or find proper title

freezed_requirements.txt

@@ -0,0 +1,44 @@
+asgiref==3.5.0
+attrs==21.4.0
+black==22.1.0
+certifi==2021.10.8
+charset-normalizer==2.0.12
+click==8.0.4
+coverage==6.3.1
+defusedxml==0.7.1
+dicttoxml==1.7.16
+Django==4.0.2
+django-crontab==0.7.1
+django-stubs==1.9.0
+django-stubs-ext==0.3.1
+djangorestframework==3.13.1
+djangorestframework-xml==2.0.0
+docformatter==1.4
+environ==1.0
+flake8==4.0.1
+idna==3.3
+iniconfig==1.1.1
+mccabe==0.6.1
+mypy==0.931
+mypy-extensions==0.4.3
+packaging==21.3
+pathspec==0.9.0
+platformdirs==2.5.0
+pluggy==1.0.0
+py==1.11.0
+pycodestyle==2.8.0
+pyflakes==2.4.0
+pyparsing==3.0.7
+pytest==7.0.1
+pytest-cov==3.0.0
+pytest-django==4.5.2
+pytz==2021.3
+requests==2.27.1
+sqlparse==0.4.2
+toml==0.10.2
+tomli==2.0.1
+types-pytz==2021.3.5
+types-PyYAML==6.0.4
+typing_extensions==4.1.1
+untokenize==0.1.1
+urllib3==1.26.8

requirements.txt

@@ -41,3 +41,4 @@ typing-extensions==4.1.1
 untokenize==0.1.1
 urllib3==1.26.8
 djangorestframework-xml==2.0.0
+django-crontab==0.7.1

src/telecaster/Clients/BaseClient.py

@@ -5,13 +5,18 @@ class BaseClient:
     def get(self, url, timeout: int = 5):
         response = requests.get(url, timeout=timeout)
         if response and response.status_code == 200:
-            return response.content
+            return response
         else:
             print(response.text)
             return response.status_code
 
-    def post(self, url, body):
-        pass
+    def post(self, url, body, timeout: int = 5, **kwargs):
+        response = requests.post(url, body, timeout=timeout)
+        if response and response.status_code == 200:
+            return response
+        else:
+            print(response.text)
+            return response.status_code
 
     def patch(self):
         pass

src/telecaster/Clients/CosmohomeClient.py

@@ -1,6 +1,9 @@
 import hashlib
 import struct
 from collections import defaultdict
+from datetime import datetime
+
+from dicttoxml import dicttoxml
 
 from telecaster.Clients.PrestaShopClient import PrestaShopClient
 
@@ -38,7 +41,7 @@ class CosmohomeClient(PrestaShopClient):
         cache_dict = self.caches[cache_key]
         for item in values:
             key = get_nested(item, id_key_path)
-            cache_dict[key] = item
+            cache_dict[str(key)] = item
 
     def load_cache(self):
         categories = self.get_categories()  # {'display': '[id,name, link_rewrite, id_parent]'})
@@ -46,20 +49,14 @@ class CosmohomeClient(PrestaShopClient):
         feature_values = self.get_feature_values()
         product_options = self.get_product_options()
         product_option_values = self.get_product_option_values()
+        specific_prices = self.get_specific_prices()
+
+        self.update_cache('specific_prices', specific_prices, ['id_product_attribute'])
+        self.update_cache('specific_prices', specific_prices, ['id_product'])
         # tax_rule_groups = self.get_tax_rule_groups()
         # tax_rules = self.get_tax_rules()
         # taxes = self.get_taxes()
 
-        categories = map(lambda x: x['category'], categories)
-        features = map(lambda x: x['product_feature'], features)
-        feature_values = map(lambda x: x['product_feature_value'], feature_values)
-        product_options = map(lambda x: x['product_option'], product_options)
-        product_option_values = map(lambda x: x['product_option_value'],
-                                    product_option_values)
-        # tax_rule_groups = map(lambda x: x['tax_rule_group'], tax_rule_groups)
-        # tax_rules = map(lambda x: x['tax_rule'], tax_rules)
-        # taxes = map(lambda x: x['tax'], taxes)
-
         self.update_cache('categories', categories, ['id'])
         self.update_cache('features', features, ['id'])
         self.update_cache('feature_values', feature_values, ['id'])
@@ -69,22 +66,13 @@ class CosmohomeClient(PrestaShopClient):
         # self.update_cache('tax_rules', tax_rules, ['id'])
         # self.update_cache('taxes', taxes, ['id'])
 
-    def parse_product(self, product, combinations):
-        product = product.get('product')
+    def parse_product(self, product, combinations=None):
         product_combinations = self.get_product_combinations(product, combinations)
 
-        # Move this to pre-computed cache when perform batches
-        id_product_attributes = [c.get('id') for c in product_combinations]
-        sp = self.get_specific_prices(
-            {'filter[id_product_attribute]': f"{'|'.join(id_product_attributes)}"})
-        sp = list(map(lambda x: x['specific_price'], sp))
-        self.update_cache('specific_prices', sp, ['id_product_attribute'])
-        self.update_cache('specific_prices', sp, ['id_product'])
-        ## END OF Specific prices calculations
-
         serialized_product = {
             "productId": product.get('id'),
             "title": self.sanitize_language(product.get('name')),
+            # TODO FIX PRODUCT_URL - it has no domain + path!!
             "productURL": product.get('link_rewrite'),
             # add to the url the attributes like #/attr_id/attr_id/...
             "Category": self.get_category_tree(product.get('id_category_default'),
@@ -108,10 +96,13 @@ class CosmohomeClient(PrestaShopClient):
         for combination in product_combinations:
             serialized_combinations.append({
                 **serialized_product,
-                "productId": product.get('id') + numHash(combination.get('reference'), 10),
+                "productId": str(product.get('id')) + numHash(
+                    combination.get('reference'), 10),
                 "Barcode": combination.get('ean13') or serialized_product.get(
                     'Barcode'),
-                "Name": combination.get('name'),
+                # TODO: Product-URL should be overriden for combination
+                # TODO: Name is not a good idea it does not exist
+                # "Name": combination.get('name'),
                 "weight": product.get('weight'),
                 "MPN": combination.get('reference') or combination.get(
                     'mpn') or serialized_product.get('MPN'),
@@ -127,7 +118,8 @@ class CosmohomeClient(PrestaShopClient):
                 "Size": self.get_size(product) or serialized_product.get('Size'),
             })
 
-        return [serialized_product] if not combinations else serialized_combinations
+        return [
+            serialized_product] if not serialized_combinations else serialized_combinations
 
     def sanitize_language(self, value):
         if isinstance(value, str):
@@ -139,7 +131,7 @@ class CosmohomeClient(PrestaShopClient):
 
     def get_category_tree(self, category_id, min_depth=0):
         categories = self.caches['categories']
-        category = categories[category_id]
+        category = categories.get(category_id)
         category_tree = []
         while category and int(category.get('level_depth')) > min_depth:
             if category:
@@ -148,7 +140,7 @@ class CosmohomeClient(PrestaShopClient):
             else:
                 break
 
-        return [self.sanitize_language(c) for c in category_tree]
+        return ' > '.join([self.sanitize_language(c) for c in category_tree])
 
     def generate_image_url(self, product):
         image_id = product.get('id_default_image')
@@ -156,9 +148,8 @@ class CosmohomeClient(PrestaShopClient):
         return f"{CANONICAL_DOMAIN}/{image_id}-thickbox_default/{link_rewrite}.jpg"
 
     def get_color(self, product):
-        options = product.get('associations').get('product_option_values')
+        options = product.get('associations').get('product_option_values') or []
         for option in options:
-            option = option.get('product_option_value')
             option_value = self.caches['product_option_values'].get(option.get('id'))
             if option_value and option_value.get(
                     'id_attribute_group') in COLOR_PRODUCT_OPTION_IDS:
@@ -166,9 +157,8 @@ class CosmohomeClient(PrestaShopClient):
         return None
 
     def get_size(self, product):
-        options = product.get('associations').get('product_option_values')
+        options = product.get('associations').get('product_option_values') or []
         for option in options:
-            option = option.get('product_option_value')
             option_value = self.caches['product_option_values'].get(option.get('id'))
             if option_value and option_value.get(
                     'id_attribute_group') in SIZE_PRODUCT_OPTION_IDS:
@@ -177,15 +167,15 @@ class CosmohomeClient(PrestaShopClient):
 
     def get_product_combinations(self, product, combinations):
         if combinations:
-            combinations = map(lambda x: x['combination'], combinations)
             self.update_cache('combinations', combinations, ['id'])
 
         combinations = self.caches['combinations']
 
         product_combinations = product.get('associations').get('combinations')
-        product_combination_ids = map(lambda x: x['combination']['id'],
-                                      product_combinations)
+        if not product_combinations:
+            return []
 
+        product_combination_ids = map(lambda x: x['id'], product_combinations)
         return [combinations.get(cid) for cid in product_combination_ids]
 
     def calculate_final_price(self, product, price=None):
@@ -195,10 +185,82 @@ class CosmohomeClient(PrestaShopClient):
         price_with_tax = float(price) * (1 + TAX)
 
         sp = self.caches['specific_prices'].get(product.get('id'))
-        discount = sp.get('reduction')
-        type = sp.get('reduction_type')
-        if type == 'percentage':
-            final_price_with_tax = price_with_tax * (1 - float(discount))
+        if sp:
+            discount = sp.get('reduction')
+            type = sp.get('reduction_type')
+            if type == 'percentage':
+                final_price_with_tax = price_with_tax * (1 - float(discount))
+            else:
+                final_price_with_tax = price_with_tax - float(discount)
         else:
-            final_price_with_tax = price_with_tax - float(discount)
+            final_price_with_tax = price_with_tax
         return round(final_price_with_tax, 2)
+
+
+def batched_array(array, batch_size=50):
+    l = len(array)
+    offset = 0
+    while offset < l:
+        yield array[offset:offset + batch_size]
+        offset += batch_size
+
+
+def batch_products(client, params=None, batch_size=200, hard_limit=100000):
+    offset = 0
+    while True:
+        products = client.get_products({
+            'display': 'full',
+            **(params if params else {}),
+            'limit': f"{offset},{batch_size}",
+            'sort': "[id_ASC]"
+        })
+        if not products:
+            break
+
+        combinations = client.get_combinations({
+            'filter[id_product]': f"[{'|'.join([str(p.get('id')) for p in products])}]"
+        })
+        client.update_cache("combinations", combinations, ['id'])
+
+        offset += len(products)
+        print(f"Fetched {offset} products")
+        yield from products
+        if offset >= hard_limit:
+            print(f"Hard limit reached {hard_limit}")
+            break
+
+
+def generate_xml():
+    client = CosmohomeClient()
+    print("Loading cache")
+    client.load_cache()
+    print("Cache loaded")
+
+    products = []
+    for product in batch_products(client, params={'filter[active]': '1'},
+                                  batch_size=200):
+        products += client.parse_product(product)
+
+    xml = dicttoxml(
+        products,
+        cdata=True,
+        xml_declaration=False,
+        custom_root="products",
+        attr_type=False,
+        item_func=lambda x: x[:-1]
+    )
+    xml_string = xml.decode('utf-8')
+    final_xml = (
+        ''
+        '<store name="cosmohome.gr" url="https://cosmohome.gr" encoding="utf8">'
+        f'<date>{datetime.now().strftime("%Y-%m-%d %H:%M")}</date>'
+        f"{xml_string}"
+        '</store>'
+    )
+    return final_xml
+
+
+def generate_xml_task():
+    xml = generate_xml()
+    with open('cosmohome_products.xml', 'w') as file:
+        file.write(xml)

src/telecaster/Clients/PrestaShopClient.py

@@ -13,58 +13,67 @@ class PrestaShopClient(BaseClient):
         updated_params = {**(params or {}), 'ws_key': self.token}
         return f'{self.base_url}/api{url}?{urllib.parse.urlencode(updated_params)}'
 
-    def request(self, url, params=None, timeout=20):
+    def request_xml(self, url, params=None, timeout=20):
         constructed_params = {'display': 'full', **(params or {})}
         full_url = self.build_url(url, constructed_params)
-        xml_content = self.get(full_url, timeout=timeout)
-        xml = ElementTree.fromstring(xml_content)
+        response = self.get(full_url, timeout=timeout)
+        xml = ElementTree.fromstring(response.content)
         result = [parse_xml_to_json(child) for child in list(xml)[0]]
         return result
 
-    def get_products(self, params=None):
+    def request(self, url, params=None, timeout=20):
         constructed_params = {
             'display': 'full',
-            **(params or {}),
+            'output_format': 'JSON',
+            'ps_method': 'GET',
+            **(params or {})
         }
-        full_url = self.build_url('/products', constructed_params)
-        # we need to parse these since the response is in XML
-        # check parsers folder for relevant functions/classes
-        products = self.get(full_url)
-
-        xml_products = ElementTree.fromstring(products)
-        products = list(xml_products)[0]
-        result = [parse_xml_to_json(child) for child in products]
+        full_url = self.build_url(url, constructed_params)
+        response = self.get(full_url, timeout=timeout)
+        return response.json()
 
-        return result
+    def get_products(self, params=None):
+        return self.normalise(self.request('/products', params=params), 'products')
 
     def get_categories(self, params=None):
-        return self.request('/categories', params)
+        return self.normalise(self.request('/categories', params), 'categories')
 
     def get_features(self, params=None):
-        return self.request('/product_features', params)
+        return self.normalise(self.request('/product_features', params), 'product_features')
+
     def get_feature_values(self, params=None):
-        return self.request('/product_feature_values', params)
+        return self.normalise(self.request('/product_feature_values', params), 'product_feature_values')
 
     def get_combinations(self, params=None):
-        return self.request('/combinations', params)
+        return self.normalise(self.request('/combinations', params), 'combinations')
 
     def get_product_options(self, params=None):
-        return self.request('/product_options', params)
+        return self.normalise(self.request('/product_options', params), 'product_options')
 
     def get_product_option_values(self, params=None):
-        return self.request('/product_option_values', params)
+        return self.normalise(self.request('/product_option_values', params), 'product_option_values')
 
     def get_images(self, params=None):
-        return self.request('/images', params)
+        return self.normalise(self.request('/images', params), 'images')
 
     def get_tax_rule_groups(self, params=None):
-        return self.request('/tax_rule_groups', params)
+        return self.normalise(self.request('/tax_rule_groups', params), 'tax_rule_groups')
 
     def get_tax_rules(self, params=None):
-        return self.request('/tax_rules', params)
+        return self.normalise(self.request('/tax_rules', params), 'tax_rules')
 
     def get_taxes(self, params=None):
-        return self.request('/taxes', params)
+        return self.normalise(self.request('/taxes', params), 'taxes')
 
     def get_specific_prices(self, params=None):
-        return self.request('/specific_prices', params)
+        return self.normalise(self.request('/specific_prices', params), 'specific_prices')
+
+    @staticmethod
+    def normalise(array, key):
+        if array is None:
+            return []
+        elif isinstance(array, dict):
+            return array.get(key)
+        elif isinstance(array, list):
+            return array
+        return []

src/telecaster/settings.py

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django_crontab',
     'telecaster',
 ]
 
@@ -122,3 +123,8 @@ STATIC_URL = 'static/'
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+
+CRONJOBS = [
+    ('*/23 * * * *', 'src.telecaster.Clients.CosmohomeClient.generate_xml_task'),
+]

src/telecaster/views/CosmohomeView.py

@@ -1,60 +1,14 @@
-from rest_framework import views, viewsets
+from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.request import Request
-from rest_framework.response import Response
-from rest_framework_xml.renderers import XMLRenderer
-from django.http import JsonResponse
+from django.http import HttpResponse
 
-from ..Clients.CosmohomeClient import CosmohomeClient
-from ..Clients.PrestaShopClient import PrestaShopClient
-from ..serializers import XmlGeneratorSerializer
-from ..sanitizers.prestashop_backoffice_api_sanitizer import PrestashopBackOfficeApiSanitizer
+from ..Clients.CosmohomeClient import generate_xml_task
 
 
 class CosmohomeView(viewsets.ViewSet):
-    renderer_classes = [
-        XMLRenderer,
-    ]
-
     @classmethod
     @action(methods=['get'], detail=False, url_path='generate/xml')
-    def generate_xml(cls, request: Request, *args, **kwargs) -> JsonResponse:
-        client = CosmohomeClient()
-        client.load_cache()
-
-        products = client.get_products({'limit': 5, 'filter[active]': 1, 'filter[id]': 1615})
-        combinations = client.get_combinations({
-            'filter[id_product]': f"[{'|'.join([p.get('product').get('id') for p in products])}]"
-        })
-
-        client.parse_product(products[0], combinations)
-
-        # retrieve categories first since they are needed for products
-        # categories = prestashop_client.get_categories({
-        #     'display': '[id,name, link_rewrite]'
-        # })
-        # features = prestashop_client.get_features()
-        # feature_values = prestashop_client.get_feature_values()
-        # products = prestashop_client.get_products({'limit': 5, 'filter[active]': 1, 'filter[id]': 1615})
-        # combinations = prestashop_client.get_combinations({
-        #     'filter[id_product]': f"[{'|'.join([p.get('product').get('id') for p in products])}]"
-        # })
-        #
-        # product_options = prestashop_client.get_product_options()
-        # product_option_values = prestashop_client.get_product_option_values()
-
-        sanitizer = PrestashopBackOfficeApiSanitizer()
-
-        # sanitized_categories = sanitizer.categories(categories)
-        #
-        # categories_dict = {category.category_id: vars(category) for category in sanitized_categories}
-        # sanitized_products = sanitizer.products(products)
-        #
-        # # serialize response
-        # serializer = XmlGeneratorSerializer()
-        # response_xml = serializer.for_xml(
-        #     products=sanitized_products, categories=categories_dict, site_url=url
-        # )
-
-        # return Response(response_xml, content_type='text/xml')
-        return Response("ok")
+    def generate_xml(cls, request: Request, *args, **kwargs) -> HttpResponse:
+        final_xml = generate_xml_task()
+        return HttpResponse('ok')