fix sanitizer,serializer for prestashop/skroutz implementation

.idea/telecaster.iml

@@ -3,6 +3,7 @@
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/virtualenv" />
+      <excludeFolder url="file://$MODULE_DIR$/env" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

lint_types/report.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuite errors="0" failures="1" name="mypy" skips="0" tests="1" time="1.375">
+  <testcase classname="mypy" file="mypy" line="1" name="mypy-py3_8-darwin" time="1.375">
+    <failure message="mypy produced messages">telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:1: error: Relative import climbs too many namespaces
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:2: error: Relative import climbs too many namespaces
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:6: error: Function is missing a type annotation
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:41: error: Function is missing a type annotation
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:42: error: Call to untyped function "product" in typed context
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:45: error: Function is missing a type annotation
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:53: error: Function is missing a type annotation
+telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py:54: error: Call to untyped function "category" in typed context
+telecaster/telecaster/parsers/parse_xml_to_json.py:1: error: Function is missing a type annotation
+telecaster/telecaster/parsers/parse_xml_to_json.py:5: error: Call to untyped function "parse_xml_to_json" in typed context
+telecaster/telecaster/parsers/parse_xml_to_json.py:12: error: Function is missing a type annotation
+telecaster/telecaster/parsers/parse_xml_to_json.py:17: error: Call to untyped function "parse_prestashop_xml_products" in typed context
+telecaster/telecaster/parsers/cdata_parser.py:1: error: Function is missing a type annotation
+telecaster/telecaster/Clients/BaseClient.py:1: error: Library stubs not installed for "requests" (or incompatible with Python 3.8)
+telecaster/telecaster/Clients/BaseClient.py:1: note: Hint: "python3 -m pip install types-requests"
+telecaster/telecaster/Clients/BaseClient.py:1: note: (or run "mypy --install-types" to install all missing stub packages)
+telecaster/telecaster/Clients/BaseClient.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+telecaster/telecaster/Clients/BaseClient.py:5: error: Function is missing a type annotation
+telecaster/telecaster/Clients/BaseClient.py:13: error: Function is missing a type annotation
+telecaster/telecaster/Clients/BaseClient.py:16: error: Function is missing a return type annotation
+telecaster/telecaster/Clients/BaseClient.py:16: note: Use "-&gt; None" if function does not return a value
+telecaster/telecaster/Clients/BaseClient.py:19: error: Function is missing a return type annotation
+telecaster/telecaster/Clients/BaseClient.py:19: note: Use "-&gt; None" if function does not return a value
+telecaster/telecaster/Clients/BaseClient.py:22: error: Function is missing a return type annotation
+telecaster/telecaster/Clients/BaseClient.py:22: note: Use "-&gt; None" if function does not return a value
+telecaster/telecaster/settings.py:28: error: Need type annotation for "ALLOWED_HOSTS" (hint: "ALLOWED_HOSTS: List[&lt;type&gt;] = ...")
+telecaster/telecaster/serializers/XmlGeneratorSerializer.py:10: error: Class cannot subclass "Serializer" (has type "Any")
+telecaster/telecaster/serializers/XmlGeneratorSerializer.py:12: error: Function is missing a type annotation
+telecaster/telecaster/serializers/XmlGeneratorSerializer.py:16: error: Function is missing a type annotation
+telecaster/telecaster/serializers/XmlGeneratorSerializer.py:29: error: Function is missing a type annotation
+telecaster/telecaster/serializers/XmlGeneratorSerializer.py:71: error: Call to untyped function "validate_product_for_skroutz" of "XmlGeneratorSerializer" in typed context
+telecaster/telecaster/Clients/PrestaShopClient.py:4: error: Relative import climbs too many namespaces
+telecaster/telecaster/Clients/PrestaShopClient.py:7: error: Class cannot subclass "BaseClient" (has type "Any")
+telecaster/telecaster/Clients/PrestaShopClient.py:8: error: Function is missing a type annotation
+telecaster/telecaster/Clients/PrestaShopClient.py:12: error: Function is missing a type annotation
+telecaster/telecaster/Clients/PrestaShopClient.py:16: error: Function is missing a type annotation
+telecaster/telecaster/Clients/PrestaShopClient.py:21: error: Call to untyped function "build_url" in typed context
+telecaster/telecaster/Clients/PrestaShopClient.py:32: error: Function is missing a type annotation
+telecaster/telecaster/Clients/PrestaShopClient.py:34: error: Call to untyped function "build_url" in typed context
+telecaster/telecaster/views/XmlGeneratorView.py:11: error: Class cannot subclass "APIView" (has type "Any")
+telecaster/telecaster/views/XmlGeneratorView.py:17: error: Function is missing a type annotation for one or more arguments
+telecaster/telecaster/views/XmlGeneratorView.py:21: error: Function is missing a type annotation for one or more arguments
+telecaster/telecaster/views/XmlGeneratorView.py:39: error: Call to untyped function "for_xml" of "XmlGeneratorSerializer" in typed context
+telecaster/telecaster/views/XmlGeneratorView.py:42: error: Returning Any from function declared to return "JsonResponse"
+telecaster/telecaster/tests/conftest.py:8: error: Function is missing a return type annotation
+telecaster/telecaster/tests/conftest.py:13: error: Function is missing a type annotation
+telecaster/telecaster/tests/conftest.py:14: error: Function is missing a type annotation
+telecaster/telecaster/tests/conftest.py:24: error: Function is missing a type annotation
+telecaster/telecaster/tests/conftest.py:25: error: Function is missing a type annotation
+telecaster/telecaster/tests/conftest.py:35: error: Function is missing a return type annotation
+telecaster/telecaster/tests/conftest.py:42: error: Function is missing a type annotation
+telecaster/telecaster/tests/conftest.py:49: error: Function is missing a type annotation
+telecaster/telecaster/tests/parsers/test_presta_shop_parser.py:9: error: Function is missing a return type annotation
+telecaster/telecaster/tests/parsers/test_presta_shop_parser.py:15: error: Function is missing a return type annotation
+telecaster/telecaster/tests/parsers/test_presta_shop_parser.py:20: error: Function is missing a type annotation
+telecaster/telecaster/tests/parsers/test_presta_shop_parser.py:28: error: Function is missing a type annotation</failure>
+  </testcase>
+</testsuite>

requirements.txt

@@ -6,6 +6,7 @@ certifi==2021.10.8
 charset-normalizer==2.0.12
 click==8.0.4
 coverage==6.3.1
+defusedxml==0.7.1
 Django==4.0.2
 django-stubs==1.9.0
 django-stubs-ext==0.3.1

telecaster/telecaster/Clients/PrestaShopClient.py

@@ -1,5 +1,7 @@
-from .BaseClient import BaseClient
+import xml.etree.ElementTree as ElementTree
 import urllib.parse
+from .BaseClient import BaseClient
+from ..parsers.parse_xml_to_json import parse_xml_to_json
 
 
 class PrestaShopClient(BaseClient):
@@ -13,7 +15,7 @@ class PrestaShopClient(BaseClient):
 
     def get_products(self, params={}):
         constructed_params = {
-            'display': '[name,id,id_default_image,id_category_default,price,wholesale_price,manufacturer_name,ean13,delivery_in_stock,additional_shipping_cost,description,quantity]',
+            'display': '[name, id, id_default_image, id_category_default,price,wholesale_price,manufacturer_name,ean13,delivery_in_stock,additional_shipping_cost,description,quantity]',
             **params,
         }
         full_url = self.build_url('/products', constructed_params)
@@ -21,10 +23,22 @@ class PrestaShopClient(BaseClient):
         # check parsers folder for relevant functions/classes
         products = self.get(full_url)
 
-        return products
+        xml_products = ElementTree.fromstring(products)
+        products = list(xml_products)[0]
+        result = [parse_xml_to_json(child) for child in products]
+
+        return result
 
     def get_categories(self, params={}):
         constructed_params = {'display': '[id,name]', **params}
         full_url = self.build_url('/categories', constructed_params)
-        categories = self.get(full_url)
-        return categories
+        xml_categories_content = self.get(full_url)
+        xml_categories = ElementTree.fromstring(xml_categories_content)
+        categories = list(xml_categories)[0]
+        # result = {}
+        # for child in categories:
+        #     child_ojb = parse_xml_to_json(child)
+        #     result[child_ojb['id']] = child_ojb
+        # Alternatively if we wanna convert to Array check use the below
+        result = [parse_xml_to_json(child) for child in categories]
+        return result

telecaster/telecaster/models/CategoryModel.py

@@ -0,0 +1,4 @@
+class CategoryModel:
+    def __init__(self, category_id: int, name: str):
+        self.category_id = category_id
+        self.name = name

telecaster/telecaster/models/ProductModel.py

@@ -0,0 +1,31 @@
+from .CategoryModel import CategoryModel
+
+
+class ProductModel:
+    def __init__(
+        self,
+        product_id: str,
+        name: str,
+        image: str,
+        price: float,
+        wholesale_price: float,
+        manufacturer: str,
+        ena13: str,
+        in_stock: int,
+        additional_shipping_cost: float,
+        description: str,
+        quantity: int,
+        category: CategoryModel,
+    ):
+        self.product_id = product_id
+        self.name = name
+        self.image = image
+        self.category = category
+        self.price = price
+        self.wholesale_price = wholesale_price
+        self.manufacturer = manufacturer
+        self.ean13 = ena13
+        self.in_stock = in_stock
+        self.additional_shipping_cost = additional_shipping_cost
+        self.description = description
+        self.quantity = quantity

telecaster/telecaster/models/XmlGeneratorModel.py

@@ -1,5 +0,0 @@
-from django.db import models
-
-
-class XmlGeneratorModel:
-    pass

telecaster/telecaster/models/__init__.py

@@ -0,0 +1,4 @@
+from .ProductModel import ProductModel
+from .CategoryModel import CategoryModel
+
+__all__ = ['ProductModel', 'CategoryModel']

telecaster/telecaster/parsers/PrestaShopParser.py

@@ -1,28 +0,0 @@
-import xml.etree.ElementTree as ElementTree
-
-from .parse_xml_to_json import parse_xml_to_json
-
-
-class PrestaShopParser:
-    def __init__(self):
-        pass
-
-    @classmethod
-    # parses xml categories response to a dictionary
-    def parse_categories(self, xml_categories_content):
-        xml_categories = ElementTree.fromstring(xml_categories_content)
-        categories = list(xml_categories)[0]
-        result = {}
-        for child in categories:
-            child_ojb = parse_xml_to_json(child)
-            result[child_ojb['id']] = child_ojb
-        # Alternatively if we wanna convert to Array check use the below
-        # result = [parse_xml_to_json(child) for child in categories]
-        return result
-
-    @classmethod
-    def parse_products(self, xml_products_content):
-        xml_products = ElementTree.fromstring(xml_products_content)
-        products = list(xml_products)[0]
-        result = [parse_xml_to_json(child) for child in products]
-        return result

telecaster/telecaster/parsers/cdata_parser.py

@@ -0,0 +1,2 @@
+def cdata_parser(text=None):
+    return f"![CDATA[{text}]]"

telecaster/telecaster/parsers/skroutz_parser.py

@@ -1,2 +0,0 @@
-class SkroutzParser:
-    pass

telecaster/telecaster/sanitizers/prestashop_backoffice_api_sanitizer.py

@@ -0,0 +1,55 @@
+from ..models.ProductModel import ProductModel
+from ..models.CategoryModel import CategoryModel
+
+
+class PrestashopBackOfficeApiSanitizer:
+    def product(self, raw_product):
+        """Sanitizes a single product."""
+        if not isinstance(raw_product, dict):
+            return None
+
+        if not raw_product.get('id'):
+            return None
+
+        if not raw_product['ean13']:
+            ean13 = None
+        else:
+            ean13 = str(raw_product['ean13'])
+
+        if raw_product['delivery_in_stock'] is True:
+            in_stock = 'Y'
+        else:
+            in_stock = 'N'
+
+        sanitized_product = {
+            'product_id': int(raw_product['id']),
+            'name': raw_product.get('name', {}).get('language', ''),
+            'image': raw_product['id_default_image'],
+            'price': float(raw_product['price']),
+            'wholesale_price': float(raw_product['wholesale_price']),
+            'manufacturer': str(raw_product['manufacturer_name']),
+            'ena13': ean13,
+            'in_stock': str(in_stock),
+            'additional_shipping_cost': float(raw_product['additional_shipping_cost']),
+            'description': str(raw_product.get('description', {}).get('language', '')),
+            'quantity': int(raw_product['quantity']),
+            'category': CategoryModel(category_id=int(raw_product['id_category_default']), name=''),
+        }
+
+        return ProductModel(**sanitized_product)
+
+    def products(self, raw_products):
+        result = [self.product(raw_product) for raw_product in raw_products]
+        return result
+
+    def category(self, raw_category):
+        sanitized_category = {
+            'category_id': raw_category['id'],
+            'name': raw_category.get('name', {}).get('language', ''),
+        }
+
+        return CategoryModel(**sanitized_category)
+
+    def categories(self, raw_categories):
+        result = [self.category(raw_product) for raw_product in raw_categories]
+        return result

telecaster/telecaster/serializers/PrestaShopProductSerializer.py

@@ -1,9 +0,0 @@
-from rest_framework import serializers
-
-
-class PrestaShopProductSerializer(serializers.BaseSerializer):
-    def to_internal_value(self, data):
-        return data
-
-    def to_representation(self, instance):
-        return {'id': instance['id']}

telecaster/telecaster/serializers/XmlGeneratorSerializer.py

@@ -1,11 +1,96 @@
+import xml.etree.ElementTree as gfg
+import datetime
 from rest_framework import serializers
-from ..models import XmlGeneratorModel
+from io import BytesIO
+from ..parsers.cdata_parser import cdata_parser
+from ..types.skroutz_cdata_fields import skroutz_cdata_fields
+from ..types.skroutz_required_fields import skroutz_required_fields
 
 
 class XmlGeneratorSerializer(serializers.Serializer):
-    class Meta:
-        model = XmlGeneratorModel
-
     @classmethod
     def for_api(validated_data):
         return validated_data
+
+    @classmethod
+    def validate_product_for_skroutz(self, product):
+        for required_field in skroutz_required_fields:
+            if product.get(required_field) is None:
+                print(
+                    'Skipping product with id %s due to missing required fields: [%s]',
+                    product.get('id', 'undefined'),
+                    required_field,
+                )
+                return False
+
+        return True
+
+    @classmethod
+    def for_xml(self, products=[], categories={}):
+        """Generates an xml based on -> https://developer.skroutz.gr/el/feedspec/#xml"""
+
+        root = gfg.Element("mywebstore")
+
+        # generate current date
+        created_at = gfg.Element("created_at")
+        created_at.text = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
+        root.append(created_at)
+
+        tree = gfg.ElementTree(root)
+
+        products_element = gfg.Element('products')
+        root.append(products_element)
+
+        # convert Model to Skroutz Key
+        products = [vars(prod) for prod in products]
+        skroutz_products = []
+
+        for product in products:
+            skroutz_product = {
+                'id': str(product.get('product_id')),
+                'name': product.get('name'),
+                'link': product.get('link', 'hardcodedlin.com'),  # fix this from prestashop API
+                'image': product.get('image'),
+                'additionalimage': product.get('additionalimage'),
+                'category': categories.get(product.get('category', {}).category_id, {}).get('name', ''),
+                'price_with_vat': product.get('wholesale_price'),
+                'vat': product.get('vat'),
+                'manufacturer': product.get('manufacturer'),
+                'mpn': product.get('mpn', 'M7652C'),  # fix this from prestashop API,
+                'ean': product.get('ean13'),
+                'instock': product.get('in_stock'),
+                'availability': str(
+                    product.get('availability', 'Παραδοση 1 εως 3 ημερε')
+                ),  # fix this from prestashop API
+                'size': product.get('size'),
+                'weight': product.get('weight'),
+                'color': product.get('color'),  # maybe make it array based on skroutz
+                'description': product.get('description'),
+                'quantity': product.get('quantity'),
+            }
+            if self.validate_product_for_skroutz(skroutz_product):
+                skroutz_products.append(skroutz_product)
+
+        for product in skroutz_products:
+            product_element = gfg.Element('product')
+
+            for key, value in product.items():
+                if not value:
+                    continue
+
+                child = gfg.Element(key)
+                if key in skroutz_cdata_fields:
+                    print(key, value)
+
+                    child.text = cdata_parser(value)
+                else:
+                    child.text = value
+                product_element.append(child)
+
+            products_element.append(product_element)
+
+        # for developing purposes in order to view your xml we use the below lines
+        f = BytesIO()
+        tree.write(f, encoding='utf-8', xml_declaration=True)
+
+        return f.getvalue()

telecaster/telecaster/serializers/__init__.py

@@ -1,4 +1,3 @@
 from .XmlGeneratorSerializer import XmlGeneratorSerializer
-from .PrestaShopProductSerializer import PrestaShopProductSerializer
 
-__all__ = ['XmlGeneratorSerializer', 'PrestaShopProductSerializer']
+__all__ = ['XmlGeneratorSerializer']

telecaster/telecaster/settings.py

@@ -50,6 +50,7 @@ MIDDLEWARE = [
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
+
 ROOT_URLCONF = 'telecaster.urls'
 
 TEMPLATES = [

telecaster/telecaster/tests/parsers/test_presta_shop_parser.py

@@ -25,8 +25,9 @@ def test_categories(mock_categories_response):
     assert list(first_category.keys()) == ['id', 'name']
 
 
-def test_parse_products(mock_products_response):
-    products = PrestaShopParser.parse_products(mock_products_response)
+def test_parse_products(mock_products_response, mock_categories_response):
+    categories = PrestaShopParser.parse_categories(mock_categories_response)
+    products = PrestaShopParser.parse_products(mock_products_response, categories)
     first_product = products[0]
 
     assert len(products) == 4
@@ -43,4 +44,5 @@ def test_parse_products(mock_products_response):
         'additional_shipping_cost',
         'name',
         'description',
+        'category_name',  # not sure if this conversion is right
     ]

telecaster/telecaster/types/skroutz_cdata_fields.py

@@ -0,0 +1,8 @@
+skroutz_cdata_fields = {
+    'name': True,
+    'link': True,
+    'image': True,
+    'additionalimage': True,
+    'category': True,
+    'manufacturer': True,
+}

telecaster/telecaster/types/skroutz_required_fields.py

@@ -0,0 +1,11 @@
+skroutz_required_fields = [
+    'id',
+    'name',
+    'link',
+    'image',
+    'category',
+    'price_with_vat',
+    'availability',
+    'manufacturer',
+    'mpn',
+]

telecaster/telecaster/views/XmlGeneratorView.py

@@ -1,13 +1,18 @@
 from rest_framework import views
 from rest_framework.request import Request
-
+from rest_framework.response import Response
+from rest_framework_xml.renderers import XMLRenderer
 from django.http import JsonResponse
 from ..Clients.PrestaShopClient import PrestaShopClient
-from ..parsers.PrestaShopParser import PrestaShopParser
-from ..serializers import PrestaShopProductSerializer
+from ..serializers import XmlGeneratorSerializer
+from ..sanitizers.prestashop_backoffice_api_sanitizer import PrestashopBackOfficeApiSanitizer
 
 
 class XmlGeneratorView(views.APIView):
+    renderer_classes = [
+        XMLRenderer,
+    ]
+
     @classmethod
     def get(cls, request: Request, *args, **kwargs) -> JsonResponse:
         return JsonResponse({}, safe=False)
@@ -19,20 +24,20 @@ class XmlGeneratorView(views.APIView):
 
         prestashop_client = PrestaShopClient(base_url=url, token=token)
 
-        products_response = prestashop_client.get_products({'limit': 10})
-        categories_response = prestashop_client.get_categories()
+        # retrieve categories first since they are needed for products
+        categories = prestashop_client.get_categories()
+        products = prestashop_client.get_products({'limit': 4})
+        sanitizer = PrestashopBackOfficeApiSanitizer()
 
-        products = PrestaShopParser.parse_products(products_response)
-        categories = PrestaShopParser.parse_categories(categories_response)
+        sanitized_categories = sanitizer.categories(categories)
 
-        for product in products:
-            # worst line in the world
-            # maybe we should have the name directly in the parser
-            product['category'] = (
-                categories.get(product['id_category_default'], {}).get('name', {}).get('language', '')
-            )
+        categories_dict = {category.category_id: vars(category) for category in sanitized_categories}
+        sanitized_products = sanitizer.products(products)
 
-        serializer = PrestaShopProductSerializer(data=products, many=True)
-        serializer.is_valid()
+        # serialize response
+        serializer = XmlGeneratorSerializer()
+        response_xml = serializer.for_xml(products=sanitized_products, categories=categories_dict)
+        breakpoint()
 
-        return JsonResponse(serializer.data, safe=False)
+        return Response(response_xml, content_type='text/xml')
+        # return JsonResponse()