Merge branch 'obelink-xml-v3' into feat/magento-app-mappings

composer.json

@@ -9,20 +9,21 @@
         }
     ],
     "require": {
-        "php": "8.*|7.4.*|7.3.*|7.2.*",
+        "php": ">=8.1",
         "ext-json": "*",
         "ext-iconv": "*",
         "ext-xmlwriter": "*",
         "ext-xmlreader": "*",
         "ext-curl": "*",
         "ext-zip": "*",
         "ext-simplexml": "*",
-        "symfony/yaml": "6.*|5.*|^4.3|^3.4",
+        "symfony/yaml": "6.*",
         "beberlei/assert": "^3.3",
         "adhocore/cli": "^0.9.0",
         "firebase/php-jwt": "^v6.8",
         "aspera/xlsx-reader": "^1.1.0",
-        "psr/log": "^3.0"
+        "psr/log": "^3.0",
+        "induxx/shared-app-libraries": "dev-main as 1.0"
     },
     "require-dev": {
         "roave/security-advisories": "dev-master",
@@ -47,6 +48,12 @@
             "Tests\\Misery\\": "tests/"
         }
     },
+    "repositories": [
+        {
+            "type": "composer",
+            "url": "https://app-satis.induxx.be"
+        }
+    ],
     "scripts": {
         "test": [
             "php vendor/bin/phpunit",

config/template/akeneo/README.md

@@ -0,0 +1,80 @@
+## Templates
+
+We support the following Akeneo endpoints.
+
+- attributes
+- family
+- family_variants
+- options
+- products
+- product-models
+- categories
+- reference-entities
+
+If you like to add support to the template system, make sure you have the required converter(s) in place.
+
+You can the API in 3 different Styles: JSON, CSV & as ITEM.
+
+### [JSON]
+Will not change anything, all Payloads are as is.
+Multidimensional Payload Structure is kept (We don't use any converters).
+The Style is not very flexible, to make structural changes or the read multidimensional data.
+
+### [CSV]
+Will convert the Payload to CSV style flat data.
+This style is not very flexible to make structural changes.
+The data is flat, so it's relatively easy to manipulate but looses on structural integrity, as you don't have contextual awareness.
+
+### [ITEM] - [V3-edition]
+Will convert the Payload into a easy parsable Item. 
+The data becomes discoverable, searchable across other 
+This type will give you the most flexibility to manipulate the data.
+You can export to CSV and/or API from this point.
+
+### Examples
+
+```yaml
+transformation_steps:
+  - {'run': akeneo/json/pull_akeneo-reference-entities.yaml, 'with': {'endpoint': ['brand']}}
+  - {'run': akeneo/json/pull_akeneo-entities.yaml, 'with': {'endpoint': ['attributes', 'families']}}
+
+  - {'run': akeneo/json/push_akeneo-reference-entities.yaml, 'with': {'endpoint': ['brand']}}
+  - {'run': akeneo/json/push_akeneo-entities.yaml, 'with': {'endpoint': ['attributes', 'families']}}
+```
+
+In this example the only custom file is change_other_attributes.yaml.
+
+```yaml
+context:
+  akeneo_read_connection: '%source_resource%'
+  akeneo_write_connection: '%target_resource%'
+
+transformation_steps:
+  - {'run': akeneo/json/pull_akeneo-entities.yaml, 'with': {'endpoint': ['attributes']}}
+  # DO SOMETHING
+  - {'run': change_other_attributes.yaml, 'with': {'endpoint': ['attributes']}}
+  - {'run': akeneo/json/push_akeneo-entities.yaml, 'with': {'endpoint': ['attributes']}}
+```
+
+```yaml
+# change_other_attributes.yaml
+
+sources:
+  - '%workpath%/akeneo_%endpoint%.jsonl'
+
+pipeline:
+  input:
+    reader:
+      type: jsonl
+      filename: '%workpath%/akeneo_%endpoint%.jsonl'
+      filter:
+        group: 'other'
+
+  actions:
+    # DO SOMETHING
+
+  output:
+    writer:
+      type: jsonl
+      filename: '%workpath%/akeneo_%endpoint%.jsonl'
+```

config/template/akeneo/csv/pull_akeneo-entities.yaml

@@ -0,0 +1,12 @@
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: '%endpoint%'
+      method: GET
+  output:
+    writer:
+      type: csv
+      filename: 'akeneo_%endpoint%.csv'
+      converter: 'akeneo/%endpoint%/csv'

config/template/akeneo/csv/pull_akeneo-identifiers.yaml

@@ -0,0 +1,15 @@
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: '%endpoint%'
+      method: GET
+  actions:
+    remove_unwanted:
+      action: retain
+      keys: [ identifier, code ]
+  output:
+    writer:
+      type: csv
+      filename: 'akeneo_%endpoint%_identifiers.csv'

config/template/akeneo/csv/pull_akeneo-reference-entities.yaml

@@ -0,0 +1,13 @@
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: reference-entities
+      method: GET
+      identifier_filter_list: ['%endpoint%']
+  output:
+    writer:
+      type: csv
+      filename: 'akeneo_reference-entities_%endpoint%.csv'
+      converter: 'akeneo/reference-entities/csv'

config/template/akeneo/csv/push_akeneo-entities.yaml

@@ -0,0 +1,16 @@
+sources:
+  - '%workpath%/akeneo_%endpoint%.csv'
+
+pipeline:
+  input:
+    reader:
+      type: csv
+      filename: '%workpath%/akeneo_%endpoint%.csv'
+      converter: 'akeneo/%endpoint%/csv'
+  output:
+    http:
+      type: rest_api
+      account: '%akeneo_write_connection%'
+      endpoint: '%endpoint%'
+      method: MULTI_PATCH
+      converter: 'akeneo/%endpoint%/api'

config/template/akeneo/csv/push_akeneo-reference-entities.yaml

@@ -0,0 +1,16 @@
+sources:
+  - '%workpath%/akeneo_reference-entities_%endpoint%.csv'
+
+pipeline:
+  input:
+    reader:
+      type: csv
+      filename: '%workpath%/akeneo_reference-entities_%endpoint%.csv'
+      converter: 'akeneo/reference_entities/csv'
+  output:
+    http:
+      type: rest_api
+      account: '%akeneo_write_connection%'
+      endpoint: reference-entities
+      method: PATCH
+      converter: 'akeneo/reference_entities/api'

config/template/akeneo/jsonl/pull_akeneo-entities.yaml

@@ -0,0 +1,20 @@
+context:
+  filter: []
+  identifier_filter_list: []
+  query: ''
+
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: '%endpoint%'
+      method: GET
+      filter: '%filter%'
+      identifier_filter_list: '%identifier_filter_list%'
+      limiters:
+        querystring: '%query%'
+  output:
+    writer:
+      type: jsonl
+      filename: 'akeneo_%endpoint%.jsonl'

config/template/akeneo/jsonl/pull_akeneo-entity_options.yaml

@@ -0,0 +1,13 @@
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: '%endpoint%'
+      method: GET
+      multiple: true
+      attribute_list: '%attribute_options%'
+  output:
+    writer:
+      type: jsonl
+      filename: 'akeneo_%endpoint%.jsonl'

config/template/akeneo/jsonl/pull_akeneo-reference-entities.yaml

@@ -0,0 +1,13 @@
+pipeline:
+  input:
+    http:
+      type: rest_api
+      account: '%akeneo_read_connection%'
+      endpoint: reference-entities
+      method: GET
+      identifier_filter_list: ['%endpoint%']
+
+  output:
+    writer:
+      type: jsonl
+      filename: 'akeneo_reference-entities_%endpoint%.jsonl'

config/template/akeneo/jsonl/push_akeneo-entities.yaml

@@ -0,0 +1,14 @@
+sources:
+  - '%workpath%/akeneo_%endpoint%.jsonl'
+
+pipeline:
+  input:
+    reader:
+      type: jsonl
+      filename: '%workpath%/akeneo_%endpoint%.jsonl'
+  output:
+    http:
+      type: rest_api
+      account: '%akeneo_write_connection%'
+      endpoint: '%endpoint%'
+      method: MULTI_PATCH

config/template/akeneo/jsonl/push_akeneo-reference-entities.yaml

@@ -0,0 +1,16 @@
+sources:
+  - '%workpath%/akeneo_reference-entities_%endpoint%.jsonl'
+
+pipeline:
+  input:
+    reader:
+      type: jsonl
+      filename: '%workpath%/akeneo_reference-entities_%endpoint%.jsonl'
+      converter: 'akeneo/reference_entities/csv'
+  output:
+    http:
+      type: rest_api
+      account: '%akeneo_write_connection%'
+      endpoint: reference-entities
+      method: PATCH
+      converter: 'akeneo/reference_entities/api'

docker/fpm/Dockerfile

@@ -1,4 +1,4 @@
-FROM php:8.0-fpm
+FROM php:8.1-fpm
 
 COPY --from=composer /usr/bin/composer /usr/bin/composer
 
@@ -13,10 +13,14 @@ COPY app.ini $PHP_INI_DIR/conf.d/app.ini
 RUN apt update
 RUN apt install -y moreutils
 
+# pyhton panda (Transformations : Use a Virtual Environment /opt/venv)
 RUN apt-get update && \
-    apt-get --no-install-recommends --no-install-suggests --yes --quiet install python3 python3-pip && \
-    pip install pandas && \
-    apt-get clean && apt-get --yes --quiet autoremove --purge && \
-    rm -rf  /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-            /usr/share/doc/* /usr/share/groff/* /usr/share/info/* /usr/share/linda/* \
-            /usr/share/lintian/* /usr/share/locale/* /usr/share/man/*
+    apt-get install --no-install-recommends --yes python3 python3-pip python3-venv && \
+    python3 -m venv /opt/venv && \
+    /opt/venv/bin/pip install pandas && \
+    apt-get clean && apt-get --yes autoremove --purge && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+           /usr/share/doc/* /usr/share/groff/* /usr/share/info/* /usr/share/linda/* \
+           /usr/share/lintian/* /usr/share/locale/* /usr/share/man/*
+
+ENV PATH="/opt/venv/bin:$PATH"

src/Command/TransformationCommand.php

@@ -3,6 +3,10 @@
 namespace Misery\Command;
 
 use Ahc\Cli\Input\Command;
+use App\Component\ChangeManager\ChangeManager;
+use App\Component\Common\Resource\ChangeResource;
+use App\Infra\Adapter\KeyValueStore\FileKeyValueStoreAdapter;
+use App\Infra\Redis\RedisIdentityScope;
 use Assert\Assertion;
 use Misery\Component\Common\FileManager\LocalFileManager;
 use Misery\Component\Common\Functions\ArrayFunctions;
@@ -74,15 +78,34 @@ public function execute(string $file, string $source, string $workpath, bool $de
             new OutputLogger()
         );
 
+        // setting up a fake File based key-value store for our change-manager
+        $configurationFactory->setChangeManager(
+            new ChangeManager(
+                new ChangeResource(
+                    new FileKeyValueStoreAdapter($workpath.DIRECTORY_SEPARATOR.'store'),
+                    new RedisIdentityScope()
+                )
+            )
+        );
+
+        // reading the app_context file
+        $transformationDir = pathinfo($file, PATHINFO_DIRNAME);
+        $contextFile = $transformationDir.DIRECTORY_SEPARATOR.'app_context.yaml';
+        $context = (is_file($contextFile)) ? Yaml::parseFile($contextFile) : [];
+
         $transformationFile = ArrayFunctions::array_filter_recursive(Yaml::parseFile($file), function ($value) {
             return $value !== NULL;
         });
+
+        // merging it with the original MAIN-step.yaml, after this point the two are merged
+        $transformationFile = ArrayFunctions::array_merge_recursive($context, $transformationFile);
+
         $configuration = $configurationFactory->parseDirectivesFromConfiguration(
             array_replace_recursive($transformationFile, [
                 'context' => [
                     # emulated operation datetime stamps
-                    'operation_create_datetime' => (new \DateTime('NOW'))->format('Hd-m-Y-H-i-s'),
-                    'last_completed_operation_datetime' => (new \DateTime('NOW'))->modify('-2 hours')->format('Hd-m-Y-H-i-s'),
+                    'operation_create_datetime' => (new \DateTime('NOW'))->format($transformationFile['context']['date_format'] ?? 'Y-m-d H:i:s'),
+                    'last_completed_operation_datetime' => (new \DateTime('NOW'))->modify('-2 hours')->format($transformationFile['context']['date_format'] ?? 'Y-m-d H:i:s'),
                     'transformation_file' => $file,
                     'sources' => $source,
                     'scripts' => __DIR__.'/../../scripts',

src/Component/Action/ActionItemInterface.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Misery\Component\Action;
+
+use Misery\Model\DataStructure\ItemInterface;
+
+interface ActionItemInterface
+{
+    public function applyAsItem(ItemInterface $item): ItemInterface;
+}

src/Component/Action/ConcatAction.php

@@ -27,9 +27,9 @@ public function apply(array $item): array
             return $item;
         }
 
-        if (array_key_exists($field, $item)) {
-            $item[$field] = ValueFormatter::format($this->getOption('format'), $item);
-        }
+        // don't check if array_key_exist here, concat should always work, if the field doesn't exist
+        // somewhat the point to form a new field from concatination
+        $item[$field] = ValueFormatter::format($this->getOption('format'), $item);
 
         return $item;
     }