diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31419d6..cd51971 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,9 +20,6 @@ jobs: - uses: Gr1N/setup-poetry@v8 - - name: Check dependencies - run: make doctor - - uses: actions/cache@v2 with: path: .venv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 339701f..16c20cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,12 +15,6 @@ * Linux: [https://graphviz.org/download](https://graphviz.org/download/) * Windows: [https://graphviz.org/download](https://graphviz.org/download/) -To confirm these system dependencies are configured correctly: - -```text -$ make bootstrap -$ make doctor -``` ### Installation diff --git a/Makefile b/Makefile index a559c3a..c28dfc1 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,6 @@ $(DEPENDENCIES): poetry.lock @ rm -rf ~/Library/Preferences/pypoetry @ poetry config virtualenvs.in-project true poetry install - @ touch $@ ifndef CI poetry.lock: pyproject.toml diff --git a/README.md b/README.md index 9bd4fd5..3c1d06b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Wppt +# Wppt +#### Webhook Payload Proxy Transformer [![Unix Build Status](https://img.shields.io/github/actions/workflow/status/grafuls/wppt/main.yml?branch=main&label=linux)](https://github.com/grafuls/wppt/actions) [![Coverage Status](https://img.shields.io/codecov/c/gh/grafuls/wppt)](https://codecov.io/gh/grafuls/wppt) @@ -10,7 +11,7 @@ Wppt(Pronounced: [ˈwɪpɪt]) provides an easy way to intercept, manage, manipul Some services/platforms don't provide an easy-to-use integration vehicle for transforming and syncing requests, payloads and automation e.g. Gitlab -> Jira. Instead of paying for an expensive third-party service to provide integrations you can do this easily yourself on-premise. -## How wppt Does Work? +## How Does `wppt` Work? Wppt leverages Flask dynamic routing. The endpoint is variable and defined via one or multiple yaml files. Based on the endpoint url, `wppt` parses all the yaml files stored on the `transformers` directory, and retrieves the outgoing webhook url and the translations. It then parses all the translations and converts the existing data from the incoming webhook into a new payload structure as defined on the yaml. @@ -24,7 +25,7 @@ gitlab2jira: translations: data: name: '[{data[project][name]}][{data[object_kind]}] {data[object_attributes][title]}' - description: 'Description: {data[object_attributes][description]}\nURL:{data[object_attributes][url]}' + description: 'Description: {data[object_attributes][description]}\nURL:{data[object_attributes][url]}' ``` Given the following incoming webhook payload to `http://{FQDN}:5005/gitlab2jira/`: @@ -60,7 +61,9 @@ Given the following incoming webhook payload to `http://{FQDN}:5005/gitlab2jira/ Install it directly into a poetry virtual environment: -```text +```bash +$ git clone https://github.com/redhat-performance/wppt +$ cd wppt $ make install ``` @@ -68,6 +71,20 @@ $ make install After installation, the server can be started with: -```text +```bash $ make run ``` + +### Via Podman + +#### Building the image +```bash +$ cd docker +$ podman build -t wppt . +``` + +#### Running +```bash +$ podman run -it --rm -v /path/to/local/transformers/:/opt/wppt/transformers -p 5005:5005 wppt +``` + diff --git a/bin/checksum b/bin/checksum new file mode 100755 index 0000000..73e0196 --- /dev/null +++ b/bin/checksum @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import hashlib +import sys + + +def run(paths): + sha = hashlib.sha1() + + for path in paths: + try: + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + sha.update(chunk) + except IOError: + sha.update(path.encode()) + + print(sha.hexdigest()) + + +if __name__ == '__main__': + run(sys.argv[1:]) \ No newline at end of file diff --git a/bin/open b/bin/open new file mode 100755 index 0000000..079de50 --- /dev/null +++ b/bin/open @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys + + +COMMANDS = { + 'linux': "open", + 'win32': "cmd /c start", + 'cygwin': "cygstart", + 'darwin': "open", +} + + +def run(path): + command = COMMANDS.get(sys.platform, "open") + os.system(command + ' ' + path) + + +if __name__ == '__main__': + run(sys.argv[-1]) \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..819a862 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,20 @@ +# Use the official Python base image +FROM python:3.11 +LABEL authors="grafuls" + +RUN apt-get install git patch make + +# Set the working directory inside the container +WORKDIR /opt +RUN git clone https://github.com/redhat-performance/wppt +WORKDIR /opt/wppt +RUN git checkout development + +# Install Poetry +RUN pip install poetry + +# Install project dependencies +RUN make install + +# Set the entrypoint command to run the project +CMD ["make", "run"] diff --git a/pyproject.toml b/pyproject.toml index 82a2626..0b19272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,14 @@ license = "GPL3" authors = ["Gonzalo Rafuls "] readme = "README.md" -homepage = "https://pypi.org/project/wppt" -documentation = "https://wppt.readthedocs.io" +homepage = "https://github.com/redhat-performance/wppt" +documentation = "https://github.com/redhat-performance/wppt/blob/main/README.md" repository = "https://github.com/redhat-performance/wppt" keywords = [] classifiers = [ # TODO: update this list to match your application: https://pypi.org/pypi?%3Aaction=list_classifiers - "Development Status :: 1 - Planning", + "Development Status :: 3 - Alpha", "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python", @@ -29,9 +29,9 @@ classifiers = [ python = "^3.11" -# TODO: Remove these and add your library's requirements -flask = "*" -requests = "*" +flask = "3.0.2" +requests = "2.31.0" +pyyaml = "6.0.1" [tool.poetry.dev-dependencies] @@ -39,7 +39,7 @@ requests = "*" black = "^22.1" tomli = "*" # missing 'black' dependency isort = "^5.10" -ipdb = "*" # missing 'black' dependency +ipdb = "0.13.13" # missing 'black' dependency # Linters mypy = "^1.0" @@ -68,10 +68,6 @@ sniffer = "*" MacFSEvents = { version = "*", platform = "darwin" } pync = { version = "*", platform = "darwin" } -[tool.poetry.scripts] - -wppt = "wppt.cli:main" - [tool.black] quiet = true diff --git a/tests/conftest.py b/tests/conftest.py index a847a4b..a2077d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,12 @@ -"""Integration tests configuration file.""" +import pytest -# pylint: disable=unused-import +from wppt.app import app as flask_app + +@pytest.fixture(scope="module") +def test_client(): + """ + | Creates a test client for the app. + """ + with flask_app.test_client() as testing_client: + with flask_app.app_context(): + yield testing_client \ No newline at end of file diff --git a/tests/fixtures/transformers/test.yaml b/tests/fixtures/transformers/test.yaml new file mode 100644 index 0000000..8ebc4ba --- /dev/null +++ b/tests/fixtures/transformers/test.yaml @@ -0,0 +1,7 @@ +test: + enabled: true + target_webhook: https://example.com/rest/api/ + translations: + data: + name: '[{data[project][name]}][{data[object_kind]}] {data[object_attributes][title]}' + description: 'Description: {data[object_attributes][description]}\nURL:{data[object_attributes][url]}' diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..21d385c --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,76 @@ +from unittest.mock import patch +from pytest import raises +import requests + +class TestApp: + + @patch('wppt.app.parse_definitions') + @patch('wppt.app.traverse_format_dict') + @patch('wppt.app.requests.post') + def test_dinamic_transformer_enabled_transformer(self, mock_post, mock_traverse_format_dict, mock_parse_definitions, test_client): + mock_parse_definitions.return_value = { + 'transformer1': { + 'enabled': True, + 'translations': {'key1': 'value1'}, + 'target_webhook': 'https://example.com/webhook' + } + } + mock_traverse_format_dict.return_value = None + mock_post.return_value.status_code = 200 + + response = test_client.post('/transformer1/', json={'key1': 'value1'}) + + assert response.status_code == 200 + assert response.json == {'status_code': 200, 'message': 'Transformers executed.'} + mock_traverse_format_dict.assert_called_once_with({'key1': 'value1'}, {'key1': 'value1'}) + + @patch('wppt.app.parse_definitions') + def test_dinamic_transformer_disabled_transformer(self, mock_parse_definitions, test_client): + mock_parse_definitions.return_value = { + 'transformer1': { + 'enabled': False, + 'translations': {'key1': 'value1'}, + 'target_webhook': 'https://example.com/webhook' + } + } + response = test_client.post('/transformer1/', json={'key1': 'value1'}) + + assert response.json[0] == 'Transformer transformer1 is disabled' + assert response.json[1] == 400 + + @patch('wppt.app.parse_definitions') + @patch('wppt.app.traverse_format_dict', side_effect=KeyError('Invalid key')) + def test_dinamic_transformer_invalid_key(self, mock_traverse_format_dict, mock_parse_definitions, test_client): + mock_parse_definitions.return_value = { + 'transformer1': { + 'enabled': True, + 'translations': {'key1': 'value1'}, + 'target_webhook': 'https://example.com/webhook' + } + } + + response = test_client.post('/transformer1/', json={'key1': 'value1'}) + + assert response.json[0] == "'Invalid key'" + assert response.json[1] == 400 + mock_traverse_format_dict.assert_called_once_with({'key1': 'value1'}, {'key1': 'value1'}) + + @patch('wppt.app.parse_definitions') + @patch('wppt.app.traverse_format_dict') + @patch('wppt.app.requests.post', side_effect=requests.exceptions.RequestException('Failed to send request')) + def test_dinamic_transformer_failed_request(self, mock_post, mock_traverse_format_dict, mock_parse_definitions, test_client): + mock_parse_definitions.return_value = { + 'transformer1': { + 'enabled': True, + 'translations': {'key1': 'value1'}, + 'target_webhook': 'https://example.com/webhook' + } + } + mock_traverse_format_dict.return_value = None + mock_post.side_effect = requests.exceptions.RequestException('Failed to send request') + + response = test_client.post('/transformer1/', json={'key1': 'value1'}) + + assert response.json['message'] == 'Failed to send the transformed webhook for transformer1.' + assert response.json['status_code'] == 400 + mock_traverse_format_dict.assert_called_once_with({'key1': 'value1'}, {'key1': 'value1'}) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..24a3d3b --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,45 @@ +from unittest.mock import patch +from wppt.utils import parse_definitions, walk_dir, read_yaml, traverse_format_dict + +class TestUtils: + + @patch('wppt.utils.walk_dir') + @patch('wppt.utils.read_yaml') + def test_parse_definitions(self, mock_read_yaml, mock_walk_dir): + mock_walk_dir.return_value = ['/path/to/file1.yaml', '/path/to/file2.yaml'] + mock_read_yaml.side_effect = [{'key1': 'value1'}, {'key2': 'value2'}] + + directory = '/path/to/directory' + expected_definitions = {'key1': 'value1', 'key2': 'value2'} + assert parse_definitions(directory) == expected_definitions + + def test_walk_dir(self): + directory = 'tests/fixtures/transformers' + file_extension = '.yaml' + expected_file_list = ['tests/fixtures/transformers/test.yaml'] + + assert walk_dir(directory, file_extension) == expected_file_list + + def test_read_yaml(self): + file_path = '../wppt/transformers/gitlab2jira.yaml' + expected_yaml_definitions = {'key1': 'value1', 'key2': 'value2'} + + # Mocking the yaml.load function + with patch('yaml.load') as mock_yaml_load: + mock_yaml_load.return_value = {'key1': 'value1', 'key2': 'value2'} + + file_path = '../wppt/transformers/gitlab2jira.yaml' + expected_yaml_definitions = {'key1': 'value1', 'key2': 'value2'} + + with patch('builtins.open', create=True) as mock_open: + mock_open.return_value.__enter__.return_value.read.return_value = 'key1: value1\nkey2: value2\n' + yaml_definitions = read_yaml(file_path) + assert yaml_definitions == expected_yaml_definitions + + def test_traverse_format_dict(self): + dictionary = {'key1': 'Hello {data}', 'key2': 'Welcome {data}'} + data = 'World' + expected_dictionary = {'key1': 'Hello World', 'key2': 'Welcome World'} + + traverse_format_dict(dictionary, data) + assert dictionary == expected_dictionary diff --git a/wppt/app.py b/wppt/app.py index dccc92e..85151f8 100644 --- a/wppt/app.py +++ b/wppt/app.py @@ -1,6 +1,6 @@ #! /usr/bin/env python -from flask import Flask, request +from flask import Flask, request, jsonify, Response import requests import json @@ -16,7 +16,7 @@ def hello(): @app.route("//", methods=["POST"]) -def dinamic_transformer(transformer: str): +def dinamic_transformer(transformer: str) -> Response: data = request.json definitions = parse_definitions(TRANSFORMERS_PATH) for _transformer, definition in definitions.items(): @@ -24,23 +24,40 @@ def dinamic_transformer(transformer: str): if definition.get("enabled"): translations = definition.get("translations") webhook_url = definition.get("target_webhook") + + headers = {"Content-`type": "application/json"} + try: + traverse_format_dict(translations, data) + except KeyError as ಠ_ಠ: + return jsonify(f"{ಠ_ಠ}", 400) + + try: + response = requests.post( + webhook_url, headers=headers, data=json.dumps(translations) + ) + except requests.exceptions.RequestException as e: + payload = { + "status_code": 400, + "error": f"{e}", + "message": f"Failed to send the transformed webhook for {_transformer}.", + } + return jsonify(payload) + + if response.status_code not in (200, 201, 202, 204): + payload = { + "status_code": response.status_code, + "error": response.text, + "message": f"Failed to send the transformed webhook for {_transformer}.", + } + return jsonify(payload) else: - return f"Transformer {_transformer} is disabled", 200 - - headers = {"Content-type": "application/json"} - try: - traverse_format_dict(translations, data) - except KeyError as ಠ_ಠ: - return f"{ಠ_ಠ}", 400 - import ipdb;ipdb.set_trace() - response = requests.post( - webhook_url, headers=headers, data=json.dumps(translations) - ) - - print(response.status_code) - print(response.text) - - return f"Transformers executed.", 200 + return jsonify(f"Transformer {_transformer} is disabled", 400) + + response = { + "status_code": 200, + "message": "Transformers executed.", + } + return jsonify(response) if __name__ == "__main__":