Define OnTask, OnScenario & OnPythonProgram plugin kinds

.pre-commit-config.yaml

@@ -3,7 +3,6 @@ repos:
     rev: stable
     hooks:
       - id: black
-        python_version: python3.6
   - repo: git://github.com/pre-commit/pre-commit-hooks
     rev: v1.2.3
     hooks:

.travis.yml

@@ -0,0 +1,19 @@
+dist: xenial
+language: python
+python:
+  - '3.6'
+  - '3.7'
+
+install:
+  - pip install poetry codacy-coverage
+  - poetry install
+
+script:
+  - make -f Makefile.ci
+
+after_success:
+  - python-codacy-coverage -r coverage.xml
+
+env:
+  global:
+    secure: duPuFLBO/bAeEnBHEQ7EHX07rLa7RO69PKH3+vTBmFVIqpZEh+X5psXTHClpFAT39AsUVvwKl5ZPwxunTOA13h6/b34M9WvArNOmMgrFvtsy2uqS9l/EJKMhhRpdL0h9Hk08Po1gJ7hEbOmFtadWtYXhge0z6bC4ykDCdKyO02sMNVceu0c+Q4j8mlV55iUjR0k9XYTRXHPyCZvhQRlWRu2SpFu+bS+i46ckdTtcLlHi0jDCAIc3od+j0ZT1CMSRdg0Bg81C9U7zCcPSpFk4AMdUjXw1h8k+7iFzzknyMDyVRi9pocPk8WXEN8i0OTGIsYCPWhgqcNP1o37xCXQj/7qPjvVYbF32vS9Ab+0RGySjlwBS19c5DoU/46IK6YR9Gpy8jmbTGbKdsK2grmg5AHs+Xv5/a5zbFoHiLRMSvqT5S8ZiAoHDKuBAQKXnHJEVYxmr1GqeOz6xPyEQlzrlbkKQlkO+D4oo3EALkq43dPo/fyKwgGVrOCyoEAfpRh/R448mVZKkuOb+IKnQCsRILpPeD8ZxQfQriVHeS4pMvZHmWA4hogZdGWe1a8xwDD2/RVclmaofyoT2HPPFgkY0wBDNlPC/RYvyXTSmAQ8XEk0CfzYh8IPsLys6v1E8MZqWG7s2xrbXQCz5jPGmmZ7MTuA20vUXGvyJCoPC6AJ/+xE=

CHANGELOG.md

@@ -7,8 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Plugin contracts aiming to eventually deprecate OnTaskSequence:
+  OnTask, OnScenario, OnPythonProgram.
+
 ### Changed
 
 - Open-sourcing of this project in https://github.com/zalando-incubator.
+- `transformer.plugins.Plugin` is renamed
+  `transformer.plugins.contracts.OnTaskSequence`.
 
 [Unreleased]: https://github.com/olivierlacan/keep-a-changelog/compare/v1.0.0...HEAD

README.md

@@ -1,5 +1,8 @@
 <p align="center"><img src="images/transformer.png"/></div>
 
+[![Build Status](https://travis-ci.org/zalando-incubator/Transformer.svg?branch=master)](https://travis-ci.org/zalando-incubator/Transformer)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/10b3feb4e4814429bf288b87443a6c72)](https://www.codacy.com/app/thilp/Transformer?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=zalando-incubator/Transformer&amp;utm_campaign=Badge_Grade)
+
 # Transformer
 
 A tool to transform/convert web browser sessions ([HAR files][]) into

pyproject.toml

@@ -11,7 +11,7 @@ authors = [
 license = "MIT"
 
 [tool.poetry.dependencies]
-python = "^3.6"
+python = ">=3.6"
 pendulum = "^2.0"
 chevron = "^0.13"
 

transformer/locust.py

@@ -25,7 +25,7 @@ def _locust_task(task: Union[Task, Task2]) -> py.Function:
     """
     if isinstance(task, Task):
         # TODO: remove when Task2 has replaced Task.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         task = Task2.from_task(task)
 
     return py.Function(name=task.name, params=["self"], statements=task.statements)
@@ -101,7 +101,7 @@ def locust_program(scenarios: Sequence[Scenario]) -> py.Program:
     """
     global_code_blocks = {
         # TODO: Replace me with a plugin framework that accesses the full tree.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         block_name: py.OpaqueBlock("\n".join(block), comments=[block_name])
         for scenario in scenarios
         for block_name, block in scenario.global_code_blocks.items()

transformer/plugins/__init__.py

@@ -1,5 +0,0 @@
-from typing import Sequence, Callable
-
-from transformer.task import Task
-
-Plugin = Callable[[Sequence[Task]], Sequence[Task]]

transformer/plugins/contracts.py

@@ -0,0 +1,72 @@
+"""
+This module defines the various kinds of plugins supported by Transformer.
+
+Transformer plugins are just functions that accept certain inputs and have
+certain outputs. Different kinds of plugins have different input and output
+types. These input and output types are formalized here using Python's
+annotation syntax and the typing module.
+
+# Plugin kinds
+
+## OnTask
+
+Kind of "stateless" plugins that operate independently on each task.
+When implementing one, imagine their execution could be parallelized by
+Transformer in the future.
+
+Example: a plugin that injects a header in all requests.
+
+## OnScenario
+
+Kind of plugins that operate on scenarios.
+
+Each scenario is the root of a tree composed of smaller scenarios and tasks
+(the leaves of this tree). Therefore, in an OnScenario plugin, you have the
+possibility of inspecting the subtree and making decisions based on that.
+However, OnScenario plugins will be applied to all scenarios by Transformer,
+so you don't need to recursively apply the plugin yourself on all subtrees.
+If you do that, the plugin will be applied many times more than necessary.
+
+Example: a plugin that keeps track of how long each scenario runs.
+
+## OnPythonProgram
+
+Kind of plugins that operate on the whole syntax tree.
+
+The input and output of this kind of plugins is the complete, final locustfile
+generated by Transformer, represented as a syntax tree.
+OnPythonProgram plugins therefore have the most freedom compared to other
+plugin kinds, because they can change anything.
+Their downside is that manipulating the syntax tree is more complex than the
+scenario tree or individual tasks.
+
+Example: a plugin that injects some code in the global scope.
+"""
+from typing import Sequence, Callable
+
+from transformer import python
+from transformer import scenario  # noqa: F401
+from transformer.task import Task, Task2
+
+
+OnTask = Callable[[Task2], Task2]
+
+
+OnScenario = Callable[["scenario.Scenario"], "scenario.Scenario"]
+
+
+OnPythonProgram = Callable[[python.Program], python.Program]
+
+
+# Historically Transformer has only one kind of plugin, which transformed a
+# sequence of Task objects into another such sequence. Operating on a full list
+# of tasks (instead of task by task) offered more leeway: a plugin could e.g.
+# add a new task, or change only the first task.
+# However this OnTaskSequence model is too constraining for some use-cases,
+# e.g. when a plugin needs to inject code in the global scope, and having to
+# deal with a full, immutable list of tasks in plugins that independently
+# operate on each task implies a lot of verbosity and redundancy.
+# For these reasons, other plugin kinds were created to offer a more varied
+# choice for plugin implementers.
+# See https://github.com/zalando-incubator/Transformer/issues/10.
+OnTaskSequence = Callable[[Sequence[Task]], Sequence[Task]]

transformer/scenario.py

@@ -15,7 +15,7 @@
 )
 
 from transformer.naming import to_identifier
-from transformer.plugins import Plugin
+from transformer.plugins.contracts import OnTaskSequence
 from transformer.request import Request
 from transformer.task import Task, Task2
 
@@ -81,7 +81,10 @@ class Scenario(NamedTuple):
 
     @classmethod
     def from_path(
-        cls, path: Path, plugins: Sequence[Plugin] = (), short_name: bool = False
+        cls,
+        path: Path,
+        plugins: Sequence[OnTaskSequence] = (),
+        short_name: bool = False,
     ) -> "Scenario":
         """
         Makes a Scenario (possibly containing sub-scenarios) out of the provided
@@ -105,7 +108,7 @@ def from_path(
 
     @classmethod
     def from_dir(
-        cls, path: Path, plugins: Sequence[Plugin], short_name: bool
+        cls, path: Path, plugins: Sequence[OnTaskSequence], short_name: bool
     ) -> "Scenario":
         """
         Makes a Scenario out of the provided directory path.
@@ -208,7 +211,7 @@ def _check_dangling_weights(cls, path, scenarios, weight_files):
 
     @classmethod
     def from_har_file(
-        cls, path: Path, plugins: Sequence[Plugin], short_name: bool
+        cls, path: Path, plugins: Sequence[OnTaskSequence], short_name: bool
     ) -> "Scenario":
         """
         Creates a Scenario given a HAR file.
@@ -269,7 +272,7 @@ def weight_from_path(cls, path: Path) -> int:
     @property
     def global_code_blocks(self) -> Mapping[str, Sequence[str]]:
         # TODO: Replace me with a plugin framework that accesses the full tree.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         return {
             block_name: block_lines
             for child in self.children

transformer/task.py

@@ -70,7 +70,7 @@ def __init__(
         request: Request,
         statements: Sequence[py.Statement] = (),
         # TODO: Replace me with a plugin framework that accesses the full tree.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         global_code_blocks: Mapping[str, Sequence[str]] = IMMUTABLE_EMPTY_DICT,
     ) -> None:
         self.name = name
@@ -89,7 +89,7 @@ def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task2"]:
         """
         # TODO: Update me when merging Task with Task2: "statements" needs to
         #   contain the equivalent of LocustRequest.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         for req in sorted(requests, key=lambda r: r.timestamp):
             if not on_blacklist(req.url.netloc):
                 yield cls(name=req.task_name(), request=req, statements=...)
@@ -98,7 +98,7 @@ def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task2"]:
     def from_task(cls, task: "Task") -> "Task2":
         # TODO: Remove me as soon as the old Task is no longer used and Task2 is
         #   renamed to Task.
-        #   See https://github.bus.zalan.do/TIP/docs/issues/395.
+        #   See https://github.com/zalando-incubator/Transformer/issues/11.
         locust_request = task.locust_request
         if locust_request is None:
             locust_request = LocustRequest.from_request(task.request)

transformer/transform.py

@@ -7,17 +7,19 @@
 
 import transformer.python as py
 from transformer.locust import locustfile
-from transformer.plugins import Plugin
+from transformer.plugins.contracts import OnTaskSequence
 from transformer.scenario import Scenario
 
 
 def transform(
-    scenarios_path: Union[str, Path], plugins: Sequence[Plugin] = ()
+    scenarios_path: Union[str, Path], plugins: Sequence[OnTaskSequence] = ()
 ) -> py.Program:
     return locustfile([Scenario.from_path(Path(scenarios_path), plugins)])
 
 
-def main(scenarios_path: Union[str, Path], plugins: Sequence[Plugin] = ()) -> str:
+def main(
+    scenarios_path: Union[str, Path], plugins: Sequence[OnTaskSequence] = ()
+) -> str:
     """
     Converts a WeightedFilePaths into a Locustfile.
     """