Pre commit test

.flake8

@@ -1,11 +1,10 @@
 [flake8]
 ignore = D203, W503
-select = Q0
 exclude =
     # No need to traverse our git directory
     .git,
     # There's no value in checking cache directories
     __pycache__,
 max-complexity = 10
 max-line-length = 100
-avoid-escape = True
+avoid-escape = True

.github/workflows/precommit_autoupdate.yaml

@@ -0,0 +1,40 @@
+# Adapted from https://github.com/browniebroke/browniebroke.com, Copyright (c) 2017 Bruno Allaname, available under the
+# MIT license
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  autoupdate-precommit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+#        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run pre-commit autoupdate
+        run: pre-commit autoupdate
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.CPR_GITHUB_TOKEN }}
+          branch: update/pre-commit-autoupdate
+          title: Auto-update pre-commit hooks
+          commit-message: Auto-update pre-commit hooks
+          body: |
+            Update versions of tools in pre-commit 
+            configs to latest version
+          labels: dependencies

.github/workflows/run_precommit.yaml

@@ -0,0 +1,29 @@
+name: Python package
+
+on: [push, pull_request]
+
+jobs:
+  run-precommit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+#        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pre-commit
+        run: |
+          pip install pre-commit
+
+      - name: Run pre-commit autoupdate
+        run: pre-commit autoupdate
+
+      - name: Run pre-commit hooks with hook-stage "manual" (=checks only)
+        run: |
+          pre-commit run --hook-stage manual --all-files

.github/workflows/run_tests.yaml

@@ -3,8 +3,7 @@ name: Python package
 on: [push, pull_request]
 
 jobs:
-  build:
-
+  run-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/todo_to_issue.yml → .github/workflows/todo_to_issue.yaml

@@ -1,9 +1,9 @@
 name: "Run TODO to Issue"
 on: ["push"]
 jobs:
-    build:
+    todo-to-issue:
         runs-on: "ubuntu-latest"
         steps:
             - uses: "actions/checkout@v3"
             - name: "TODO to Issue"
-              uses: "alstr/todo-to-issue-action@v4"
+              uses: "alstr/todo-to-issue-action@v4"

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+        types: [ yaml ]
+      - id: check-toml
+        types: [ toml ]
+      - id: check-merge-conflict
+      - id: check-ast
+        types: [ python ]
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.6.0
+    hooks:
+      - id: poetry-check
+  - repo: https://github.com/google/yapf
+    rev: v0.40.2
+    hooks:
+      - id: yapf
+        name: "yapf"
+        additional_dependencies: [toml]
+        args: [ '--style', 'pyproject.toml', '--parallel', '--in-place' ]
+        stages: [ commit ]
+        types: [ python ]
+      - id: yapf
+        name: "yapf (check)"
+        additional_dependencies: [toml]
+        args: [ '--style', 'pyproject.toml', '--parallel', '--diff' ]
+        stages: [ manual ]
+        types: [ python ]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort
+        stages: [ commit ]
+        types: [ python ]
+      - id: isort
+        name: isort (check)
+        args: [ '--check-only' ]
+        stages: [ manual ]
+        types: [ python ]
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-quotes==3.3.2]

CONTRIBUTING.md

@@ -0,0 +1,47 @@
+# Contributing to omnipy development
+
+## Development setup
+
+- Install Poetry:
+  - `curl -sSL https://install.python-poetry.org | python3 -`
+
+- Configure locally installed virtualenv (under `.venv`):
+  - `poetry config virtualenvs.in-project true`
+
+- Install dependencies:
+  - `poetry install --with dev --with docs`
+
+- Update all dependencies:
+  - `poetry update`
+
+- Update single dependency, e.g.:
+  - `poetry update prefect`
+
+- If a dependency is not updated to the latest version available on Pypi, you might need to clear
+  the pip cache of poetry:
+  - `poetry cache clear pypi --all`
+
+### For mypy support in PyCharm
+
+- In PyCharm, install "Mypy" plugin (not "Mypy (Official)")
+  - `which mypy` to get path to mypy binary
+  - In the PyCharm settings for the mypy plugin:
+    - Select the mypy binary 
+    - Select `pyproject.toml` as the mypy config file
+
+### For automatic formatting and linting
+
+The setup for automatic formatting and linting is rather complex. The main alternative is to use 
+black, which is easier to set up, but it does not have as many options and the main omnipy developer
+is opinionated against the default black setup. The yapf config is not fully
+defined. 
+
+- To install git hooks that automagically format and lint before every commit:
+  - `pre-commit install`
+
+- To update pre-commit-managed dependencies to the latest repos' versions:
+  - `pre-commit autoupdate`
+
+- In PyCharm -> File Watchers:
+  - Click arrow icon pointing down and to the left
+  - Select `pycharm_file_watchers.xml`

INSTALL.md

@@ -0,0 +1,40 @@
+# Installation and usage of omnipy
+
+## Installation
+
+`pip install omnipy`
+
+
+## Run example scripts:
+  - Install `omnipy-examples`:
+    - `pip install omnipy-examples`
+  - Example script:
+    - `omnipy-examples isajson`
+  - For help on the command line interface:
+    - `omnipy-examples --help`
+  - For help on a particular example:
+    - `omnipy-examples isajson --help`
+
+### Output of flow runs
+
+The output will by default appear in the `data` directory, with a timestamp. 
+
+  - It is recommended to install a file viewer that are capable of browsing tar.gz files. 
+    For instance, the "File Expander" plugin in PyCharm is excellent for this.
+  - To unpack the compressed files of a run on the command line 
+    (just make sure to replace the datetime string from this example): 
+
+```
+for f in $(ls data/2023_02_03-12_51_51/*.tar.gz); do mkdir ${f%.tar.gz}; tar xfzv $f -C ${f%.tar.gz}; done
+```
+
+### Run with the Prefect engine
+
+Omnipy is integrated with the powerful [Prefect](https://prefect.io) data flow orchestration library.
+
+- To run an example using the `prefect` engine, e.g.:
+  - `omnipy-examples --engine prefect isajson`
+- After completion of some runs, you can check the flow logs and orchestration options in the Prefect UI:
+  - `prefect orion start`
+
+More info on Prefect configuration will come soon...

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023 Omnipy contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.