Refactor (#44)

* Add pyproject.toml

* Add new checks

* Add new observer with dispatcher

* Move coingecko config out of module

* Add CoinGecko prices and config args/vars

* Implement crosschain checks

* Run isort

* Remove logging from checks

* Add asyncio support to Datadog client

* Remove empty file

* Replace mypy with pyright

Pyright has better integration with VS Code

* Implement check protocol

* Consolidate check protocol

* Clean up

* Add sample input files

* Update prometheus-client

* Expose prometheus metrics

* Update CI workflow

* Update README

* Rename price feed online check

This addresses the confusion around the slot values.

* Improve event configuration

Document env. variabels in README and bail out earlier if they are not passed.

* Add type hints for Observer inputs

* Add service tag to Datadog events

.github/actions/python-poetry/action.yml

@@ -0,0 +1,24 @@
+name: Python Poetry
+description: Sets up a Python environment with Poetry
+
+inputs:
+  python-version:
+    required: false
+    description: Python version
+    default: "3.10"
+  poetry-version:
+    required: false
+    description: Poetry version
+    default: "1.2.2"
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v2
+      with:
+        python-version: ${{ inputs.python-version }}
+    - uses: abatilo/[email protected]
+      with:
+        poetry-version: ${{ inputs.poetry-version }}
+    - run: poetry install
+      shell: sh

.github/workflows/ci.yaml

@@ -0,0 +1,57 @@
+name: CI
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - "main"
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/python-poetry
+      - uses: pre-commit/[email protected]
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/python-poetry
+      - run: poetry run pytest
+        env:
+          TEST_MODE: "1"
+  build-and-push-ecr:
+    runs-on: ubuntu-latest
+    needs: [pre-commit, run-tests]
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v2
+      - uses: aws-actions/configure-aws-credentials@8a84b07f2009032ade05a88a28750d733cc30db1
+        with:
+          role-to-assume: arn:aws:iam::192824654885:role/github-actions-ecr
+          aws-region: eu-west-2
+      - uses: docker/login-action@v2
+        with:
+          registry: public.ecr.aws
+        env:
+          AWS_REGION: us-east-1
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+          endpoint: builders
+      - uses: haya14busa/action-cond@v1
+        id: image_tag
+        with:
+          cond: ${{ startsWith(github.ref, 'refs/tags/') }}
+          if_true: ${{ github.ref_name }}
+          if_false: ${{ github.sha }}
+      - uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: public.ecr.aws/pyth-network/observer:${{ steps.image_tag.outputs.value }}

.gitignore

@@ -1,9 +1,7 @@
-.env
-ve
-venv
-pyth-client-py
-publishers.json
-**/__pycache__
+__pycache__/
+*.py[cod]
+*$py.class
 
 .DS_Store
-.coverage
+.envrc
+.coverage

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: local
+    hooks:
+      - id: isort
+        name: isort
+        entry: poetry run isort --profile=black --check .
+        language: system
+      - id: black
+        name: black
+        entry: poetry run black --check .
+        pass_filenames: false
+        language: system
+      - id: pyright
+        name: pyright
+        entry: poetry run pyright pyth_observer/ tests/
+        pass_filenames: false
+        language: system
+      - id: pyflakes
+        name: pyflakes
+        entry: poetry run pyflakes pyth_observer/ tests/
+        pass_filenames: false
+        language: system

Dockerfile

@@ -1,16 +1,95 @@
-# syntax=docker/dockerfile:1.2
-FROM python:3.8-slim-bullseye
+# Reference: https://bmaingret.github.io/blog/2021-11-15-Docker-and-Poetry#multi-stage-build
 
-ARG UID=10000
-ARG GID=10000
-ARG DIR=/data
+ARG APP_NAME=pyth-observer
+ARG APP_PACKAGE=pyth_observer
+ARG APP_PATH=/opt/$APP_NAME
+ARG PYTHON_VERSION=3.10.4
+ARG POETRY_VERSION=1.2.2
 
-WORKDIR ${DIR}
-RUN groupadd -o -g ${GID} -r app && adduser --system --home ${DIR} --ingroup app --uid ${UID} app
+#
+# Stage: base
+#
 
-COPY --chown=app:app . ${DIR}
-RUN pip install --no-cache-dir -r requirements.txt
-USER app
+FROM python:$PYTHON_VERSION as base
 
-ENTRYPOINT ["python3", "/data/observer.py"]
-CMD ["-l", "debug", "--network", "devnet"]
+ARG APP_NAME
+ARG APP_PATH
+ARG POETRY_VERSION
+
+ENV \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1
+ENV \
+    POETRY_VERSION=$POETRY_VERSION \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1
+
+# Install Poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python
+ENV PATH="$POETRY_HOME/bin:$PATH"
+
+WORKDIR $APP_PATH
+COPY . .
+
+#
+# Stage: development
+#
+
+FROM base as development
+
+ARG APP_NAME
+ARG APP_PATH
+
+WORKDIR $APP_PATH
+RUN poetry install
+
+ENV APP_NAME=$APP_NAME
+
+ENTRYPOINT ["poetry", "run"]
+CMD ["$APP_NAME"]
+
+#
+# Stage: build
+#
+
+FROM base as build
+
+ARG APP_NAME
+ARG APP_PATH
+
+WORKDIR $APP_PATH
+RUN poetry build --format wheel
+RUN poetry export --format requirements.txt --output constraints.txt --without-hashes
+
+#
+# Stage: production
+#
+
+FROM python:$PYTHON_VERSION as production
+
+ARG APP_NAME
+ARG APP_PATH
+
+ENV \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1
+
+ENV \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100
+
+# Get build artifact wheel and install it respecting dependency versions
+WORKDIR $APP_PATH
+COPY --from=build $APP_PATH/dist/*.whl ./
+COPY --from=build $APP_PATH/constraints.txt ./
+RUN pip install ./*.whl --requirement constraints.txt
+
+COPY ./entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["$APP_NAME"]