[*] bring up a private postgres on start up, do a migration, use mini…

…mal python installs

.dockerignore

@@ -0,0 +1,26 @@
+__pycache__
+**/__pycache__
+*.pyc
+**/*.pyc
+*.pyo
+**/*.pyo
+*.pyd
+**/*.pyd
+.Python
+env
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+.git
+.mypy_cache
+.pytest_cache
+.hypothesis
+python/
+*.egg-info

.gitignore

@@ -9,3 +9,4 @@ build/
 # Ignore OSX finder directories
 .DS_Store
 
+python/

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.9

README.rst

@@ -12,3 +12,13 @@ Capabilities
 - Aggressively query/scrape an `APC BackUPS Pro 500 <http://www.apc.com/shop/us/en/products/APC-Back-UPS-Pro-500-Lithium-Ion-UPS/P-BG500>`_.
 - Ingest metrics from the above into Graphite
 - Write snapshot data into Redis
+
+Development
+--------------
+
+.. code-block:: console
+
+    $ python3.10 -m pip install --user invoke
+    $ invoke setup
+    $ . python/bin/activate
+    (python) $ ...

config/003_initialize_powerscout_schema.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env sh
+
+PGCTL=$(find / -name pg_ctl -print -quit)
+if [ "x$PGCTL" = 'x' ]; then
+    echo 'Unable to find pg_ctl!!'
+    exit 404
+fi
+echo 'Using '$PGCTL
+
+exec /powerscout/python/bin/python <(cat <<EOF
+import os
+import shlex
+import subprocess
+import signal
+from contextlib import suppress
+from tempfile import TemporaryDirectory, NamedTemporaryFile
+
+from pytest_postgresql.executor import PostgreSQLExecutor
+
+postgres_user = 'postgres'
+with suppress(KeyError):
+    postgres_user = os.environ["POSTGRES_USER"]
+
+postgres_password = ''
+with suppress(KeyError):
+    if os.environ.get("POSTGRES_HOST_AUTH_METHOD") != "trust":
+        postgres_password = os.environ["POSTGRES_PASSWORD"]
+
+credentials = postgres_user
+if postgres_password:
+    credentials = f"{postgres_user}:{postgres_password}"
+
+with (
+    TemporaryDirectory() as data,
+    TemporaryDirectory() as socket,
+    NamedTemporaryFile() as log,
+    PostgreSQLExecutor(
+        "${PGCTL}",
+        "127.0.0.1",
+        5433,
+        data,
+        socket,
+        log.name,
+        "-w",
+        "postgres",
+    ) as temp_database
+):
+    model_db_url = f"postgresql://{temp_database.host}:{temp_database.port}/postgres"
+    destination_db = f"postgresql://{credentials}@/postgres?host=/var/run/postgresql/"
+
+    subprocess.check_call(shlex.split(f"psql -v ON_ERROR_STOP=1 '{destination_db}' -f /powerscout/sql/01-datatypes.sql"))
+    subprocess.check_call(shlex.split(f"psql -v ON_ERROR_STOP=1 '{model_db_url}' -f /powerscout/sql/01-datatypes.sql -f /powerscout/sql/02-init.sql"))
+
+    migra_fh = subprocess.Popen(
+        shlex.split(f"migra --unsafe --with-privileges {destination_db} {model_db_url}"),
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL
+    )
+    stdout, stderr = migra_fh.communicate()
+    if migra_fh.returncode == 0:
+        print("Nothing to migrate to!")
+        raise SystemExit(0)
+    elif migra_fh.returncode == 2:
+        with NamedTemporaryFile('w+b') as fh:
+            print(f"Migration statements at {fh.name}:\n{stdout.decode()}")
+            fh.write(stdout)
+            fh.flush()
+            subprocess.check_call(shlex.split(f"psql -v ON_ERROR_STOP=1 '{destination_db}' -f {fh.name}"))
+        subprocess.check_call(shlex.split(f"psql -v ON_ERROR_STOP=1 '{destination_db}' -f /powerscout/sql/03-after-migration.sql"))
+        raise SystemExit(0)
+    else:
+        print("Encountered error!")
+        print(stderr.decode())
+        raise SystemExit(migra_fh.returncode)
+
+EOF
+)

config/30add_path.sh

@@ -0,0 +1 @@
+export PATH=${PATH}:/powerscout/python/bin

config/Dockerfile

@@ -0,0 +1,59 @@
+# syntax=docker/dockerfile:1
+
+FROM docker.io/python:3.10-alpine AS builder
+
+RUN python -m pip install -U pip setuptools wheel && apk add zip
+WORKDIR /usr/local/bin
+ADD scripts/recursiveldd.py recursive-ldd
+
+WORKDIR /artifacts
+RUN python -m venv --symlinks --upgrade-deps /app && \
+    mkdir wheels && \
+    apk add zip && \
+    cd /usr/local/lib/python3.10 && \
+    python -m compileall && \
+    zip ../python310.zip $(find . -type d -path './site-packages' -prune -o $(find . -name "*.so" -print | xargs -n 1 dirname | sort | uniq | xargs -n1 -I {}\; echo '-type d -path ' {}\; '-prune -o') -type f) && \
+                           find . -type d -path './site-packages' -prune -o $(find . -name "*.so" -print | xargs -n 1 dirname | sort | uniq | xargs -n1 -I {}\; echo '-type d -path ' {}\; '-prune -o') -type f -exec rm {} \; && \
+    find . -empty -type d -delete && \
+    recursive-ldd /usr/local/bin/python $(find . -type f -name "*.so" -print)
+
+WORKDIR /
+
+WORKDIR /build/powerscout
+ADD requirements.deps.lock .
+RUN python -m pip wheel -r requirements.deps.lock -w /artifacts/wheels/
+ADD requirements.lock .
+RUN python -m pip wheel -r requirements.lock -w /artifacts/wheels/
+RUN /app/bin/python -m pip install --no-index -f /artifacts/wheels/ -r requirements.lock -r requirements.deps.lock
+
+ADD dist/ dist/
+RUN /app/bin/python -m pip install --no-index -f dist powerscout && \
+    /app/bin/python -m pip uninstall -y pip
+
+
+RUN export DIRECT_LDD_FILES=$(recursive-ldd /usr/local/bin/python $(find /usr/local/lib/python3.10 -type f -name "*.so" -print) | grep -vE '^/usr/local/lib/python3.10') && \
+    echo "${DIRECT_LDD_FILES}" && \
+    find /usr/local/lib/python3.10 /app \( -name '*.pyx' -o -name '*.pyi' -o -name '*.c' -o -name '*.pyd' \) -exec rm -f -- {} \; && \
+    mkdir /arch && \
+    tar -cf - \
+        /app \
+        /etc/passwd \
+        /usr/local/bin/python* \
+        /usr/local/lib/python310.zip \
+        /usr/local/lib/python3.10 \
+        ${DIRECT_LDD_FILES} \ | \
+    tar -C /arch -xvf -  && \
+    ls /arch
+
+FROM scratch
+ENV PYTHONHOME='/usr/local'
+EXPOSE 8080
+
+COPY --from=builder /lib/ld-musl-x86_64.so.1 /lib/ld-musl-x86_64.so.1
+COPY --from=builder /usr/bin/env /usr/bin/env
+COPY --from=builder /bin/sh /bin/sh
+COPY --from=builder /bin/busybox /bin/busybox
+COPY --from=builder /arch /
+
+ENTRYPOINT ["/app/bin/python", "-m", "powerscout"]
+CMD ["-d", "cli"]

config/Dockerfile.schema

@@ -0,0 +1,58 @@
+# syntax=docker/dockerfile:1
+
+FROM timescale/timescaledb:latest-pg14 AS builder
+WORKDIR /powerscout
+ADD db-requirements.lock /powerscout/db-requirements.lock
+RUN apk add --no-cache python3 gcc python3-dev musl-dev patchelf linux-headers zip && \
+    cd /usr/lib/python3.11 && \
+    (find . -name "__pycache__" -print0 | xargs -0 rm -rf ) && \
+    (find . -name "*.py" -print0 | xargs -0 python3 -m compileall -f -b -d /powerscout/python/lib) && \
+    (find . -name "__pycache__" -print0 | xargs -0 -r rm -rf ) && \
+    zip /tmp/python311.zip $(find . -type f -name "*.pyc" -print) && \
+    cd /powerscout && \
+    python3 -m ensurepip && \
+    python3 -m pip install -U pip && \
+    python3 -m pip install wheel && mkdir /wheels && \
+    python3 -m pip wheel -w /wheels -r db-requirements.lock && \
+    python3 -m venv --copies --without-pip python && \
+    # copy the runtime python library into the venv
+    cp /usr/lib/libpython3* python/lib/. && \
+    cp /tmp/python311.zip python/lib/. && \
+    cp -R $(echo /usr/lib/python3.*/lib-dynload) $(echo python/lib/python3.*)/lib-dynload && \
+    # Iterate through shared library dependencies, both successful ones that we can search for
+    # having ``/usr/lib`` and scraping the "Error: relocation ... failed" to find the remainder
+    (cd /usr/lib && \
+        for filename in $(\
+            for filename in $(echo python3.11/lib-dynload/*);\
+            do ldd $filename 2>&1 | grep -vE reloca; \
+            done | grep loading | cut -f5 -d' ' | cut -f1 -d':');\
+        do cp $filename /powerscout/python/lib/.;\
+        done) && \
+    # Gather error'ed "I can't find this shared object by name" pieces:
+    (cd /usr/lib && \
+        for filename in $(\
+            for filename in $(echo python3.11/lib-dynload/*); \
+            do ldd $filename 2>&1 | grep -vE reloca; \
+            done | grep '/usr/lib' | cut -f1 -d' '); \
+        do cp $filename /powerscout/python/lib/.; \
+        done) && \
+    # migra depends quietly on setuptools, so install it directly
+    python3 -m pip --python python/bin/python install \
+        setuptools && \
+    # install the dependencies
+    python3 -m pip --python python/bin/python install \
+        --find-links=/wheels -r db-requirements.lock && \
+    # patch the python executable to look at itself
+    for filename in $(echo /powerscout/python/bin/python*);\
+    do \
+        patchelf --set-rpath /powerscout/python/lib $filename; \
+    done && \
+    rm -rf /wheels
+RUN sed -i'' 's|/usr/bin|/powerscout/python/bin|g' /powerscout/python/pyvenv.cfg
+FROM timescale/timescaledb:latest-pg14
+ADD config/30add_path.sh /etc/profile.d/
+ENV PATH="${PATH}:/powerscout/python/bin"
+COPY --from=builder /powerscout /powerscout
+WORKDIR /powerscout
+ADD config/003_initialize_powerscout_schema.sh /docker-entrypoint-initdb.d/
+COPY sql/ /powerscout/sql

config/docker-compose.db.yml

@@ -0,0 +1,32 @@
+version: "3.9"
+# Boot a cluster that has a fresh timescaledb
+
+services:
+  api:
+    networks:
+      - frontend
+      - backend
+    depends_on:
+      - database
+    environment:
+      POWERSCOUT_POSTGRES_URI=${POWERSCOUT_POSTGRES_URI:-postgres://database/powerscout}
+
+  database:
+    image: autumnjolitz/powerscout-db
+    networks:
+      - backend
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    volumes:
+      -
+        type: 'volume'
+        source: 'database'
+        target: '/var/lib/postgresql/data'
+    ports:
+      - target: 5432
+        published: 5432
+        protocol: tcp
+        mode: host
+
+networks:
+  backend:

config/docker-compose.serve.yml

@@ -0,0 +1,9 @@
+version: "3.9"
+
+services:
+  api:
+    ports:
+      - target: 8080
+        published: "${POWERSCOUT_FORWARDED_PORT:-8080}"
+        protocol: tcp
+        mode: host

config/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3.9"
+
+services:
+  api:
+    image: autumnjolitz/powerscout
+    build:
+      dockerfile: ./config/Dockerfile
+      context: '..'
+    networks:
+      - frontend
+    environment:
+      - POWERSCOUT_POSTGRES_URI
+    ports:
+      - 8080
+  database:
+    image: autumnjolitz/powerscout-db
+    build:
+      dockerfile: ./config/Dockerfile.schema
+      context: '..'
+    ports:
+      - 5432
+
+networks:
+  frontend:
+
+volumes:
+  database:

config/example.yml

@@ -7,3 +7,5 @@ REDIS_URI: redis://localhost:6379/0
 GRAPHITE_FLUSH_PERIOD_SECONDS: 10
 MAX_GRAPHITE_QUEUE_LENGTH: 10
 LOWEST_GRID_VAC_ALLOWED: 110
+POSTGRES_URI: postgres://localhost/powerscout
+

config/nyx.yml

@@ -0,0 +1,5 @@
+GRAPHITE_HOSTNAME: nyx.lan
+APC_ENDPOINT: apcda36f5.lan
+APC_USERNAME: apc
+APC_PASSWORD: apc
+POSTGRES_URI: postgres://localhost:5432/powerscout

db-requirements.lock

@@ -0,0 +1,17 @@
+greenlet==2.0.2
+iniconfig==2.0.0
+migra==3.0.1663481299
+mirakuru==2.5.1
+packaging==23.1
+pluggy==1.2.0
+port-for==0.7.1
+psutil==5.9.5
+psycopg==3.1.10
+psycopg2-binary==2.9.7
+pytest==7.4.0
+pytest-postgresql==5.0.0
+schemainspect==3.1.1663587362
+six==1.16.0
+SQLAlchemy==2.0.19
+sqlbag==0.1.1617247075
+typing_extensions==4.7.1

db-requirements.txt

@@ -0,0 +1,3 @@
+migra
+psycopg2-binary
+pytest-postgresql

devel-requirements.txt

@@ -0,0 +1,4 @@
+invoke
+check-manifest
+black
+typing-extensions

powerscout.sublime-project

@@ -0,0 +1,11 @@
+{
+	"folders":
+	[
+		{
+			"path": "."
+		}
+	],
+    "settings": {
+        "python_interpreter": "/Users/BenJolitz/.virtualenvs/cpython36/bin/python",
+    }
+}