Merge branch 'release-v0.2.27'

.github/workflows/create-release.yml

@@ -5,7 +5,7 @@ on:
     tags:
       - 'v[0-9].[0-9]+.[0-9]+' # Push events to matching vX.Y.Z, but not vX.Y.Zdev
 
-name: Create Release
+name: Create GitHub Release
 
 jobs:
   build:
@@ -26,3 +26,23 @@ jobs:
             See Changelog for details
           draft: false
           prerelease: false
+  deploy:
+    name: Push Release to PyPi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.github/workflows/pythontest.yaml

@@ -1,12 +1,12 @@
 name: Test Python
 
-on: [push, pull_request]
+on: [ push, pull_request ]
 jobs:
   lint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [ 3.8 ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -25,35 +25,27 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [ 3.8 ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Build panoptes-utils image
         run: |
-          PANOPTES_UTILS=${PWD} INCLUDE_BASE=true scripts/setup-local-environment.sh
+          INCLUDE_BASE=true scripts/setup-local-environment.sh
       - name: Test with pytest in panoptes-utils container
         run: |
-          mkdir -p coverage_dir && chmod 777 coverage_dir
-          ci_env=`bash <(curl -s https://codecov.io/env)`
-          docker run -i \
-            $ci_env \
-            -e REPORT_FILE="/tmp/coverage/coverage.xml" \
-            --network "host" \
-            -v $PWD/coverage_dir:/var/panoptes/logs \
-            -v $PWD/coverage_dir:/tmp/coverage \
-            panoptes-utils:develop \
-            "/var/panoptes/panoptes-utils/scripts/testing/run-tests.sh"
+          mkdir -p logs && chmod -R 777 logs
+          PANLOG="${PWD}/logs" SLEEP_TIME=0 scripts/testing/test-software.sh
       - name: Upload coverage report to codecov.io
         uses: codecov/codecov-action@v1
         if: success()
         with:
           name: codecov-upload
-          file: coverage_dir/coverage.xml
+          file: logs/coverage.xml
           fail_ci_if_error: true
       - name: Create log file artifact
         uses: actions/upload-artifact@v1
         if: always()
         with:
           name: log-files
-          path: coverage_dir/panoptes-testing.log
+          path: logs/panoptes-testing.log

.travis.yml

@@ -10,9 +10,8 @@ python:
 services:
   - docker
 before_install:
-  - PANOPTES_UTILS="${TRAVIS_BUILD_DIR}" INCLUDE_BASE=true scripts/setup-local-environment.sh
+  - PANDIR="${PWD}/../" INCLUDE_BASE=true scripts/setup-local-environment.sh
 install: true
 script:
-  - docker run -it
-    panoptes-utils:develop
-    "/var/panoptes/panoptes-utils/scripts/testing/run-tests.sh"
+  - mkdir -p logs && chmod -R 777 logs
+  - PANLOG="${PWD}/logs" SLEEP_TIME=0 scripts/testing/test-software.sh

CHANGELOG.rst

@@ -2,7 +2,67 @@
 Changelog
 =========
 
-0.2.24 - 2020-08-20
+0.2.27 - 2020-09-12
+-------------------
+
+Added
+^^^^^
+
+* Config server startup controlled via envvars, incorporating ``python-dotenv`` (@wtgee #241):
+
+  * ``PANOPTES_CONFIG_HOST`` and ``PANOPTES_CONFIG_PORT``
+  * ``PANOPTES_CONFIG_FILE``
+
+Changed
+^^^^^^^
+
+* Config server updates (@wtgee #241):
+
+  * Config server with project dir mounted can be started via ``docker/docker-testing.yaml``.
+  * Config server flask instances are run on ``gevent`` wsgi server instead of development server.
+  * ``host`` and ``port`` are specified at the top-level command, e.g. ``panoptes-config-server --host foobar --port 9999 get``.
+  * ``host`` and ``port`` respect the above envvars above.
+  * Options changed from ``auto-save`` and ``ignore-local`` to ``save-local`` and ``load-local``.
+  * ``run`` command adds the``config_server.running=True`` entry to the server.
+  * ``stop`` command added that sets ``config_server.running=False`` to break loop.
+
+
+* Testing  (@wtgee #241): 
+
+  * All testing is started from ``scripts/test-software.sh``.
+  * The ``panoptes-config-server`` is started as an external service, not in the pytest conf.
+  * Added a ``tests/env`` file that is used by the docker compose file for setting vars inside the running containers.
+  * Config server uses ``tests/testing.yaml`` for all testing.
+  * Testing logs are stored in ``./logs``, relative to the project root outside the container.
+  * Coverage file is stored in ``/var/panoptes/logs``.
+  * A ``scripts/wait-for-it.sh`` script added to check that config server is running properly before starting tests.
+
+
+Fixed
+^^^^^
+
+* ``scripts/setup-local-environment.sh`` properly uses new base image if requested. (@wtgee #241)
+* Docker images: fixed the created ``ssh`` directory for ``$PANUSER``. (@wtgee #240)
+
+
+0.2.26 - 2020-08-21
+-------------------
+
+This release is mostly cleanup and testing of our autobuild features.
+
+Changed
+^^^^^^^
+
+* Splitting the `panoptes-base` files into separate folder. (#238)
+* Consolidate the GitHub Actions for building and publishing a release package. (#239)
+
+Fixed
+^^^^^
+
+* Fix Github Actions for building releases. (#238)
+
+
+0.2.25 - 2020-08-20
 -------------------
 
 Added

conftest.py

@@ -1,43 +1,36 @@
-import os
 import copy
-import pytest
-import time
+import logging
+import os
 import shutil
 import tempfile
-
-import logging
-from _pytest.logging import caplog as _caplog
 from contextlib import suppress
 
-from panoptes.utils.logging import logger
-from panoptes.utils.database import PanDB
-from panoptes.utils.config.client import get_config
-from panoptes.utils.config.client import set_config
-from panoptes.utils.config.server import config_server
-
 # Doctest modules
 import numpy as np
+import pytest
+from _pytest.logging import caplog as _caplog  # noqa
 from matplotlib import pyplot as plt
+from panoptes.utils.database import PanDB
+from panoptes.utils.logging import logger
 
 _all_databases = ['file', 'memory']
 
 logger.enable('panoptes')
 logger.level("testing", no=15, icon="🤖", color="<YELLOW><black>")
-log_file_path = os.path.join(
-    os.getenv('PANLOG', '/var/panoptes/logs'),
-    'panoptes-testing.log'
-)
 log_fmt = "<lvl>{level:.1s}</lvl> " \
           "<light-blue>{time:MM-DD HH:mm:ss.ss!UTC}</>" \
           "<blue>({time:HH:mm:ss.ss})</> " \
           "| <c>{name} {function}:{line}</c> | " \
           "<lvl>{message}</lvl>\n"
 
-startup_message = ' STARTING NEW PYTEST RUN '
+# Put the log file in the tmp dir.
+log_file_path = os.path.expandvars('${PANLOG}/panoptes-testing.log')
+startup_message = f' STARTING NEW PYTEST RUN - LOGS: {log_file_path} '
 logger.add(log_file_path,
            enqueue=True,  # multiprocessing
            format=log_fmt,
            colorize=True,
+           # TODO decide on these options
            backtrace=True,
            diagnose=True,
            catch=True,
@@ -69,58 +62,18 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope='session')
-def db_name():
-    return 'panoptes_testing'
+def config_host():
+    return os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
 
 
 @pytest.fixture(scope='session')
-def images_dir(tmpdir_factory):
-    directory = tmpdir_factory.mktemp('images')
-    return str(directory)
+def config_port():
+    return os.getenv('PANOPTES_CONFIG_PORT', 6563)
 
 
 @pytest.fixture(scope='session')
 def config_path():
-    return os.path.expandvars('${PANDIR}/panoptes-utils/tests/panoptes_utils_testing.yaml')
-
-
-@pytest.fixture(scope='session', autouse=True)
-def static_config_server(config_path, images_dir, db_name):
-    logger.log('testing', f'Starting static_config_server for testing session')
-
-    proc = config_server(
-        config_file=config_path,
-        ignore_local=True,
-        auto_save=False
-    )
-
-    logger.log('testing', f'static_config_server started with {proc.pid=}')
-
-    # Give server time to start
-    while get_config('name') is None:  # pragma: no cover
-        logger.log('testing', f'Waiting for static_config_server {proc.pid=}, sleeping 1 second.')
-        time.sleep(1)
-
-    logger.log('testing', f'Startup config_server name=[{get_config("name")}]')
-
-    # Adjust various config items for testing
-    unit_id = 'PAN000'
-    logger.log('testing', f'Setting testing name and unit_id to {unit_id}')
-    set_config('pan_id', unit_id)
-
-    logger.log('testing', f'Setting testing database to {db_name}')
-    set_config('db.name', db_name)
-
-    fields_file = 'simulator.yaml'
-    logger.log('testing', f'Setting testing scheduler fields_file to {fields_file}')
-    set_config('scheduler.fields_file', fields_file)
-
-    logger.log('testing', f'Setting temporary image directory for testing')
-    set_config('directories.images', images_dir)
-
-    yield
-    logger.log('testing', f'Killing static_config_server started with PID={proc.pid}')
-    proc.terminate()
+    return os.getenv('PANOPTES_CONFIG_FILE', '/var/panoptes/panoptes-utils/tests/testing.yaml')
 
 
 @pytest.fixture(scope='function', params=_all_databases)
@@ -139,6 +92,7 @@ def db_type(request):
 
 @pytest.fixture(scope='function')
 def db(db_type):
+    # TODO don't hardcode the db name.
     return PanDB(db_type=db_type, db_name='panoptes_testing', storage_dir='testing', connect=True)
 
 

docker/Dockerfile

@@ -7,16 +7,18 @@ LABEL maintainers="[email protected]"
 LABEL repo="github.com/panoptes/panoptes-utils"
 
 ARG panuser=panoptes
+ARG userid=1000
 ARG pan_dir=/var/panoptes
 ARG conda_env_name="panoptes"
 
 ARG pip_extras="[testing,google,social]"
 
+ENV USERID $userid
 ENV PANUSER $panuser
 ENV PANDIR $pan_dir
 
 # Install module
-USER ${PANUSER}
+USER "${PANUSER}"
 WORKDIR "${PANDIR}/panoptes-utils"
 
 # Can't seem to get around the hard-coding here.