Merge branch 'release-v0.2.20'

.github/workflows/pythontest.yaml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

.readthedocs.yaml

@@ -12,7 +12,7 @@ sphinx:
 formats: all
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: docs/requirements.txt
     - method: pip

CHANGELOG.rst

@@ -2,7 +2,45 @@
 Changelog
 =========
 
-All notable changes to this project will be documented in this file.
+0.2.20 - 2020-06-09
+-------------------
+
+Moving to python 3.8.
+
+Changed
+^^^^^^^
+
+* **Breaking** Python minimum version changed to ``3.8``. (#217)
+* Running pytest locally will generate coverage report in terminal. (#218)
+* Lots of documentation. (#218)
+* Removing the environment section from the readme. (#218)
+* Config Server (#217)
+
+  * Better logging.
+  * Cleaning up doctests.
+  * Removing all dynamic server items from this repo as they are not needed.
+  * Wait for config_server to start.
+  * Fixing starting within fixture.
+  * Config items no longer assume any defaults for either directories or files. A config file name is always required and it should always be an absolute path. (#218)
+  * Adding test file for config items. (#218)
+  * ``panoptes-config-server`` re-worked and now includes ``run``, ``get``, and ``set`` subcomamnds. (#221)
+
+* Testing (#218)
+
+  * Log files are rotated for each testing run.
+  * Fix env vars (mostly need to make sure the `export` option exists in the `env` file.
+  * Pytest commands moved to `setup.cfg` instead of `run-tests.sh`
+  * Remove old markers
+  * Setting `--strict-markers` options.
+  * Add `astrometry` marker for tests requiring solve and `theskyx` marker for running alongside TheSkyX.
+  * Coverage reports generated in xml and output in terminal.
+
+* Serializers update. (#217)
+
+  * Make the parsing and serializing functions public.
+  * Use pendulum for parsing times instead of astropy Time.
+  * Better naming of public functions. (#218)
+
 
 0.2.19 - 2020-06-04
 -------------------

bin/cr2-to-jpg

@@ -1,4 +1,5 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
 
 usage() {
   echo -n "##################################################

conftest.py

@@ -1,14 +1,5 @@
-# This is in the root PANDIR directory so that pytest will recognize the
-# options added below without having to also specify pocs/test, or a
-# one of the tests in that directory, on the command line; i.e. pytest
-# doesn't load pocs/tests/conftest.py until after it has searched for
-# tests.
-# In addition, there are fixtures defined here that are available to
-# all tests, not just those in pocs/tests.
-
 import os
 import copy
-import subprocess
 import pytest
 import time
 import shutil
@@ -20,6 +11,7 @@
 
 from panoptes.utils.logging import logger
 from panoptes.utils.database import PanDB
+from panoptes.utils.config.client import get_config
 from panoptes.utils.config.client import set_config
 from panoptes.utils.config.server import config_server
 
@@ -40,40 +32,34 @@
           "<blue>({time:HH:mm:ss.ss})</> " \
           "| <c>{name} {function}:{line}</c> | " \
           "<lvl>{message}</lvl>\n"
+
+startup_message = ' STARTING NEW PYTEST RUN '
 logger.add(log_file_path,
            enqueue=True,  # multiprocessing
            format=log_fmt,
            colorize=True,
            backtrace=True,
            diagnose=True,
+           catch=True,
+           # Start new log file for each testing run.
+           rotation=lambda msg, _: startup_message in msg,
            level='TRACE')
+logger.log('testing', '*' * 25 + startup_message + '*' * 25)
 
 
 def pytest_addoption(parser):
     db_names = ",".join(_all_databases) + ' (or all for all databases)'
     group = parser.getgroup("PANOPTES pytest options")
     group.addoption(
-        "--with-hardware",
-        nargs='+',
-        default=[],
-        help=("A comma separated list of hardware to test."))
-    group.addoption(
-        "--without-hardware",
-        nargs='+',
-        default=[],
-        help=("A comma separated list of hardware to NOT test. "))
-    group.addoption(
-        "--solve",
+        "--astrometry",
         action="store_true",
         default=False,
         help="If tests that require solving should be run")
     group.addoption(
-        "--test-cloud-storage",
+        "--theskyx",
         action="store_true",
         default=False,
-        dest="test_cloud_storage",
-        help="Tests cloud storage functions." +
-             "Requires $PANOPTES_CLOUD_KEY to be set to path of valid json service key")
+        help="If running tests alongside a running TheSkyX program.")
     group.addoption(
         "--test-databases",
         nargs="+",
@@ -82,27 +68,6 @@ def pytest_addoption(parser):
               ". Note that travis-ci will test all of them by default."))
 
 
-@pytest.fixture(scope='session')
-def config_host():
-    return 'localhost'
-
-
-@pytest.fixture(scope='session')
-def static_config_port():
-    """Used for the session-scoped config_server where no config values
-    are expected to change during testing.
-    """
-    return '6563'
-
-
-@pytest.fixture(scope='module')
-def config_port():
-    """Used for the function-scoped config_server when it is required to change
-    config values during testing. See `dynamic_config_server` docs below.
-    """
-    return '4861'
-
-
 @pytest.fixture(scope='session')
 def db_name():
     return 'panoptes_testing'
@@ -116,95 +81,45 @@ def images_dir(tmpdir_factory):
 
 @pytest.fixture(scope='session')
 def config_path():
-    return os.path.join(os.getenv('PANDIR'),
-                        'panoptes-utils',
-                        'tests',
-                        'panoptes_utils_testing.yaml'
-                        )
+    return os.path.expandvars('${PANDIR}/panoptes-utils/tests/panoptes_utils_testing.yaml')
 
 
 @pytest.fixture(scope='session', autouse=True)
-def static_config_server(config_host, static_config_port, config_path, images_dir, db_name):
-    print(f'Starting config_server for testing session')
+def static_config_server(config_path, images_dir, db_name):
+    logger.log('testing', f'Starting static_config_server for testing session')
 
     proc = config_server(
-        host=config_host,
-        port=static_config_port,
         config_file=config_path,
         ignore_local=True,
+        auto_save=False
     )
 
-    print(f'config_server started with PID={proc.pid}')
+    logger.log('testing', f'static_config_server started with {proc.pid=}')
 
     # Give server time to start
-    time.sleep(1)
-
-    # Adjust various config items for testing
-    unit_name = 'Generic PANOPTES Unit'
-    unit_id = 'PAN000'
-    print(f'Setting testing name and unit_id to {unit_id}')
-    set_config('name', unit_name, port=static_config_port)
-    set_config('pan_id', unit_id, port=static_config_port)
+    while get_config('name') is None:  # pragma: no cover
+        logger.log('testing', f'Waiting for static_config_server {proc.pid=}, sleeping 1 second.')
+        time.sleep(1)
 
-    print(f'Setting testing database to {db_name}')
-    set_config('db.name', db_name, port=static_config_port)
-
-    fields_file = 'simulator.yaml'
-    print(f'Setting testing scheduler fields_file to {fields_file}')
-    set_config('scheduler.fields_file', fields_file, port=static_config_port)
-
-    # TODO(wtgee): determine if we need separate directories for each module.
-    print(f'Setting temporary image directory for testing')
-    set_config('directories.images', images_dir, port=static_config_port)
-
-    yield
-    print(f'Killing config_server started with PID={proc.pid}')
-    proc.terminate()
-
-
-@pytest.fixture(scope='function')
-def dynamic_config_server(config_host, config_port, config_path, images_dir, db_name):
-    """If a test requires changing the configuration we use a function-scoped testing
-    server. We only do this on tests that require it so we are not constantly starting and stopping
-    the config server unless necessary.  To use this, each test that requires it must use the
-    `dynamic_config_server` and `config_port` fixtures and must pass the `config_port` to all
-    instances that are created (propogated through PanBase).
-    """
-
-    print(f'Starting config_server for testing function')
-
-    proc = config_server(
-        host=config_host,
-        port=config_port,
-        config_file=config_path,
-        ignore_local=True,
-    )
-
-    print(f'config_server started with PID={proc.pid}')
-
-    # Give server time to start
-    time.sleep(1)
+    logger.log('testing', f'Startup config_server name=[{get_config("name")}]')
 
     # Adjust various config items for testing
-    unit_name = 'Generic PANOPTES Unit'
     unit_id = 'PAN000'
-    print(f'Setting testing name and unit_id to {unit_id}')
-    set_config('name', unit_name, port=config_port)
-    set_config('pan_id', unit_id, port=config_port)
+    logger.log('testing', f'Setting testing name and unit_id to {unit_id}')
+    set_config('pan_id', unit_id)
 
-    print(f'Setting testing database to {db_name}')
-    set_config('db.name', db_name, port=config_port)
+    logger.log('testing', f'Setting testing database to {db_name}')
+    set_config('db.name', db_name)
 
     fields_file = 'simulator.yaml'
-    print(f'Setting testing scheduler fields_file to {fields_file}')
-    set_config('scheduler.fields_file', fields_file, port=config_port)
+    logger.log('testing', f'Setting testing scheduler fields_file to {fields_file}')
+    set_config('scheduler.fields_file', fields_file)
 
-    # TODO(wtgee): determine if we need separate directories for each module.
-    print(f'Setting temporary image directory for testing')
-    set_config('directories.images', images_dir, port=config_port)
+    logger.log('testing', f'Setting temporary image directory for testing')
+    set_config('directories.images', images_dir)
 
     yield
-    print(f'Killing config_server started with PID={proc.pid}')
+    logger.log('testing', f'Killing static_config_server started with PID={proc.pid}')
     proc.terminate()
 
 

docker/setup-local-environment.sh

@@ -6,7 +6,7 @@ cd "${PANOPTES_UTILS}"
 
 echo "Building local panoptes-utils:latest"
 docker build \
-    --quiet \
+    --quiet --force-rm \
     -t "panoptes-utils:latest" \
     -f "${PANOPTES_UTILS}/docker/latest.Dockerfile" \
     "${PANOPTES_UTILS}"
@@ -16,7 +16,7 @@ sed -i s'/^\.git$/\!\.git/' .dockerignore
 
 echo "Building local panoptes-utils:develop"
 docker build \
-    --quiet \
+    --quiet --force-rm \
     --build-arg IMAGE_URL="panoptes-utils:latest" \
     -t "panoptes-utils:develop" \
     -f "${PANOPTES_UTILS}/docker/develop.Dockerfile" \