Merge branch 'release-v0.2.28'

CHANGELOG.rst

@@ -2,6 +2,22 @@
 Changelog
 =========
 
+0.2.28 - 2020-09-15
+-------------------
+
+Added
+^^^^^
+
+* Add bit_depth argument to mask_saturated, no longer convert to float64 by default (@AnthonyHorton #244)
+
+Changed
+^^^^^^^
+
+* Single cloudbuild file for both ``panoptes-base`` and ``panoptes-utils``. (#242)
+* Add ``astropy`` channel.
+* Remove the miniforge installer from the docker image and clean up build args. (@wtgee #245)
+* Changed relative to absolute imports. (@wtgee #246)
+
 0.2.27 - 2020-09-12
 -------------------
 

CONTRIBUTING.md

@@ -72,8 +72,8 @@ instead of `My File.py`.
 - Use root-relative imports (i.e. relative to the POCS directory). This means that rather
   than using a directory relative imports such as:
   ```python
-  from ..base import PanBase
-  from ..utils import current_time
+  from panoptes.utils.base import PanBase
+  from panoptes.utils.utils import current_time
   ```
   Import from the top-down instead:
   ```python

docker/base/Dockerfile

@@ -80,12 +80,21 @@ COPY --chown=panoptes:panoptes ./scripts/download-data.py "${PANDIR}/scripts/dow
 RUN wget -q "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \
         -O "${PANDIR}/scripts/install-miniforge.sh" && \
     /bin/sh "${PANDIR}/scripts/install-miniforge.sh" -b -f -p "${PANDIR}/conda" && \
+    # Remove install file.
+    rm "${PANDIR}/scripts/install-miniforge.sh" && \
     # Make sure to use conda for some of the larger modules.
-    "${PANDIR}/conda/bin/conda" create -y -n "${conda_env_name}" python=3.8 \
+    "${PANDIR}/conda/bin/conda" create \
+        --yes --name "${conda_env_name}" \
+        --channel astropy \
+        --channel conda-forge \
+        # Python version
+        python=3.8 \
+        # Packages
         astroplan \
         astropy \
         bokeh \
         click \
+        gevent \
         loguru \
         matplotlib \
         numpy \

docker/cloudbuild.yaml

@@ -4,25 +4,25 @@ options:
 timeout: 18000s  # 5 hours
 
 substitutions:
+  _TAG: latest
   _PLATFORMS: linux/amd64,linux/arm64
-  _BASE_IMAGE: panoptes-base:latest
+  _BASE_IMAGE: panoptes-base
   _IMAGE_NAME: panoptes-utils
   _REPO_URL: https://github.com/panoptes/panoptes-utils
-  _TAG: latest
 
 steps:
   # Fetch the repo from github
   - name: gcr.io/cloud-builders/git
     id: "clone-repo"
-    args: ["clone", "${_REPO_URL}"]
-    waitFor: ["-"]
+    args: [ "clone", "${_REPO_URL}" ]
+    waitFor: [ "-" ]
 
   # Pull the cached image.
   - name: 'gcr.io/cloud-builders/docker'
     id: "pull-cached-image"
     entrypoint: 'bash'
-    args: ['-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG} || exit 0']
-    waitFor: ["-"]
+    args: [ '-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG} || exit 0' ]
+    waitFor: [ "-" ]
 
   # Set up multiarch support
   - name: "gcr.io/cloud-builders/docker"
@@ -34,7 +34,7 @@ steps:
       - "--privileged"
       - "--rm"
       - "docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64"
-    waitFor: ["-"]
+    waitFor: [ "-" ]
 
   # Build builder
   - name: "gcr.io/cloud-builders/docker"
@@ -46,7 +46,23 @@ steps:
       - "create"
       - "--use"
       - "--driver=docker-container"
-    waitFor: ["setup-buildx"]
+    waitFor: [ "setup-buildx" ]
+
+  # Build panoptes-base.
+  - name: "gcr.io/cloud-builders/docker"
+    id: "build-base"
+    env:
+      - "DOCKER_CLI_EXPERIMENTAL=enabled"
+    args:
+      - "buildx"
+      - "build"
+      - "--push"
+      - "--platform=${_PLATFORMS}"
+      - "-f=docker/base/Dockerfile"
+      - "--tag=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}:${_TAG}"
+      - "--cache-from=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}:${_TAG}"
+      - "."
+    waitFor: [ "build-builder", "pull-cached-image" ]
 
   # Build with cloned panoptes-utils as source directory
   - name: "gcr.io/cloud-builders/docker"
@@ -59,8 +75,8 @@ steps:
       - "--push"
       - "--platform=${_PLATFORMS}"
       - "-f=docker/Dockerfile"
-      - "--build-arg=image_url=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}"
+      - "--build-arg=image_url=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}:${_TAG}"
       - "--tag=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}"
       - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}"
       - "panoptes-utils"
-    waitFor: ["build-builder", "clone-repo"]
+    waitFor: [ "build-base" ]

scripts/download-data.py

@@ -3,11 +3,9 @@
 import os
 import shutil
 
+import click
 from astroplan import download_IERS_A
 from astropy.utils import data
-
-import click
-
 from loguru import logger
 
 DEFAULT_DATA_FOLDER = os.path.expandvars("$PANDIR/astrometry/data")
@@ -117,16 +115,17 @@ def download_iers(self):
             download_IERS_A(show_progress=self.verbose)
             return True
         except Exception as e:
-            logger.warning(f'Failed to download IERS A bulletin: {e}')
+            logger.warning(f'Failed to download IERS A bulletin: {e!r}')
             return False
 
     def download_one_file(self, fn):
         """Downloads one astrometry.net file into self.data_folder."""
-        dest = "{}/{}".format(self.data_folder, os.path.basename(fn))
+        dest = f"{self.data_folder}/{os.path.basename(fn)}"
         if os.path.exists(dest):
             return True
-        url = "http://data.astrometry.net/{}".format(fn)
+        url = f"http://data.astrometry.net/{fn}"
         try:
+            logger.debug(f'Downloading {url=}')
             df = data.download_file(url)
         except Exception as e:
             if not self.keep_going:
@@ -135,7 +134,7 @@ def download_one_file(self, fn):
             return False
         # The file has been downloaded to some directory. Move the file into the data folder.
         try:
-            self.create_data_folder()
+            os.makedirs(self.data_folder, exist_ok=True)
             shutil.move(df, dest)
             return True
         except OSError as e:
@@ -144,12 +143,6 @@ def download_one_file(self, fn):
             logger.warning(f"Problem saving {url}. Check permissions: {e}")
             return False
 
-    def create_data_folder(self):
-        """Creates the data folder if it does not exist."""
-        if not os.path.exists(self.data_folder):
-            logger.info("Creating data folder: {}.".format(self.data_folder))
-            os.makedirs(self.data_folder)
-
 
 if __name__ == '__main__':
     main()

scripts/setup-local-environment.sh

@@ -28,7 +28,8 @@ build_base() {
 build_develop() {
   echo "Building local panoptes-utils:${TAG} from ${BASE_IMAGE_URL} in ${PANOPTES_UTILS}"
   docker build \
-    --build-arg "userid=$(id -u),image_url=${BASE_IMAGE_URL}" \
+    --build-arg userid="$(id -u)" \
+    --build-arg image_url="${BASE_IMAGE_URL}" \
     -t "panoptes-utils:${TAG}" \
     -f "${PANOPTES_UTILS}/docker/Dockerfile" \
     "${PANOPTES_UTILS}"

src/panoptes/utils/__init__.py

@@ -10,8 +10,19 @@
 finally:
     del get_distribution, DistributionNotFound
 
+# TODO Make this better.
+from panoptes.utils.utils import listify
+from panoptes.utils.utils import get_quantity_value
+from panoptes.utils.utils import altaz_to_radec
+from panoptes.utils.utils import image_id_from_path
+from panoptes.utils.utils import sequence_id_from_path
+from panoptes.utils.utils import get_free_space
+from panoptes.utils.utils import string_to_params
+from panoptes.utils.time import current_time
+from panoptes.utils.time import flatten_time
+from panoptes.utils.time import wait_for_events
+from panoptes.utils.time import CountdownTimer
+
 from loguru import logger
-from .utils import *
-from .time import *
 
 logger.disable('panoptes')

src/panoptes/utils/config/cli.py

@@ -1,10 +1,9 @@
 import time
 
 import click
-
-from .client import get_config, set_config, server_is_running
-from .server import config_server
-from ..logging import logger
+from panoptes.utils.config import server
+from panoptes.utils.config.client import get_config, set_config, server_is_running
+from panoptes.utils.logging import logger
 
 
 @click.group()
@@ -52,7 +51,7 @@ def run(context, config_file=None, save_local=True, load_local=False):
     """
     host = context.obj.get('host')
     port = context.obj.get('port')
-    server_process = config_server(
+    server_process = server.config_server(
         config_file,
         host=host,
         port=port,

src/panoptes/utils/config/client.py

@@ -2,11 +2,10 @@
 
 import requests
 from dotenv import load_dotenv
-
-from ..error import InvalidConfig
-from ..logging import logger
-from ..serializers import from_json
-from ..serializers import to_json
+from panoptes.utils.error import InvalidConfig
+from panoptes.utils.logging import logger
+from panoptes.utils.serializers import from_json
+from panoptes.utils.serializers import to_json
 
 load_dotenv()
 

src/panoptes/utils/config/helpers.py

@@ -1,10 +1,10 @@
 import os
 from contextlib import suppress
 
-from ..logging import logger
-from ..serializers import from_yaml
-from ..serializers import to_yaml
-from ..utils import listify
+from panoptes.utils.logging import logger
+from panoptes.utils.serializers import from_yaml
+from panoptes.utils.serializers import to_yaml
+from panoptes.utils.utils import listify
 
 
 def load_config(config_files=None, parse=True, load_local=True):

src/panoptes/utils/config/server.py

@@ -7,13 +7,12 @@
 from flask import request
 from flask.json import JSONEncoder
 from gevent.pywsgi import WSGIServer
+from panoptes.utils.config.helpers import load_config
+from panoptes.utils.config.helpers import save_config
+from panoptes.utils.logging import logger
+from panoptes.utils.serializers import serialize_object
 from scalpl import Cut
 
-from .helpers import load_config
-from .helpers import save_config
-from ..logging import logger
-from ..serializers import serialize_object
-
 # Turn off noisy logging for Flask wsgi server.
 logging.getLogger('werkzeug').setLevel(logging.WARNING)
 logging.getLogger('gevent').setLevel(logging.WARNING)

src/panoptes/utils/data/__init__.py

@@ -1 +1 @@
-from .metadata import get_metadata, search_observations
+from panoptes.utils.data.metadata import get_metadata, search_observations

src/panoptes/utils/data/metadata.py

@@ -1,18 +1,15 @@
 from contextlib import suppress
 
-from astropy.coordinates import SkyCoord
+import pandas as pd
 from astropy import units as u
+from astropy.coordinates import SkyCoord
 from astropy.utils.data import download_file
-
-import pandas as pd
-
 from dateutil.parser import parse as date_parse
+from panoptes.utils import listify
+from panoptes.utils.logging import logger
+from panoptes.utils.time import current_time
 from tqdm import tqdm
 
-from .. import listify
-from ..time import current_time
-from ..logging import logger
-
 OBS_BASE_URL = 'https://storage.googleapis.com/panoptes-observations'
 
 

src/panoptes/utils/database/__init__.py

@@ -1 +1 @@
-from .base import AbstractPanDB, PanDB
+from panoptes.utils.database.base import AbstractPanDB, PanDB

src/panoptes/utils/database/base.py

@@ -1,8 +1,8 @@
 import abc
 
-from ..time import current_time
-from ..library import load_module
-from ..logging import logger
+from panoptes.utils.library import load_module
+from panoptes.utils.logging import logger
+from panoptes.utils.time import current_time
 
 
 def create_storage_obj(collection, data, obj_id):