Merge branch 'release-v0.2.21'

.dockerignore

@@ -1,9 +1,9 @@
-docs/*
 .idea
 .venv
 venv
 
-.git
+# Need to pass git to allow version :(
+!.git
 .github
 
 *.md

.gcloudignore

@@ -1,9 +1,9 @@
-docs/*
 .idea
 .venv
 venv
 
-.git
+# Need to pass git to allow version :(
+!.git
 .github
 
 *.md

CHANGELOG.rst

@@ -2,6 +2,32 @@
 Changelog
 =========
 
+0.2.21 - 2020-07-05
+-------------------
+
+Added
+^^^^^
+
+* Added `arm64` build for Docker based off `ubuntu` image. (#223)
+
+Changed
+^^^^^^^
+
+* Docker
+
+  * Changed base image to `ubuntu`. (#223)
+  * `amd64` and `arm64` images built by default. (#223)
+  * Ubuntu has changed `sextractor` to `source-extractor` (yay). (#223)
+
+* Config Server
+
+  * Better parsing of directories entry in config server. (#222)
+  * Make config server less noisy. (#222)
+
+* Bump PyYaml to latest for security warning. (#222)
+* Remove pendulum because too hard to build on `arm processors <https://github.com/sdispater/pendulum/issues/457>`_. (#223)
+
+
 0.2.20 - 2020-06-09
 -------------------
 

bin/cr2-to-jpg

@@ -55,6 +55,12 @@ else
     fi
 fi
 
+# Test for file
+if [[ ! -s "${JPG}" ]]; then
+    echo "JPG was not extracted successfully."
+    exit 1
+fi
+
 if [[ -n "$TITLE" ]]
   then
   	echo "Adding title \"${TITLE}\""

conftest.py

@@ -129,14 +129,17 @@ def db_type(request):
     if request.param not in db_list and 'all' not in db_list:  # pragma: no cover
         pytest.skip(f"Skipping {request.param} DB, set --test-all-databases=True")
 
-    PanDB.permanently_erase_database(
-        request.param, 'panoptes_testing', really='Yes', dangerous='Totally')
+    PanDB.permanently_erase_database(request.param,
+                                     'panoptes_testing',
+                                     storage_dir='testing',
+                                     really='Yes',
+                                     dangerous='Totally')
     return request.param
 
 
 @pytest.fixture(scope='function')
 def db(db_type):
-    return PanDB(db_type=db_type, db_name='panoptes_testing', connect=True)
+    return PanDB(db_type=db_type, db_name='panoptes_testing', storage_dir='testing', connect=True)
 
 
 @pytest.fixture(scope='function')

docker/cloudbuild.yaml

@@ -1,18 +1,45 @@
 steps:
-- name: 'docker'
-  id: 'amd64-build'
-  args:
-  - 'build'
-  - '-f=docker/${_TAG}.Dockerfile'
-  - '--tag=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}'
-  - '.'
+  # Set up multiarch support
+  - name: 'gcr.io/cloud-builders/docker'
+    id: 'setup-buildx'
+    env:
+      - 'DOCKER_CLI_EXPERIMENTAL=enabled'
+    args:
+      - 'run'
+      - '--privileged'
+      - '--rm'
+      - 'docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64'
+    waitFor: ['-']
 
-- name: 'docker'
-  id: 'amd64-push'
-  args:
-  - 'push'
-  - 'gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}'
-  waitFor: ['amd64-build']
+  # Build builder
+  - name: 'gcr.io/cloud-builders/docker'
+    id: 'build-builder'
+    env:
+      - 'DOCKER_CLI_EXPERIMENTAL=enabled'
+    args:
+      - 'buildx'
+      - 'create'
+      - '--use'
+      - '--driver=docker-container'
+    waitFor: ['setup-buildx']
+
+  # Build
+  - name: 'gcr.io/cloud-builders/docker'
+    id: 'build-images'
+    env:
+      - 'DOCKER_CLI_EXPERIMENTAL=enabled'
+    args:
+      - 'buildx'
+      - 'build'
+      - '--push'
+      - '--platform=linux/amd64,linux/arm64'
+      - '-f=docker/${_TAG}.Dockerfile'
+      - '--tag=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}'
+      - '--cache-from=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}'
+      - '.'
+    waitFor: ['build-builder']
 
 images:
-  - 'gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}'
+  - 'gcr.io/${PROJECT_ID}/panoptes-utils:latest'
+  - 'gcr.io/${PROJECT_ID}/panoptes-utils:amd64'
+  - 'gcr.io/${PROJECT_ID}/panoptes-utils:arm64'

docker/develop.Dockerfile

@@ -14,24 +14,19 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
 ENV SHELL /bin/zsh
 
-ARG userid=1000
-
-ENV USERID $userid
 ENV PANDIR $pan_dir
 ENV PANLOG "$pan_dir/logs"
 ENV PANUSER panoptes
 ENV POCS $pocs_dir
 ENV SOLVE_FIELD /usr/bin/solve-field
 
 # Install module
-USER ${PANUSER}
-COPY --chown=panoptes:panoptes . "${PANDIR}/panoptes-utils/"
+COPY . "${PANDIR}/panoptes-utils/"
 RUN wget -qO- $cr2_url > "${PANDIR}/panoptes-utils/tests/data/canon.cr2" && \
     cd "${PANDIR}/panoptes-utils" && \
-    pip install -e ".[testing]"
+    pip3 install -e ".[testing,google]"
 
 # Cleanup apt.
-USER root
 RUN apt-get autoremove --purge -y && \
     apt-get -y clean && \
     rm -rf /var/lib/apt/lists/* && \

docker/latest.Dockerfile

@@ -1,4 +1,4 @@
-ARG IMAGE_URL=python:3.8-slim-buster
+ARG IMAGE_URL=ubuntu
 
 FROM ${IMAGE_URL} AS base-image
 LABEL description="Installs the panoptes-utils module from pip. \
@@ -32,7 +32,9 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         gosu wget curl bzip2 ca-certificates zsh openssh-client nano \
         astrometry.net sextractor dcraw exiftool libcfitsio-dev libcfitsio-bin imagemagick \
-        libfreetype6-dev libpng-dev fonts-lato libsnappy-dev \
+        libfreetype6-dev libpng-dev fonts-lato libsnappy-dev libjpeg-dev \
+        python3-pip python3-scipy python3-dev python3-pandas python3-matplotlib \
+        libffi-dev libssl-dev \
         gcc git pkg-config sudo && \
     # Oh My ZSH. :)
     mkdir -p "${ZSH_CUSTOM}" && \
@@ -57,18 +59,25 @@ RUN apt-get update && \
     # Update permissions for current user.
     chown -R ${PANUSER}:${PANUSER} "/home/${panuser}" && \
     chown -R ${PANUSER}:${PANUSER} ${PANDIR} && \
-    # Install module
-    pip install "panoptes-utils[testing]" && \
     # astrometry.net folders
     mkdir -p "${astrometry_dir}" && \
     echo "add_path ${astrometry_dir}" >> /etc/astrometry.cfg && \
+    # Preinstall some modules.
+    pip3 install astropy astroplan click loguru && \
     # astrometry.net index files
-    python /tmp/download-data.py \
+    python3 /tmp/download-data.py \
         --wide-field --narrow-field \
         --folder "${astrometry_dir}" \
         --verbose && \
     chown -R ${PANUSER}:${PANUSER} ${astrometry_dir} && \
     chmod -R 777 ${astrometry_dir} && \
+    # Allow sudo without password
+    echo "$PANUSER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# Install module
+COPY . "${PANDIR}/panoptes-utils"
+RUN cd "${PANDIR}/panoptes-utils" && \
+    pip3 install ".[testing,google]" && \
     # Cleanup
     apt-get autoremove --purge -y \
         autoconf \

docs/config-server.rst

@@ -193,17 +193,6 @@ The ``panoptes-config-server set`` command will set the value for the given key.
 .. code-block:: bash
 
     $ panoptes-config-server set 'location.horizon' '37 deg'
-    {
-      "elevation": 3400,
-      "flat_horizon": -6,
-      "focus_horizon": -12,
-      "gmt_offset": -600,
-      "horizon": "37 deg",
-      "latitude": 19.54,
-      "longitude": -155.58,
-      "name": "New Location",
-      "observe_horizon": -18,
-      "timezone": "US/Hawaii"
-    }
+    {'location.horizon': <Quantity 37. deg>}
 
-See ``panoptes-config-server get --help`` and ``panoptes-config-server set --help`` for more details.
+See ``panoptes-config-server get --help`` and ``panoptes-config-server set --help`` for more details.

resources/sextractor/panoptes.sex

@@ -18,7 +18,7 @@ DETECT_THRESH    3.0            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2
 ANALYSIS_THRESH  1.5            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2
 
 FILTER           Y              # apply filter for detection (Y or N)?
-FILTER_NAME      /usr/share/sextractor/default.conv   # name of the file containing the filter
+FILTER_NAME      /usr/share/source-extractor/default.conv   # name of the file containing the filter
 
 DEBLEND_NTHRESH  32             # Number of deblending sub-thresholds
 DEBLEND_MINCONT  0.005          # Minimum contrast parameter for deblending
@@ -59,7 +59,7 @@ PIXEL_SCALE      10.3           # size of pixel in arcsec (0=use FITS WCS info)
 #------------------------- Star/Galaxy Separation ----------------------------
 
 SEEING_FWHM      1.5            # stellar FWHM in arcsec
-STARNNW_NAME     /usr/share/sextractor/default.nnw    # Neural-Network_Weight table filename
+STARNNW_NAME     /usr/share/source-extractor/default.nnw    # Neural-Network_Weight table filename
 
 #------------------------------ Background -----------------------------------
 
@@ -99,6 +99,6 @@ VERBOSE_TYPE     NORMAL         # can be QUIET, NORMAL or FULL
 HEADER_SUFFIX    .head          # Filename extension for additional headers
 WRITE_XML        N              # Write XML file (Y/N)?
 XML_NAME         sex.xml        # Filename for XML output
-XSL_URL          file:///usr/local/share/sextractor/sextractor.xsl
+XSL_URL          file:///usr/share/source-extractor/sextractor.xsl
                                 # Filename for XSL style-sheet
 

scripts/testing/run-tests.sh

@@ -9,7 +9,7 @@ coverage erase
 
 # Run coverage over the pytest suite.
 echo "Starting tests"
-coverage run "$(command -v pytest)"
+coverage run "$(command -v pytest-3)"
 
 echo "Combining coverage"
 coverage combine