Merge branch 'release-0.2.1'

CHANGELOG.md

@@ -6,24 +6,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 - [Changelog](#changelog)
-  - [[0.2.0] - 2020-03-04](#020---2020-03-04)
-    - [Added](#added)
+  - [[0.2.1] - 2020-03-04](#021---2020-03-04)
     - [Bug fixes](#bug-fixes)
     - [Changed](#changed)
+  - [[0.2.0] - 2020-03-04](#020---2020-03-04)
+    - [Added](#added)
+    - [Bug fixes](#bug-fixes-1)
+    - [Changed](#changed-1)
   - [[0.1.0] - 2020-03-04](#010---2020-03-04)
   - [[0.0.8] - 2019-06-29](#008---2019-06-29)
     - [Added](#added-1)
-    - [Bug fixes](#bug-fixes-1)
-    - [Changed](#changed-1)
+    - [Bug fixes](#bug-fixes-2)
+    - [Changed](#changed-2)
   - [[0.0.7] - 2019-05-26](#007---2019-05-26)
     - [Added](#added-2)
-    - [Changed](#changed-2)
+    - [Changed](#changed-3)
   - [[0.0.6] - 2019-04-29](#006---2019-04-29)
     - [Added](#added-3)
-    - [Changed](#changed-3)
+    - [Changed](#changed-4)
   - [[0.0.5] - 2019-04-09](#005---2019-04-09)
     - [Added](#added-4)
-    - [Changed](#changed-4)
+    - [Changed](#changed-5)
+
+## [0.2.1] - 2020-03-04
+
+### Bug fixes
+
+* Misc bugs introduced as part of last release, including to `download-data.py` script.
+
+### Changed
+
+* New script for downloading data, `scripts/download-data.py`. This helped resolve some issues with the relative imports introduced in `v0.2.0` and is cleaner. (#129)
 
 ## [0.2.0] - 2020-03-04
 

conda-requirements-amd64.yaml

@@ -1,4 +1,3 @@
-name: panoptes
 channels:
   - astropy
   - conda-forge
@@ -21,7 +20,6 @@ dependencies:
   - pytest-remotedata>=0.3.1
   - pytest
   - python-dateutil
-  - python-json-logger
   - PyYAML
   - pyzmq
   - readline

docker/Dockerfile.utils

@@ -21,8 +21,9 @@ RUN cd && \
     # First deal with pip and PyYAML - see https://github.com/pypa/pip/issues/5247
     pip install --no-cache-dir --no-deps --ignore-installed pip PyYAML && \
     # Install anaconda packages
-    /opt/conda/bin/conda install --yes -c conda-forge -c astropy \
-        --file "${PANDIR}/panoptes-utils/conda-requirements-${arch}.yaml" && \
+    /opt/conda/bin/conda env update --name base \
+        --file "${PANDIR}/panoptes-utils/conda-requirements-${arch}.yaml" \
+        --prune && \
     /opt/conda/bin/conda clean --all --yes && \
     /opt/conda/bin/conda clean -f --yes && \
     # Install requirements
@@ -34,9 +35,10 @@ RUN cd && \
     chgrp -R panoptes /opt/conda && \
     chmod -R 775 /opt/conda && \
     # Download astrometry.net files
-    python panoptes/utils/data.py \
+    python scripts/download-data.py \
         --wide-field --narrow-field \
-        --folder /astrometry/ && \
+        --folder /astrometry/ \
+        --verbose && \
     # Cleanup apt.
     apt-get autoremove --purge -y gcc pkg-config && \
     apt-get autoremove --purge -y && \

panoptes/utils/data.py

@@ -1,28 +1,13 @@
 # Downloads IERS Bulletin A (Earth Orientation Parameters, used by astroplan)
 # and astrometry.net indices.
 
-import argparse
 import os
 import shutil
-import sys
-import warnings
 
-from .logger import logger
-
-# Use custom location for download
-from astropy.utils.iers import conf as iers_conf
-iers_conf.iers_auto_url = 'https://storage.googleapis.com/panoptes-resources/iers/ser7.dat'
-iers_conf.iers_auto_url_mirror = 'https://storage.googleapis.com/panoptes-resources/iers/ser7.dat'
-
-# Importing download_IERS_A can emit a scary warnings, so we suppress it.
-with warnings.catch_warnings():
-    warnings.filterwarnings('ignore', message='Your version of the IERS Bulletin A')
-    from astroplan import download_IERS_A
-    # And pep8 checks complain about an import after the top of the file, but not when
-    # in this block. Weird.
-    from astropy.utils import data
+from astroplan import download_IERS_A
+from astropy.utils import data
 
-DEFAULT_DATA_FOLDER = "{}/astrometry/data".format(os.getenv('PANDIR'))
+from .logger import logger
 
 
 class Downloader:
@@ -35,40 +20,61 @@ class Downloader:
     which stars are in an arbitrary image of the night sky.
     """
 
-    def __init__(self, data_folder=None, wide_field=True, narrow_field=False, keep_going=True):
+    def __init__(self,
+                 data_folder=None,
+                 wide_field=True,
+                 narrow_field=False,
+                 keep_going=True,
+                 verbose=False):
         """
         Args:
             data_folder: Path to directory into which to copy the astrometry.net indices.
             wide_field: If True, downloads wide field astrometry.net indices.
             narrow_field: If True, downloads narrow field astrometry.net indices.
             keep_going: If False, exceptions are not suppressed. If True, returns False if there
                 are any download failures, else returns True.
+            verbose (bool, optional): If console output, default False.
         """
-        self.data_folder = data_folder or DEFAULT_DATA_FOLDER
+        self.data_folder = data_folder
         self.wide_field = wide_field
         self.narrow_field = narrow_field
         self.keep_going = keep_going
+        self.verbose = verbose
 
     def download_all_files(self):
         """Downloads the files according to the attributes of this object."""
         result = True
-        try:
-            download_IERS_A()
-        except Exception as e:
-            if not self.keep_going:
-                raise e
-            logger.warning(f'Failed to download IERS A bulletin: {e}')
-            result = False
+
+        result = self.download_iers()
+
         if self.wide_field:
             for i in range(4110, 4119):
-                if not self.download_one_file('4100/index-{}.fits'.format(i)):
+                if not self.download_one_file(f'4100/index-{i}.fits'):
                     result = False
+
         if self.narrow_field:
             for i in range(4210, 4219):
-                if not self.download_one_file('4200/index-{}.fits'.format(i)):
+                if not self.download_one_file(f'4200/index-{i}.fits'):
                     result = False
         return result
 
+    def download_iers(self):
+        """Downloads the earth rotation catalog needed for accurate coordinate positions.
+
+        Note: This download gives us a host of problems. This function should be called
+        less than every 14 days otherwise a warning will be given. Currently we
+        are downloading of our own google-based mirror of the data.
+
+        Returns:
+            TYPE: Description
+        """
+        try:
+            download_IERS_A(show_progress=self.verbose)
+            return True
+        except Exception as e:
+            logger.warning(f'Failed to download IERS A bulletin: {e}')
+            return False
+
     def download_one_file(self, fn):
         """Downloads one astrometry.net file into self.data_folder."""
         dest = "{}/{}".format(self.data_folder, os.path.basename(fn))
@@ -99,60 +105,3 @@ def create_data_folder(self):
             logger.info("Creating data folder: {}.".format(self.data_folder))
             os.makedirs(self.data_folder)
 
-
-def main():
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument(
-        '--folder',
-        help='Destination folder for astrometry indices. Default: {}'.format(DEFAULT_DATA_FOLDER),
-        default=DEFAULT_DATA_FOLDER)
-
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--keep-going',
-        action='store_true',
-        help='Ignore download failures and keep going to the other downloads (default)')
-    group.add_argument(
-        '--no-keep-going', action='store_true', help='Fail immediately if any download fails')
-
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--narrow-field', action='store_true', help='Do download narrow field indices')
-    group.add_argument(
-        '--no-narrow-field',
-        action='store_true',
-        help='Skip downloading narrow field indices (default)')
-
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--wide-field', action='store_true', help='Do download wide field indices (default)')
-    group.add_argument(
-        '--no-wide-field', action='store_true', help='Skip downloading wide field indices')
-
-    args = parser.parse_args()
-
-    if args.folder and not os.path.exists(args.folder):
-        logger.info("Warning, data folder {} does not exist.".format(args.folder))
-
-    keep_going = args.keep_going or not args.no_keep_going
-
-    # --no_narrow_field is the default, so the the args list below ignores args.no_narrow_field.
-    dl = Downloader(
-        data_folder=args.folder,
-        keep_going=keep_going,
-        narrow_field=args.narrow_field,
-        wide_field=args.wide_field or not args.no_wide_field)
-    success = dl.download_all_files()
-
-    # Docker builds are failing if one of the files is missing, which shouldn't
-    # be the case. This will all need to be reworked as part of our IERS updates.
-    if success is False and keep_going is True:
-        success = True
-
-    return success
-
-
-if __name__ == '__main__':
-    if not main():
-        sys.exit(1)

panoptes/utils/tests/serial_handlers/protocol_arduinosimulator.py

@@ -514,7 +514,7 @@ def _create_simulator(self, params):
         self.relay_queue = queue.Queue(maxsize=params.get('write_buffer_size', 100))
 
         self.device_simulator = ArduinoSimulator(message, self.relay_queue, self.json_queue,
-                                                 chunk_size, self.stop, self.logger)
+                                                 chunk_size, self.stop)
 
 
 Serial = FakeArduinoSerialHandler

scripts/download-data.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import os
+import click
+
+from panoptes.utils.data import Downloader
+from panoptes.utils.logger import logger
+
+DEFAULT_DATA_FOLDER = os.path.expandvars("$PANDIR/astrometry/data")
+
+
+@click.command()
+@click.option('--folder',
+              default=DEFAULT_DATA_FOLDER,
+              help=f'Destination folder for astrometry indices. Default: {DEFAULT_DATA_FOLDER}')
+@click.option('--keep-going/--no-keep-going',
+              default=True,
+              help='Ignore download failures and keep going to other downloads, default True.')
+@click.option('--narrow-field/--no-narrow-field',
+              default=False,
+              help='Download narrow field indices, default False.')
+@click.option('--wide-field/--no-wide-field',
+              default=False,
+              help='Download wide field indices, default False.')
+@click.option('--verbose', is_flag=True, default=False, help='Log output to console.')
+def main(folder=DEFAULT_DATA_FOLDER,
+         keep_going=True,
+         narrow_field=False,
+         wide_field=False,
+         verbose=False):
+
+    if verbose:
+        logger.enable('panoptes')
+    else:
+        logger.remove(0)
+
+    if not os.path.exists(folder):
+        logger.warning(f"Warning, data folder {folder} does not exist, will create.")
+
+    # --no_narrow_field is the default, so the the args list below ignores args.no_narrow_field.
+    dl = Downloader(
+        data_folder=folder,
+        keep_going=keep_going,
+        narrow_field=narrow_field,
+        wide_field=wide_field,
+        verbose=verbose)
+
+    success = dl.download_all_files()
+    if success:
+        logger.success(f'Downloaded all files')
+
+    return success
+
+
+if __name__ == '__main__':
+    main()