diff --git a/CHANGELOG.md b/CHANGELOG.md index daadc9bee..a22c4a26e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,24 +6,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - [Changelog](#changelog) - - [[0.2.0] - 2020-03-04](#020---2020-03-04) - - [Added](#added) + - [[0.2.1] - 2020-03-04](#021---2020-03-04) - [Bug fixes](#bug-fixes) - [Changed](#changed) + - [[0.2.0] - 2020-03-04](#020---2020-03-04) + - [Added](#added) + - [Bug fixes](#bug-fixes-1) + - [Changed](#changed-1) - [[0.1.0] - 2020-03-04](#010---2020-03-04) - [[0.0.8] - 2019-06-29](#008---2019-06-29) - [Added](#added-1) - - [Bug fixes](#bug-fixes-1) - - [Changed](#changed-1) + - [Bug fixes](#bug-fixes-2) + - [Changed](#changed-2) - [[0.0.7] - 2019-05-26](#007---2019-05-26) - [Added](#added-2) - - [Changed](#changed-2) + - [Changed](#changed-3) - [[0.0.6] - 2019-04-29](#006---2019-04-29) - [Added](#added-3) - - [Changed](#changed-3) + - [Changed](#changed-4) - [[0.0.5] - 2019-04-09](#005---2019-04-09) - [Added](#added-4) - - [Changed](#changed-4) + - [Changed](#changed-5) + +## [0.2.1] - 2020-03-04 + +### Bug fixes + +* Misc bugs introduced as part of last release, including to `download-data.py` script. + +### Changed + +* New script for downloading data, `scripts/download-data.py`. This helped resolve some issues with the relative imports introduced in `v0.2.0` and is cleaner. (#129) ## [0.2.0] - 2020-03-04 diff --git a/conda-requirements-amd64.yaml b/conda-requirements-amd64.yaml index 1e0c7e805..56ed6051f 100644 --- a/conda-requirements-amd64.yaml +++ b/conda-requirements-amd64.yaml @@ -1,4 +1,3 @@ -name: panoptes channels: - astropy - conda-forge @@ -21,7 +20,6 @@ dependencies: - pytest-remotedata>=0.3.1 - pytest - python-dateutil - - python-json-logger - PyYAML - pyzmq - readline diff --git a/docker/Dockerfile.utils b/docker/Dockerfile.utils index 8f718b9cd..d81d1f1e6 100644 --- a/docker/Dockerfile.utils +++ b/docker/Dockerfile.utils @@ -21,8 +21,9 @@ RUN cd && \ # First deal with pip and PyYAML - see https://github.com/pypa/pip/issues/5247 pip install --no-cache-dir --no-deps --ignore-installed pip PyYAML && \ # Install anaconda packages - /opt/conda/bin/conda install --yes -c conda-forge -c astropy \ - --file "${PANDIR}/panoptes-utils/conda-requirements-${arch}.yaml" && \ + /opt/conda/bin/conda env update --name base \ + --file "${PANDIR}/panoptes-utils/conda-requirements-${arch}.yaml" \ + --prune && \ /opt/conda/bin/conda clean --all --yes && \ /opt/conda/bin/conda clean -f --yes && \ # Install requirements @@ -34,9 +35,10 @@ RUN cd && \ chgrp -R panoptes /opt/conda && \ chmod -R 775 /opt/conda && \ # Download astrometry.net files - python panoptes/utils/data.py \ + python scripts/download-data.py \ --wide-field --narrow-field \ - --folder /astrometry/ && \ + --folder /astrometry/ \ + --verbose && \ # Cleanup apt. apt-get autoremove --purge -y gcc pkg-config && \ apt-get autoremove --purge -y && \ diff --git a/panoptes/utils/data.py b/panoptes/utils/data.py index ff5067f2d..57d191798 100644 --- a/panoptes/utils/data.py +++ b/panoptes/utils/data.py @@ -1,28 +1,13 @@ # Downloads IERS Bulletin A (Earth Orientation Parameters, used by astroplan) # and astrometry.net indices. -import argparse import os import shutil -import sys -import warnings -from .logger import logger - -# Use custom location for download -from astropy.utils.iers import conf as iers_conf -iers_conf.iers_auto_url = 'https://storage.googleapis.com/panoptes-resources/iers/ser7.dat' -iers_conf.iers_auto_url_mirror = 'https://storage.googleapis.com/panoptes-resources/iers/ser7.dat' - -# Importing download_IERS_A can emit a scary warnings, so we suppress it. -with warnings.catch_warnings(): - warnings.filterwarnings('ignore', message='Your version of the IERS Bulletin A') - from astroplan import download_IERS_A - # And pep8 checks complain about an import after the top of the file, but not when - # in this block. Weird. - from astropy.utils import data +from astroplan import download_IERS_A +from astropy.utils import data -DEFAULT_DATA_FOLDER = "{}/astrometry/data".format(os.getenv('PANDIR')) +from .logger import logger class Downloader: @@ -35,7 +20,12 @@ class Downloader: which stars are in an arbitrary image of the night sky. """ - def __init__(self, data_folder=None, wide_field=True, narrow_field=False, keep_going=True): + def __init__(self, + data_folder=None, + wide_field=True, + narrow_field=False, + keep_going=True, + verbose=False): """ Args: data_folder: Path to directory into which to copy the astrometry.net indices. @@ -43,32 +33,48 @@ def __init__(self, data_folder=None, wide_field=True, narrow_field=False, keep_g narrow_field: If True, downloads narrow field astrometry.net indices. keep_going: If False, exceptions are not suppressed. If True, returns False if there are any download failures, else returns True. + verbose (bool, optional): If console output, default False. """ - self.data_folder = data_folder or DEFAULT_DATA_FOLDER + self.data_folder = data_folder self.wide_field = wide_field self.narrow_field = narrow_field self.keep_going = keep_going + self.verbose = verbose def download_all_files(self): """Downloads the files according to the attributes of this object.""" result = True - try: - download_IERS_A() - except Exception as e: - if not self.keep_going: - raise e - logger.warning(f'Failed to download IERS A bulletin: {e}') - result = False + + result = self.download_iers() + if self.wide_field: for i in range(4110, 4119): - if not self.download_one_file('4100/index-{}.fits'.format(i)): + if not self.download_one_file(f'4100/index-{i}.fits'): result = False + if self.narrow_field: for i in range(4210, 4219): - if not self.download_one_file('4200/index-{}.fits'.format(i)): + if not self.download_one_file(f'4200/index-{i}.fits'): result = False return result + def download_iers(self): + """Downloads the earth rotation catalog needed for accurate coordinate positions. + + Note: This download gives us a host of problems. This function should be called + less than every 14 days otherwise a warning will be given. Currently we + are downloading of our own google-based mirror of the data. + + Returns: + TYPE: Description + """ + try: + download_IERS_A(show_progress=self.verbose) + return True + except Exception as e: + logger.warning(f'Failed to download IERS A bulletin: {e}') + return False + def download_one_file(self, fn): """Downloads one astrometry.net file into self.data_folder.""" dest = "{}/{}".format(self.data_folder, os.path.basename(fn)) @@ -99,60 +105,3 @@ def create_data_folder(self): logger.info("Creating data folder: {}.".format(self.data_folder)) os.makedirs(self.data_folder) - -def main(): - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--folder', - help='Destination folder for astrometry indices. Default: {}'.format(DEFAULT_DATA_FOLDER), - default=DEFAULT_DATA_FOLDER) - - group = parser.add_mutually_exclusive_group() - group.add_argument( - '--keep-going', - action='store_true', - help='Ignore download failures and keep going to the other downloads (default)') - group.add_argument( - '--no-keep-going', action='store_true', help='Fail immediately if any download fails') - - group = parser.add_mutually_exclusive_group() - group.add_argument( - '--narrow-field', action='store_true', help='Do download narrow field indices') - group.add_argument( - '--no-narrow-field', - action='store_true', - help='Skip downloading narrow field indices (default)') - - group = parser.add_mutually_exclusive_group() - group.add_argument( - '--wide-field', action='store_true', help='Do download wide field indices (default)') - group.add_argument( - '--no-wide-field', action='store_true', help='Skip downloading wide field indices') - - args = parser.parse_args() - - if args.folder and not os.path.exists(args.folder): - logger.info("Warning, data folder {} does not exist.".format(args.folder)) - - keep_going = args.keep_going or not args.no_keep_going - - # --no_narrow_field is the default, so the the args list below ignores args.no_narrow_field. - dl = Downloader( - data_folder=args.folder, - keep_going=keep_going, - narrow_field=args.narrow_field, - wide_field=args.wide_field or not args.no_wide_field) - success = dl.download_all_files() - - # Docker builds are failing if one of the files is missing, which shouldn't - # be the case. This will all need to be reworked as part of our IERS updates. - if success is False and keep_going is True: - success = True - - return success - - -if __name__ == '__main__': - if not main(): - sys.exit(1) diff --git a/panoptes/utils/tests/serial_handlers/protocol_arduinosimulator.py b/panoptes/utils/tests/serial_handlers/protocol_arduinosimulator.py index 740829fef..428edd42d 100644 --- a/panoptes/utils/tests/serial_handlers/protocol_arduinosimulator.py +++ b/panoptes/utils/tests/serial_handlers/protocol_arduinosimulator.py @@ -514,7 +514,7 @@ def _create_simulator(self, params): self.relay_queue = queue.Queue(maxsize=params.get('write_buffer_size', 100)) self.device_simulator = ArduinoSimulator(message, self.relay_queue, self.json_queue, - chunk_size, self.stop, self.logger) + chunk_size, self.stop) Serial = FakeArduinoSerialHandler diff --git a/scripts/download-data.py b/scripts/download-data.py new file mode 100755 index 000000000..d63c949c6 --- /dev/null +++ b/scripts/download-data.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +import os +import click + +from panoptes.utils.data import Downloader +from panoptes.utils.logger import logger + +DEFAULT_DATA_FOLDER = os.path.expandvars("$PANDIR/astrometry/data") + + +@click.command() +@click.option('--folder', + default=DEFAULT_DATA_FOLDER, + help=f'Destination folder for astrometry indices. Default: {DEFAULT_DATA_FOLDER}') +@click.option('--keep-going/--no-keep-going', + default=True, + help='Ignore download failures and keep going to other downloads, default True.') +@click.option('--narrow-field/--no-narrow-field', + default=False, + help='Download narrow field indices, default False.') +@click.option('--wide-field/--no-wide-field', + default=False, + help='Download wide field indices, default False.') +@click.option('--verbose', is_flag=True, default=False, help='Log output to console.') +def main(folder=DEFAULT_DATA_FOLDER, + keep_going=True, + narrow_field=False, + wide_field=False, + verbose=False): + + if verbose: + logger.enable('panoptes') + else: + logger.remove(0) + + if not os.path.exists(folder): + logger.warning(f"Warning, data folder {folder} does not exist, will create.") + + # --no_narrow_field is the default, so the the args list below ignores args.no_narrow_field. + dl = Downloader( + data_folder=folder, + keep_going=keep_going, + narrow_field=narrow_field, + wide_field=wide_field, + verbose=verbose) + + success = dl.download_all_files() + if success: + logger.success(f'Downloaded all files') + + return success + + +if __name__ == '__main__': + main()