From 8bc10185b49dd6c238e0ec69239132e47804ec99 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 9 Jun 2020 15:36:34 -1000 Subject: [PATCH 1/6] Update config-server.rst Fixing output for example on setting config server item. --- docs/config-server.rst | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/config-server.rst b/docs/config-server.rst index 8686d889a..4423b167c 100644 --- a/docs/config-server.rst +++ b/docs/config-server.rst @@ -193,17 +193,6 @@ The ``panoptes-config-server set`` command will set the value for the given key. .. code-block:: bash $ panoptes-config-server set 'location.horizon' '37 deg' - { - "elevation": 3400, - "flat_horizon": -6, - "focus_horizon": -12, - "gmt_offset": -600, - "horizon": "37 deg", - "latitude": 19.54, - "longitude": -155.58, - "name": "New Location", - "observe_horizon": -18, - "timezone": "US/Hawaii" - } + {'location.horizon': } -See ``panoptes-config-server get --help`` and ``panoptes-config-server set --help`` for more details. \ No newline at end of file +See ``panoptes-config-server get --help`` and ``panoptes-config-server set --help`` for more details. From 572ba3e856ebfdfd40085e020ccbac05f15a9558 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 17 Jun 2020 15:58:01 -1000 Subject: [PATCH 2/6] Small config improvements (#222) * Config client isn't so noisy. * PyYAML to >= 5.3.1 (security vulnerability). * Better parsing of base dir and friends. --- CHANGELOG.rst | 3 ++ setup.cfg | 2 +- src/panoptes/utils/config/client.py | 8 ++-- src/panoptes/utils/config/helpers.py | 64 ++++++++++++++++++++++------ 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a5247bfb9..6eaa14fb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,7 @@ Changed * Running pytest locally will generate coverage report in terminal. (#218) * Lots of documentation. (#218) * Removing the environment section from the readme. (#218) +* Bump PyYaml to latest for security warning. (#222) * Config Server (#217) * Better logging. @@ -24,6 +25,8 @@ Changed * Config items no longer assume any defaults for either directories or files. A config file name is always required and it should always be an absolute path. (#218) * Adding test file for config items. (#218) * ``panoptes-config-server`` re-worked and now includes ``run``, ``get``, and ``set`` subcomamnds. (#221) + * Better parsing of directories entry in config server. (#222) + * Make config server less noisy. (#222) * Testing (#218) diff --git a/setup.cfg b/setup.cfg index 6c40b418b..2470514b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,7 +57,7 @@ install_requires = photutils Pillow pyserial - PyYAML + PyYAML>=5.3.1 requests ruamel.yaml scalpl diff --git a/src/panoptes/utils/config/client.py b/src/panoptes/utils/config/client.py index 653d3ddab..c4a68f975 100644 --- a/src/panoptes/utils/config/client.py +++ b/src/panoptes/utils/config/client.py @@ -68,18 +68,18 @@ def get_config(key=None, host='localhost', port='6563', parse=True, default=None logger.warning(f'Problem with get_config: {e!r}') else: if response.text != 'null\n': - logger.debug(f'Received config {response.text}') + logger.trace(f'Received config {key=} {response.text=}') if parse: - logger.debug(f'Parsing config results') + logger.trace(f'Parsing config results') config_entry = from_json(response.content.decode('utf8')) else: config_entry = response.json() if config_entry is None: - logger.debug(f'No config entry found, returning {default=}') + logger.trace(f'No config entry found, returning {default=}') config_entry = default - logger.info(f'Config {key=}: {config_entry=}') + logger.trace(f'Config {key=}: {config_entry=}') return config_entry diff --git a/src/panoptes/utils/config/helpers.py b/src/panoptes/utils/config/helpers.py index c1d211ad0..3254e7dba 100644 --- a/src/panoptes/utils/config/helpers.py +++ b/src/panoptes/utils/config/helpers.py @@ -76,17 +76,18 @@ def load_config(config_files=None, parse=True, ignore_local=False): try: _add_to_conf(config, local_version, parse=parse) except Exception as e: # pragma: no cover - logger.warning(f"Problem with local config file {local_version}, skipping: {e!r}") + logger.warning( + f"Problem with local config file {local_version}, skipping: {e!r}") - # parse_config currently only corrects directory names. + # parse_config_directories currently only corrects directory names. if parse: logger.trace(f'Parsing {config=}') try: - config = parse_config(config) + config['directories'] = parse_config_directories(config['directories']) except Exception as e: logger.warning(f'Unable to parse config: {e=}') else: - logger.trace(f'Config parsed: {config=}') + logger.trace(f'Config directories parsed: {config=}') return config @@ -132,23 +133,60 @@ def save_config(path, config, overwrite=True): return True -def parse_config(config): +def parse_config_directories(directories, must_exist=False): """Parse the config dictionary for common objects. - Currently only parses the following: - * `directories` for relative path names. + Given a `base` entry that corresponds to the absolute path of a directory, + prepend the `base` to all other relative directory entries. + + If `must_exist=True`, then only update entry if the corresponding + directory exists on the filesystem. + + .. doctest:: + + >>> dirs_config = dict(base='/var/panoptes', foo='bar', baz='bam') + >>> # If the relative dir doesn't exist but is required, return as is. + >>> parse_config_directories(dirs_config, must_exist=True) + {'base': '/var/panoptes', 'foo': 'bar', 'baz': 'bam'} + + >>> # Default is to return anyway. + >>> parse_config_directories(dirs_config) + {'base': '/var/panoptes', 'foo': '/var/panoptes/bar', 'baz': '/var/panoptes/bam'} + + >>> # If 'base' is not a valid absolute directory, return all as is. + >>> dirs_config = dict(base='panoptes', foo='bar', baz='bam') + >>> parse_config_directories(dirs_config, must_exist=False) + {'base': 'panoptes', 'foo': 'bar', 'baz': 'bam'} Args: - config (dict): Config items. + directories (dict): The dictionary of directory information. Usually comes + from the "directories" entry in the config. + must_exist (bool): Only parse directory if it exists on the filesystem, + default False. Returns: - dict: Config items but with objects. + dict: The same directory but with relative directories resolved. """ - with suppress(KeyError): - for dir_name, rel_dir in config['directories'].items(): - config['directories'][dir_name] = os.path.expandvars(f'$PANDIR/{rel_dir}') - return config + # Try to get the base directory first. + base_dir = directories.get('base', os.environ['PANDIR']) + if os.path.isdir(base_dir): + logger.trace(f'Using {base_dir=} for setting config directories') + + # Add the base directory to any relative dir. + for dir_name, rel_dir in directories.items(): + # Only want relative directories. + if rel_dir.startswith('/') is False: + abs_dir = os.path.join(base_dir, rel_dir) + logger.trace(f'{base_dir=} {rel_dir=} {abs_dir=} {must_exist=}') + + if must_exist and not os.path.exists(abs_dir): + logger.warning(f'{must_exist=} but {abs_dir=} does not exist, skipping') + else: + logger.trace(f'Setting {dir_name} to {abs_dir}') + directories[dir_name] = abs_dir + + return directories def _add_to_conf(config, conf_fn, parse=False): From f8a40d6eb7c7adbd4651d9494338b9f840a36b4b Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 17 Jun 2020 16:15:11 -1000 Subject: [PATCH 3/6] Update CHANGELOG.rst Fixing changelog. --- CHANGELOG.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6eaa14fb4..efb3b5e14 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,19 @@ Changelog ========= +0.2.21dev +--------- + +Changed +^^^^^^^ + +* Bump PyYaml to latest for security warning. (#222) +* Config Server + + * Better parsing of directories entry in config server. (#222) + * Make config server less noisy. (#222) + + 0.2.20 - 2020-06-09 ------------------- @@ -14,7 +27,6 @@ Changed * Running pytest locally will generate coverage report in terminal. (#218) * Lots of documentation. (#218) * Removing the environment section from the readme. (#218) -* Bump PyYaml to latest for security warning. (#222) * Config Server (#217) * Better logging. @@ -25,8 +37,6 @@ Changed * Config items no longer assume any defaults for either directories or files. A config file name is always required and it should always be an absolute path. (#218) * Adding test file for config items. (#218) * ``panoptes-config-server`` re-worked and now includes ``run``, ``get``, and ``set`` subcomamnds. (#221) - * Better parsing of directories entry in config server. (#222) - * Make config server less noisy. (#222) * Testing (#218) From ae764b8ccbb9b4f8f822026c6c2a6f1abb40e37f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 30 Jun 2020 16:34:13 -1000 Subject: [PATCH 4/6] Docker multi arch (#223) The codecov diff appears at odd times. Not sure what the -4% difference is from. Merging anyway. :man_shrugging: --- .dockerignore | 4 +- .gcloudignore | 4 +- CHANGELOG.rst | 21 +++++-- bin/cr2-to-jpg | 6 ++ docker/cloudbuild.yaml | 55 ++++++++++++----- docker/develop.Dockerfile | 9 +-- docker/latest.Dockerfile | 19 ++++-- requirements.txt | 87 --------------------------- resources/sextractor/panoptes.sex | 6 +- scripts/testing/run-tests.sh | 2 +- setup.cfg | 10 +-- src/panoptes/utils/data/metadata.py | 14 +++-- src/panoptes/utils/images/__init__.py | 10 +-- src/panoptes/utils/images/cr2.py | 4 +- src/panoptes/utils/serializers.py | 19 +++--- src/panoptes/utils/stars.py | 2 +- 16 files changed, 119 insertions(+), 153 deletions(-) delete mode 100644 requirements.txt diff --git a/.dockerignore b/.dockerignore index d04f9f888..759af806c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,9 @@ -docs/* .idea .venv venv -.git +# Need to pass git to allow version :( +!.git .github *.md diff --git a/.gcloudignore b/.gcloudignore index 433085fb9..b26d211e7 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -1,9 +1,9 @@ -docs/* .idea .venv venv -.git +# Need to pass git to allow version :( +!.git .github *.md diff --git a/CHANGELOG.rst b/CHANGELOG.rst index efb3b5e14..5150c4d19 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,14 +5,27 @@ Changelog 0.2.21dev --------- +Added +^^^^^ + +* Added `arm64` build for Docker based off `ubuntu` image. (#223) + Changed ^^^^^^^ -* Bump PyYaml to latest for security warning. (#222) -* Config Server +* Docker - * Better parsing of directories entry in config server. (#222) - * Make config server less noisy. (#222) + * Changed base image to `ubuntu`. (#223) + * `amd64` and `arm64` images built by default. (#223) + * Ubuntu has changed `sextractor` to `source-extractor` (yay). (#223) + +* Config Server + + * Better parsing of directories entry in config server. (#222) + * Make config server less noisy. (#222) + +* Bump PyYaml to latest for security warning. (#222) +* Remove pendulum because too hard to build on `arm processors `_. (#223) 0.2.20 - 2020-06-09 diff --git a/bin/cr2-to-jpg b/bin/cr2-to-jpg index a3946813c..82c1cbbd4 100755 --- a/bin/cr2-to-jpg +++ b/bin/cr2-to-jpg @@ -55,6 +55,12 @@ else fi fi +# Test for file +if [[ ! -s "${JPG}" ]]; then + echo "JPG was not extracted successfully." + exit 1 +fi + if [[ -n "$TITLE" ]] then echo "Adding title \"${TITLE}\"" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index b35e012d5..04d987420 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,18 +1,45 @@ steps: -- name: 'docker' - id: 'amd64-build' - args: - - 'build' - - '-f=docker/${_TAG}.Dockerfile' - - '--tag=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}' - - '.' + # Set up multiarch support + - name: 'gcr.io/cloud-builders/docker' + id: 'setup-buildx' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'run' + - '--privileged' + - '--rm' + - 'docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64' + waitFor: ['-'] -- name: 'docker' - id: 'amd64-push' - args: - - 'push' - - 'gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}' - waitFor: ['amd64-build'] + # Build builder + - name: 'gcr.io/cloud-builders/docker' + id: 'build-builder' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'buildx' + - 'create' + - '--use' + - '--driver=docker-container' + waitFor: ['setup-buildx'] + + # Build + - name: 'gcr.io/cloud-builders/docker' + id: 'build-images' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'buildx' + - 'build' + - '--push' + - '--platform=linux/amd64,linux/arm64' + - '-f=docker/${_TAG}.Dockerfile' + - '--tag=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}' + - '--cache-from=gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}' + - '.' + waitFor: ['build-builder'] images: - - 'gcr.io/${PROJECT_ID}/panoptes-utils:${_TAG}' + - 'gcr.io/${PROJECT_ID}/panoptes-utils:latest' + - 'gcr.io/${PROJECT_ID}/panoptes-utils:amd64' + - 'gcr.io/${PROJECT_ID}/panoptes-utils:arm64' diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile index 3333740b2..aef1365b8 100644 --- a/docker/develop.Dockerfile +++ b/docker/develop.Dockerfile @@ -14,9 +14,6 @@ ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV SHELL /bin/zsh -ARG userid=1000 - -ENV USERID $userid ENV PANDIR $pan_dir ENV PANLOG "$pan_dir/logs" ENV PANUSER panoptes @@ -24,14 +21,12 @@ ENV POCS $pocs_dir ENV SOLVE_FIELD /usr/bin/solve-field # Install module -USER ${PANUSER} -COPY --chown=panoptes:panoptes . "${PANDIR}/panoptes-utils/" +COPY . "${PANDIR}/panoptes-utils/" RUN wget -qO- $cr2_url > "${PANDIR}/panoptes-utils/tests/data/canon.cr2" && \ cd "${PANDIR}/panoptes-utils" && \ - pip install -e ".[testing]" + pip3 install -e ".[testing,google]" # Cleanup apt. -USER root RUN apt-get autoremove --purge -y && \ apt-get -y clean && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile index 3b7c6bd9a..336c500e9 100644 --- a/docker/latest.Dockerfile +++ b/docker/latest.Dockerfile @@ -1,4 +1,4 @@ -ARG IMAGE_URL=python:3.8-slim-buster +ARG IMAGE_URL=ubuntu FROM ${IMAGE_URL} AS base-image LABEL description="Installs the panoptes-utils module from pip. \ @@ -32,7 +32,9 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ gosu wget curl bzip2 ca-certificates zsh openssh-client nano \ astrometry.net sextractor dcraw exiftool libcfitsio-dev libcfitsio-bin imagemagick \ - libfreetype6-dev libpng-dev fonts-lato libsnappy-dev \ + libfreetype6-dev libpng-dev fonts-lato libsnappy-dev libjpeg-dev \ + python3-pip python3-scipy python3-dev python3-pandas python3-matplotlib \ + libffi-dev libssl-dev \ gcc git pkg-config sudo && \ # Oh My ZSH. :) mkdir -p "${ZSH_CUSTOM}" && \ @@ -57,18 +59,25 @@ RUN apt-get update && \ # Update permissions for current user. chown -R ${PANUSER}:${PANUSER} "/home/${panuser}" && \ chown -R ${PANUSER}:${PANUSER} ${PANDIR} && \ - # Install module - pip install "panoptes-utils[testing]" && \ # astrometry.net folders mkdir -p "${astrometry_dir}" && \ echo "add_path ${astrometry_dir}" >> /etc/astrometry.cfg && \ + # Preinstall some modules. + pip3 install astropy astroplan click loguru && \ # astrometry.net index files - python /tmp/download-data.py \ + python3 /tmp/download-data.py \ --wide-field --narrow-field \ --folder "${astrometry_dir}" \ --verbose && \ chown -R ${PANUSER}:${PANUSER} ${astrometry_dir} && \ chmod -R 777 ${astrometry_dir} && \ + # Allow sudo without password + echo "$PANUSER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# Install module +COPY . "${PANDIR}/panoptes-utils" +RUN cd "${PANDIR}/panoptes-utils" && \ + pip3 install ".[testing,google]" && \ # Cleanup apt-get autoremove --purge -y \ autoconf \ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f0c4bcdff..000000000 --- a/requirements.txt +++ /dev/null @@ -1,87 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile -# -astroplan==0.6 # via panoptes-utils (setup.py) -astropy==4.0 # via astroplan, panoptes-utils (setup.py), photutils -attrs==19.3.0 # via pytest -bokeh==2.0.2 # via hvplot, panel -cachetools==4.1.0 # via google-auth -certifi==2019.11.28 # via requests -chardet==3.0.4 # via requests -click==7.0 # via flask -colorcet==2.0.2 # via hvplot -coverage==5.0.3 # via panoptes-utils (setup.py), pytest-cov -cycler==0.10.0 # via matplotlib -decorator==4.4.1 # via mocket -flask==1.1.1 # via panoptes-utils (setup.py) -google-api-core==1.16.0 # via google-cloud-bigquery, google-cloud-core -google-auth==1.13.1 # via google-api-core, google-cloud-bigquery, google-cloud-storage -google-cloud-bigquery[pandas,pyarrow]==1.24.0 # via panoptes-utils (setup.py) -google-cloud-core==1.3.0 # via google-cloud-bigquery, google-cloud-storage -google-cloud-storage==1.28.0 # via panoptes-utils (setup.py) -google-resumable-media==0.5.0 # via google-cloud-bigquery, google-cloud-storage -googleapis-common-protos==1.51.0 # via google-api-core -holoviews==1.13.2 # via hvplot, panoptes-utils (setup.py) -hvplot==0.5.2 # via panoptes-utils (setup.py) -idna==2.9 # via requests -itsdangerous==1.1.0 # via flask -jinja2==2.11.1 # via bokeh, flask -kiwisolver==1.1.0 # via matplotlib -loguru==0.4.1 # via panoptes-utils (setup.py) -markdown==3.2.1 # via panel -markupsafe==1.1.1 # via jinja2 -matplotlib==3.1.3 # via panoptes-utils (setup.py) -mocket==3.8.4 # via panoptes-utils (setup.py) -more-itertools==8.2.0 # via pytest -numpy==1.18.1 # via astroplan, astropy, bokeh, holoviews, matplotlib, pandas, panoptes-utils (setup.py), photutils, pyarrow, scipy -oauthlib==3.1.0 # via requests-oauthlib -packaging==20.3 # via bokeh, pytest -pandas==1.0.3 # via google-cloud-bigquery, holoviews, hvplot, panoptes-utils (setup.py) -panel==0.9.5 # via holoviews -param==1.9.3 # via colorcet, holoviews, panel, pyct, pyviz-comms -pendulum==2.1.0 # via panoptes-utils (setup.py) -photutils==0.7.2 # via panoptes-utils (setup.py) -pillow==7.0.0 # via bokeh, panoptes-utils (setup.py) -pluggy==0.13.1 # via pytest -protobuf==3.11.3 # via google-api-core, google-cloud-bigquery, googleapis-common-protos -py==1.8.1 # via pytest -pyarrow==0.17.0 # via google-cloud-bigquery -pyasn1-modules==0.2.8 # via google-auth -pyasn1==0.4.8 # via pyasn1-modules, rsa -pycodestyle==2.6.0 # via panoptes-utils (setup.py) -pyct==0.4.6 # via colorcet, panel -pyparsing==2.4.6 # via matplotlib, packaging -pyserial==3.4 # via panoptes-utils (setup.py) -pysocks==1.7.1 # via tweepy -pytest-cov==2.8.1 # via panoptes-utils (setup.py) -pytest-remotedata==0.3.2 # via panoptes-utils (setup.py) -pytest==5.3.5 # via panoptes-utils (setup.py), pytest-cov, pytest-remotedata -python-dateutil==2.8.1 # via bokeh, matplotlib, pandas, panoptes-utils (setup.py), pendulum -python-magic==0.4.15 # via mocket -pytz==2019.3 # via astroplan, google-api-core, pandas -pytzdata==2019.3 # via pendulum -pyviz-comms==0.7.4 # via holoviews, panel -pyyaml==5.3 # via bokeh, panoptes-utils (setup.py) -pyzmq==18.1.1 # via panoptes-utils (setup.py) -requests-oauthlib==1.3.0 # via tweepy -requests==2.23.0 # via google-api-core, panoptes-utils (setup.py), requests-oauthlib, tweepy -rsa==4.0 # via google-auth -ruamel.yaml.clib==0.2.0 # via ruamel.yaml -ruamel.yaml==0.16.10 # via panoptes-utils (setup.py) -scalpl==0.3.0 # via panoptes-utils (setup.py) -scipy==1.4.1 # via panoptes-utils (setup.py) -six==1.14.0 # via astroplan, cycler, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, mocket, packaging, protobuf, pytest-remotedata, python-dateutil, tweepy -tornado==6.0.4 # via bokeh -tqdm==4.45.0 # via panel -tweepy==3.8.0 # via panoptes-utils (setup.py) -typing-extensions==3.7.4.2 # via bokeh -urllib3==1.25.8 # via mocket, requests -versioneer==0.18 # via panoptes-utils (setup.py) -wcwidth==0.1.8 # via pytest -werkzeug==1.0.0 # via flask - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/resources/sextractor/panoptes.sex b/resources/sextractor/panoptes.sex index 76d358095..f77ab663b 100644 --- a/resources/sextractor/panoptes.sex +++ b/resources/sextractor/panoptes.sex @@ -18,7 +18,7 @@ DETECT_THRESH 3.0 # or , in mag.arcsec-2 ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME /usr/share/sextractor/default.conv # name of the file containing the filter +FILTER_NAME /usr/share/source-extractor/default.conv # name of the file containing the filter DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds DEBLEND_MINCONT 0.005 # Minimum contrast parameter for deblending @@ -59,7 +59,7 @@ PIXEL_SCALE 10.3 # size of pixel in arcsec (0=use FITS WCS info) #------------------------- Star/Galaxy Separation ---------------------------- SEEING_FWHM 1.5 # stellar FWHM in arcsec -STARNNW_NAME /usr/share/sextractor/default.nnw # Neural-Network_Weight table filename +STARNNW_NAME /usr/share/source-extractor/default.nnw # Neural-Network_Weight table filename #------------------------------ Background ----------------------------------- @@ -99,6 +99,6 @@ VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL HEADER_SUFFIX .head # Filename extension for additional headers WRITE_XML N # Write XML file (Y/N)? XML_NAME sex.xml # Filename for XML output -XSL_URL file:///usr/local/share/sextractor/sextractor.xsl +XSL_URL file:///usr/share/source-extractor/sextractor.xsl # Filename for XSL style-sheet diff --git a/scripts/testing/run-tests.sh b/scripts/testing/run-tests.sh index 91ca656f8..ae42b5f36 100755 --- a/scripts/testing/run-tests.sh +++ b/scripts/testing/run-tests.sh @@ -9,7 +9,7 @@ coverage erase # Run coverage over the pytest suite. echo "Starting tests" -coverage run "$(command -v pytest)" +coverage run "$(command -v pytest-3)" echo "Combining coverage" coverage combine diff --git a/setup.cfg b/setup.cfg index 2470514b3..dfb0f3e2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,17 +45,15 @@ install_requires = astropy click Flask - google-cloud-bigquery[pandas, pyarrow] - google-cloud-storage holoviews hvplot loguru matplotlib numpy - pandas!=1.0.4 - pendulum + pandas>1.0.0,!=1.0.4 photutils - Pillow + pillow<7 + python-dateutil pyserial PyYAML>=5.3.1 requests @@ -86,6 +84,8 @@ testing = mocket coverage pytest-remotedata>=0.3.1' +google = + google-cloud-bigquery[pandas, pyarrow] [options.entry_points] # Add here console scripts like: diff --git a/src/panoptes/utils/data/metadata.py b/src/panoptes/utils/data/metadata.py index 96545f791..c9b698c47 100644 --- a/src/panoptes/utils/data/metadata.py +++ b/src/panoptes/utils/data/metadata.py @@ -7,10 +7,11 @@ import pandas as pd import hvplot.pandas # noqa -import pendulum +from dateutil.parser import parse as date_parse from tqdm import tqdm from .. import listify +from ..time import current_time from ..logging import logger OBS_BASE_URL = 'https://storage.googleapis.com/panoptes-observations' @@ -172,12 +173,12 @@ def search_observations( start_date = '2018-01-01' if end_date is None: - end_date = pendulum.now() + end_date = current_time() with suppress(TypeError): - start_date = pendulum.parse(start_date).replace(tzinfo=None) + start_date = date_parse(start_date).replace(tzinfo=None) with suppress(TypeError): - end_date = pendulum.parse(end_date).replace(tzinfo=None) + end_date = date_parse(end_date).replace(tzinfo=None) ra_max = (coords.ra + (radius * u.degree)).value ra_min = (coords.ra - (radius * u.degree)).value @@ -189,7 +190,10 @@ def search_observations( # Get the observation list obs_df = source if obs_df is None: - local_path = download_file(source_url, cache='update', show_progress=False, pkgname='panoptes') + local_path = download_file(source_url, + cache='update', + show_progress=False, + pkgname='panoptes') obs_df = pd.read_csv(local_path).convert_dtypes() logger.debug(f'Found {len(obs_df)} total observations') diff --git a/src/panoptes/utils/images/__init__.py b/src/panoptes/utils/images/__init__.py index 3db6afabc..3150b8348 100644 --- a/src/panoptes/utils/images/__init__.py +++ b/src/panoptes/utils/images/__init__.py @@ -5,8 +5,7 @@ from contextlib import suppress from warnings import warn -import pendulum -import pendulum.exceptions +from dateutil.parser import parse as date_parse import numpy as np from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas from matplotlib.figure import Figure @@ -42,7 +41,8 @@ def crop_data(data, box_width=200, center=None, data_only=True, wcs=None, **kwar a `astropy.nddata.Cutout2D` object. """ - assert data.shape[0] >= box_width, f"Can't clip data, it's smaller than {box_width} ({data.shape})" + assert data.shape[ + 0] >= box_width, f"Can't clip data, it's smaller than {box_width} ({data.shape})" # Get the center if center is None: x_len, y_len = data.shape @@ -145,8 +145,8 @@ def _make_pretty_from_fits(fname=None, # If we don't have DATE-OBS, check filename for date try: basename = os.path.splitext(os.path.basename(fname))[0] - date_time = pendulum.parse(basename).isoformat() - except pendulum.exceptions.ParserError: # pragma: no cover + date_time = date_parse(basename).isoformat() + except Exception: # pragma: no cover # Otherwise use now date_time = current_time(pretty=True) diff --git a/src/panoptes/utils/images/cr2.py b/src/panoptes/utils/images/cr2.py index 6d797b46d..e232fb707 100644 --- a/src/panoptes/utils/images/cr2.py +++ b/src/panoptes/utils/images/cr2.py @@ -3,7 +3,7 @@ import shutil import numpy as np -import pendulum +from dateutil.parser import parse as date_parse from json import loads from warnings import warn from astropy.io import fits @@ -58,7 +58,7 @@ def cr2_to_fits( # Set the PGM as the primary data for the FITS file hdu = fits.PrimaryHDU(pgm) - obs_date = pendulum.parse(exif.get('DateTimeOriginal', '').replace(':', '-', 2)).isoformat() + obs_date = date_parse(exif.get('DateTimeOriginal', '').replace(':', '-', 2)).isoformat() # Set some default headers hdu.header.set('FILTER', 'RGGB') diff --git a/src/panoptes/utils/serializers.py b/src/panoptes/utils/serializers.py index ad138b0bb..71bbddbbd 100644 --- a/src/panoptes/utils/serializers.py +++ b/src/panoptes/utils/serializers.py @@ -6,8 +6,7 @@ from ruamel.yaml.compat import StringIO import numpy as np -import pendulum -from pendulum.parsing.exceptions import ParserError +from dateutil.parser import parse as date_parse from astropy.time import Time from astropy import units as u @@ -278,7 +277,7 @@ def deserialize_all_objects(obj): A boolean. - A `datetime.datetime` object as parsed by `pendulum.parse`. + A `datetime.datetime` object as parsed by `dateutil.parser.parse`. If a string ending with any of ``['m', 'deg', 's']``, an ``astropy.unit.Quantity`` @@ -303,10 +302,6 @@ def deserialize_all_objects(obj): if isinstance(obj, bool): return bool(obj) - # Try to turn into a time - with suppress(ParserError, TypeError): - return pendulum.parse(obj) - # Try to parse as quantity if certain type if isinstance(obj, str) and obj > '': with suppress(IndexError): @@ -315,6 +310,10 @@ def deserialize_all_objects(obj): with suppress(Exception): return u.Quantity(obj) + # Try to turn into a time + with suppress(Exception): + return date_parse(obj) + return obj @@ -325,15 +324,15 @@ def serialize_object(obj): individual objects. Also called in a loop by ``serialize_all_objects``. >>> from panoptes.utils.serializers import serialize_object - >>> import pendulum + >>> from dateutil.parser import parse as date_parse >>> from astropy import units as u >>> serialize_object(42 * u.meter) '42.0 m' - >>> party_time = pendulum.parse('1999-12-31 11:59:59') + >>> party_time = date_parse('1999-12-31 11:59:59') >>> type(party_time) - + >>> serialize_object(party_time) '1999-12-31T11:59:59.000' diff --git a/src/panoptes/utils/stars.py b/src/panoptes/utils/stars.py index f9564c774..f42f21721 100644 --- a/src/panoptes/utils/stars.py +++ b/src/panoptes/utils/stars.py @@ -387,7 +387,7 @@ def _lookup_via_sextractor(fits_file, if not os.path.exists(source_file) or force_new: logger.debug("No catalog found, building from sextractor") # Build catalog of point sources - sextractor = shutil.which('sextractor') + sextractor = shutil.which('sextractor') or shutil.which('source-extractor') assert sextractor is not None, 'sextractor not found' From 21ff97b69854375bf0f711cf74b5dc00a2b0d3ae Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 5 Jul 2020 14:46:35 -1000 Subject: [PATCH 5/6] Custom DB folder. (#225) * Allow the file storage dir to be customized. * Proper passing of db storage dir for testing. * Better params for memory db. --- conftest.py | 9 ++++++--- src/panoptes/utils/database/base.py | 3 ++- src/panoptes/utils/database/file.py | 18 +++++++++++------- src/panoptes/utils/database/memory.py | 2 +- tests/test_database.py | 5 +++-- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/conftest.py b/conftest.py index 791204e6a..a2e4675ea 100644 --- a/conftest.py +++ b/conftest.py @@ -129,14 +129,17 @@ def db_type(request): if request.param not in db_list and 'all' not in db_list: # pragma: no cover pytest.skip(f"Skipping {request.param} DB, set --test-all-databases=True") - PanDB.permanently_erase_database( - request.param, 'panoptes_testing', really='Yes', dangerous='Totally') + PanDB.permanently_erase_database(request.param, + 'panoptes_testing', + storage_dir='testing', + really='Yes', + dangerous='Totally') return request.param @pytest.fixture(scope='function') def db(db_type): - return PanDB(db_type=db_type, db_name='panoptes_testing', connect=True) + return PanDB(db_type=db_type, db_name='panoptes_testing', storage_dir='testing', connect=True) @pytest.fixture(scope='function') diff --git a/src/panoptes/utils/database/base.py b/src/panoptes/utils/database/base.py index 2b7b90578..18a1b5448 100644 --- a/src/panoptes/utils/database/base.py +++ b/src/panoptes/utils/database/base.py @@ -172,6 +172,7 @@ def __new__(cls, db_type='memory', db_name=None, *args, **kwargs): def permanently_erase_database(cls, db_type, db_name, + storage_dir=None, really=False, dangerous=False, *args, **kwargs): @@ -184,4 +185,4 @@ def permanently_erase_database(cls, raise Exception('PanDB.permanently_erase_database called with invalid args!') # Load the correct DB module and do the deletion. - get_db_class(db_type).permanently_erase_database(db_name, *args, **kwargs) + get_db_class(db_type).permanently_erase_database(db_name, storage_dir=storage_dir, *args, **kwargs) diff --git a/src/panoptes/utils/database/file.py b/src/panoptes/utils/database/file.py index 382916c94..3bcf59e47 100644 --- a/src/panoptes/utils/database/file.py +++ b/src/panoptes/utils/database/file.py @@ -14,13 +14,17 @@ class PanFileDB(AbstractPanDB): """Stores collections as files of JSON records.""" - def __init__(self, db_name='panoptes', **kwargs): - """Flat file storage for json records + def __init__(self, db_name='panoptes', storage_dir='json_store', **kwargs): + """Flat file storage for json records. This will simply store each json record inside a file corresponding to the type. Each entry will be stored in a single line. + Args: db_name (str, optional): Name of the database containing the collections. + storage_dir (str, optional): The name of the directory in $PANDIR where + the database files will be stored. Default is `json_store` for backwards + compatibility. """ super().__init__(db_name=db_name, **kwargs) @@ -28,8 +32,8 @@ def __init__(self, db_name='panoptes', **kwargs): self.db_folder = db_name # Set up storage directory. - self._storage_dir = os.path.join(os.environ['PANDIR'], 'json_store', self.db_folder) - os.makedirs(self._storage_dir, exist_ok=True) + self.storage_dir = os.path.join(os.environ['PANDIR'], storage_dir, self.db_folder) + os.makedirs(self.storage_dir, exist_ok=True) def insert_current(self, collection, obj, store_permanently=True): obj_id = self._make_id() @@ -98,14 +102,14 @@ def _get_file(self, collection, permanent=True): name = f'{collection}.json' else: name = f'current_{collection}.json' - return os.path.join(self._storage_dir, name) + return os.path.join(self.storage_dir, name) def _make_id(self): return str(uuid4()) @classmethod - def permanently_erase_database(cls, db_name): + def permanently_erase_database(cls, db_name, storage_dir='json_store'): # Clear out any .json files. - storage_dir = os.path.join(os.environ['PANDIR'], 'json_store', db_name) + storage_dir = os.path.join(os.environ['PANDIR'], storage_dir, db_name) for f in glob(os.path.join(storage_dir, '*.json')): os.remove(f) diff --git a/src/panoptes/utils/database/memory.py b/src/panoptes/utils/database/memory.py index 0682b651a..d23c0d39e 100644 --- a/src/panoptes/utils/database/memory.py +++ b/src/panoptes/utils/database/memory.py @@ -88,7 +88,7 @@ def clear_current(self, entry_type): del self.current[entry_type] @classmethod - def permanently_erase_database(cls, db_name): + def permanently_erase_database(cls, *args, **kwargs): # For some reason we're not seeing all the references disappear # after tests. Perhaps there is some global variable pointing at # the db or one of its referrers, or perhaps a pytest fixture diff --git a/tests/test_database.py b/tests/test_database.py index 89cf3e527..7a08d5b6c 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -6,7 +6,7 @@ def test_bad_db(): with pytest.raises(Exception): - PanDB('foobar') + PanDB('foobar', storage_dir='') def test_insert_and_no_permanent(db): @@ -74,5 +74,6 @@ def test_delete_file_db(): with pytest.raises(ValueError): PanDB.permanently_erase_database('memory', 'do_not_delete_me', really='Nope', dangerous='Again, we hope not') - PanDB.permanently_erase_database('file', 'panoptes_testing', really='Yes', dangerous='Totally') + PanDB.permanently_erase_database('file', 'panoptes_testing', storage_dir='testing', really='Yes', + dangerous='Totally') PanDB.permanently_erase_database('memory', 'panoptes_testing', dangerous='Totally', really='Yes') From 39c4b83cd3292bc44bd263c0f107fb87b39c9b76 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 5 Jul 2020 14:52:54 -1000 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5150c4d19..7a09c3880 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ Changelog ========= -0.2.21dev ---------- +0.2.21 - 2020-07-05 +------------------- Added ^^^^^