diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..001540d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: Run buildimages tests + +on: + push: + paths: ['src/**'] + pull_request: + paths: ['src/**'] + +jobs: + + run-tests: + + runs-on: ubuntu-latest + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Run builder tests + id: test + run: | + python3 src/tests.py diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml new file mode 100644 index 0000000..fb6e26b --- /dev/null +++ b/.github/workflows/merge.yml @@ -0,0 +1,66 @@ +name: Maybe generate Docker images + +on: [ push ] + +env: + REGISTRY: ghcr.io + +jobs: + + build-and-push-images: + + runs-on: ubuntu-latest + + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in + # this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Debug Event Payload + env: + EVENT_PAYLOAD: ${{ toJson(github.event) }} + run: echo "$EVENT_PAYLOAD" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Git branch name + id: git-branch-name + run: echo "${{ github.head_ref || github.ref_name }}" + + - uses: tchupp/actions-detect-directory-changes@v1 + id: directory-changes + with: + included-paths: "" + included-extensions: "" + if-these-paths-change-return-all-included-paths: "" + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Maybe build and push Docker images to GHCR + id: push + env: + DIR_CHANGES: ${{ steps.directory-changes.outputs.changed }} + run: | + echo "$DIR_CHANGES" + export GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64) + export NEEDED="$(python src/neededimages.py "${{ github.repository }}" "$GHCR_TOKEN" "$DIR_CHANGES" "${{ github.head_ref || github.ref_name }}")" + echo "$NEEDED" + if [ ! -z "$NEEDED" ]; then + python src/buildimages.py "${{ github.repository }}" "${{ github.head_ref || github.ref_name }}" --push --images $NEEDED + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c4e220d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Tag dev images as stable + +on: + release: + types: [published] + +env: + REGISTRY: ghcr.io + +jobs: + + build-and-push-images: + + runs-on: ubuntu-latest + + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in + # this job. + permissions: + + contents: read + packages: write + attestations: write + id-token: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag main images with release tags + id: push + run: | + python src/release.py "${{ github.repository }}" "${{ github.event.release.tag_name }}" diff --git a/base_image/Dockerfile b/base_image/Dockerfile new file mode 100644 index 0000000..0509b98 --- /dev/null +++ b/base_image/Dockerfile @@ -0,0 +1,144 @@ +ARG NOTEBOOK_TAG=2024-08-12 +FROM quay.io/jupyter/base-notebook:$NOTEBOOK_TAG as base +FROM quay.io/jupyter/docker-stacks-foundation:$NOTEBOOK_TAG + +ARG IMAGE_TAG=latest +LABEL gov.nasa.smce.fornax.jupyterhub.image="${IMAGE_TAG}" +LABEL org.opencontainers.image.source=https://github.com/fornax-navo/fornax-images +LABEL org.opencontainers.image.ref.name="Fornax Base Image" +LABEL org.opencontainers.image.version=0.2 +LABEL maintainer="Fornax Project" + +# Bring some commands from jupyter/base-notebook +# Skip the install of jupyterhub/lab etc +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT +CMD ["start-notebook.py"] +# Copy files from the image directly +COPY --from=base /usr/local/bin/start-* /usr/local/bin/ +COPY --from=base /etc/jupyter/*py /etc/jupyter/ +USER root +HEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \ + CMD /etc/jupyter/docker_healthcheck.py || exit 1 +RUN fix-permissions /etc/jupyter/ /home/${NB_USER} /opt/ +USER $NB_USER +WORKDIR $HOME +# End of imports from jupyter/base-notebook # +# ----------------------------------------- # + + +ENV CONDA_ENV=notebook + +# Ask dask to read config from ${CONDA_DIR}/etc rather than +# the default of /etc, since the non-root jovyan user can write +# to ${CONDA_DIR}/etc but not to /etc +ENV DASK_ROOT_CONFIG=${CONDA_DIR}/etc + +# COPY the current content to $HOME/build +RUN mkdir -p $HOME/build/ +COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/ +COPY --chown=$NB_UID:$NB_GID overrides.json $HOME/build/ + +USER root + +# Make /opt/ user writeable so it can be used by build-* scripts +RUN fix-permissions /opt/ + +# Install OS packages and then clean up +COPY --chown=$NB_UID:$NB_GID scripts/*.sh /opt/scripts/ +# Read apt.txt line by line, and execute apt-get install for each line +RUN cd build && bash /opt/scripts/apt-install.sh + +USER $NB_USER +# setup conda environments +RUN cd build && bash /opt/scripts/conda-env-install.sh \ + # Change dispaly name of the default kernel + && mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV" + +# Any other build-* scripts # +RUN cd $HOME/build \ + ; for script in `ls build-*`; do \ + echo "Found script ${script} ..." \ + && chmod +x $script \ + && ./$script \ + ; done +# --------------------------- # + +# cache location: use /tmp/ so we don't fill up $HOME +ENV CACHE_DIR=/tmp/cache +RUN mkdir -p $CACHE_DIR +ENV XDG_CACHE_HOME=$CACHE_DIR +ENV XDG_CACHE_DIR=$CACHE_DIR +ENV XDG_CACHE_DIR=$CACHE_DIR + +# Set default config for pip and conda +# change default pip cache from ~/.cache/pip +ENV PIP_CACHE_DIR=$CACHE_DIR/pip +RUN mkdir -p $PIP_CACHE_DIR + +COPY --chown=$NB_UID:$NB_GID condarc ${CONDA_DIR}/.condarc +# ------------------------------------# + +# add a script for to run before the jupyter session +USER root +COPY --chmod=0755 pre-notebook.sh /usr/local/bin/before-notebook.d/20-pre-notebook.sh +USER $NB_USER + +# Make $CONDA_ENV default; do it at the global level +# because ~/.bashrc is not loaded when user space is mounted +# Also, add it to before-notebook.d main script +USER root +ENV PATH=$CONDA_DIR/envs/$CONDA_ENV/bin:$PATH +RUN cat $HOME/.bashrc >> /etc/bash.bashrc \ + && printf "\nconda activate \$CONDA_ENV\n" >> /etc/bash.bashrc \ + && printf "\nconda activate \$CONDA_ENV\n" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh \ + && printf "" > $HOME/.bashrc +USER $NB_USER + +# For vscode +ENV CODE_EXECUTABLE=openvscode-server + + +# For outside mount when using outside fornax +RUN mkdir -p /opt/workspace +VOLUME /opt/workspace + +# reset user and location +RUN rm -r $HOME/build $HOME/work /tmp/* +WORKDIR ${HOME} + +# Useful environment variables +ENV NOTEBOOK_DIR=${HOME}/notebooks \ + NOUPDATE=${HOME}/.no-notebook-update.txt + +# Install OS packages and then clean up +ONBUILD RUN mkdir -p $HOME/build +ONBUILD COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/ +ONBUILD USER root +ONBUILD RUN mkdir -p build && cd build && bash /opt/scripts/apt-install.sh +ONBUILD USER $NB_USER +# ------------------------------------ # + + +# setup conda environments +ONBUILD RUN cd build && bash /opt/scripts/conda-env-install.sh +# and ensure the correct display name +ONBUILD RUN mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV" +# ----------------------- # + +# Any other build-* scripts # +ONBUILD RUN cd build \ + ; for script in `ls build-*`; do \ + echo "Found script ${script} ..." \ + && chmod +x $script \ + && ./$script \ + ; done +# --------------------------- # + +# landing page +ONBUILD COPY --chown=$NB_UID:$NB_GID --chmod=644 introduction.md* /opt/scripts/ + +ONBUILD RUN rm -r $HOME/build +ONBUILD USER ${NB_USER} +ONBUILD WORKDIR ${HOME} diff --git a/base_image/README.md b/base_image/README.md new file mode 100644 index 0000000..bcfa3a3 --- /dev/null +++ b/base_image/README.md @@ -0,0 +1,7 @@ +# Content + + +# Versions +Version number is in `.image_tags` + +- 0.1: A base image that starts from `jupyter/minimal-notebook:2023-04-17` \ No newline at end of file diff --git a/base_image/apt.txt b/base_image/apt.txt new file mode 100644 index 0000000..85ae466 --- /dev/null +++ b/base_image/apt.txt @@ -0,0 +1,14 @@ +fonts-liberation +pandoc +vim +nano +emacs-nox +fuse +bzip2 +git +curl +zip +build-essential +gcc +make +gfortran \ No newline at end of file diff --git a/base_image/build-config.sh b/base_image/build-config.sh new file mode 100644 index 0000000..25a5266 --- /dev/null +++ b/base_image/build-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +env=notebook + +# jupyter overrides +mkdir -p $CONDA_DIR/envs/$env/share/jupyter/lab/settings/ +mv ~/build/overrides.json $CONDA_DIR/envs/$env/share/jupyter/lab/settings/ + + +# disable the annoucement extension +mamba run -n $env jupyter labextension disable "@jupyterlab/apputils-extension:announcements" diff --git a/base_image/conda-notebook-lock.yml b/base_image/conda-notebook-lock.yml new file mode 100644 index 0000000..0f5e1c7 --- /dev/null +++ b/base_image/conda-notebook-lock.yml @@ -0,0 +1,394 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroquery=0.4.7=pyhd8ed1ab_0 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - gflags=2.2.2=h5888daf_1005 + - glog=0.7.1=hbabe93e_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - hpack=4.0.0=pyh9f0ad1d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=25_linux64_openblas + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numpy=1.26.4=py311h64a7726_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.11.0=he550d4f_1_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_0 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zeromq=4.3.5=h3b0a872_7 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - gitdb==4.0.11 + - gitpython==3.1.43 + - jupyter-cpu-alive==0.1.2 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - smmap==5.0.1 +prefix: /opt/conda/envs/notebook diff --git a/base_image/conda-notebook.yml b/base_image/conda-notebook.yml new file mode 100644 index 0000000..cad0d4b --- /dev/null +++ b/base_image/conda-notebook.yml @@ -0,0 +1,32 @@ +channels: + - conda-forge + - nodefaults +dependencies: + - python==3.11 + - jupyterlab==4.2.4 + - jupyterhub==5.1.0 + - notebook==7.2.1 + - ipykernel + - ipywidgets + - jupytext + - nbgitpuller + - dask + - dask-labextension + - numpy<2 + - scipy + - pandas + - astropy + - pyvo + - astroquery + - tqdm + - matplotlib + - seaborn + - openvscode-server + - jupyter-vscode-proxy + - pip + - pip: + - jupyterlab_execute_time + - jupyter-resource-usage + - jupyterlab-git + - jupyterlab-myst + - jupyter_cpu_alive diff --git a/base_image/condarc b/base_image/condarc new file mode 100644 index 0000000..1cff18c --- /dev/null +++ b/base_image/condarc @@ -0,0 +1,8 @@ +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge +envs_dirs: + - /opt/conda/envs +pkgs_dirs: + - /tmp/cache/conda \ No newline at end of file diff --git a/base_image/overrides.json b/base_image/overrides.json new file mode 100644 index 0000000..43d621c --- /dev/null +++ b/base_image/overrides.json @@ -0,0 +1,13 @@ +{ + "@jupyterlab/docmanager-extension:plugin": { + "defaultViewers": { + "markdown": "Jupytext Notebook" + } + }, + "dask-labextension:plugin": { + "autoStartClient": true + }, + "@jupyterlab/notebook-extension": { + "scrollHeadingToTop": false + } +} \ No newline at end of file diff --git a/base_image/pre-notebook.sh b/base_image/pre-notebook.sh new file mode 100644 index 0000000..2f69c1d --- /dev/null +++ b/base_image/pre-notebook.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# if something fails, keep going, we don't want to the stop the server from loading +set +e + +# include any code that needs to be run before the start of the jupyter session + +# remove the old nb_conda_kernels defined in the user directory +# so they don't show up in the Jupyterlab launcher +rm -rf ~/.local/share/jupyter/kernels/conda-base-py &>/dev/null +rm -rf ~/.local/share/jupyter/kernels/conda-env-science_demo-py &>/dev/null + +# remove old condarc in the user's home; continue gracefully +find ~/.condarc -type f ! -newermt "2024-11-26" -delete &>/dev/null || true + +# cleanup cache accmulated before the new cache location at /tmp/cache +cd $HOME +for dir in users_conda_envs .astropy/cache .cache; do + rm $dir &>/dev/null +done + +# remove any old pip cache +pip cache purge &>/dev/null + +# clean any conda cache +mamba clean -yaf &>/dev/null + +# Make sure we have a landing page +if test -z $NOTEBOOK_DIR; then export NOTEBOOK_DIR=$HOME/notebooks; fi +mkdir -p $NOTEBOOK_DIR +if test -f /opt/scripts/introduction.md && ! test -f $HOME/.no-notebook-update.txt; then + mv /opt/scripts/introduction.md $NOTEBOOK_DIR +fi +touch $NOTEBOOK_DIR/introduction.md diff --git a/base_image/scripts/apt-install.sh b/base_image/scripts/apt-install.sh new file mode 100644 index 0000000..f0dcf68 --- /dev/null +++ b/base_image/scripts/apt-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# This will be run as root inside Dockerfile + +if test -f "apt.txt" ; then + echo "Found apt.txt; using it ..." + apt-get update --fix-missing > /dev/null + xargs -a apt.txt apt-get install -y + apt-get clean + apt-get -y autoremove + rm -rf /var/lib/apt/lists/* +fi \ No newline at end of file diff --git a/base_image/scripts/conda-env-install.sh b/base_image/scripts/conda-env-install.sh new file mode 100644 index 0000000..2556ab4 --- /dev/null +++ b/base_image/scripts/conda-env-install.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e +set -o pipefail + +# handle conda environment and lock files +# look for conda-{env}-lock.yml and conda-{env}.yml files + +for envfile in `ls conda-*.yml | grep -v lock`; do + env=`echo $envfile | sed -n 's/conda-\(.*\)\.yml/\1/p'` + if test -f conda-${env}-lock.yml; then + ENVFILE=conda-${env}-lock.yml + echo "Found $ENVFILE, using it ..." + mamba env update -n ${env} -f $ENVFILE + elif test -f conda-${env}.yml; then + ENVFILE=conda-${env}.yml + echo "Found $ENVFILE, using it ..." + mamba env update -n ${env} -f $ENVFILE + elif [[ "$env"=="$CONDA_ENV" ]]; then + echo "Defaulting to basic env ..." + mamba create --name $env python=3.11 jupyterlab + fi + if [ "$env" != "$CONDA_ENV" ]; then + # add the environement as a jupyter kernel + # CONDA_ENV is defined in the dockerfile + mamba install -n $env ipykernel + mamba run -n $env python -m ipykernel install --name $env --prefix $CONDA_DIR/envs/$CONDA_ENV + fi +done + +# clean +mamba clean -yaf +pip cache purge +find ${CONDA_DIR} -follow -type f -name '*.a' -name '*.pyc' -delete +find ${CONDA_DIR} -follow -type f -name '*.js.map' -delete diff --git a/fornax_forced_photometry/README.md b/fornax_forced_photometry/README.md deleted file mode 100644 index 7cbc55b..0000000 --- a/fornax_forced_photometry/README.md +++ /dev/null @@ -1 +0,0 @@ -Packages from the astropy ecosystem + tractor and its deepndencies diff --git a/heasoft/Dockerfile b/heasoft/Dockerfile new file mode 100644 index 0000000..9f25e26 --- /dev/null +++ b/heasoft/Dockerfile @@ -0,0 +1,18 @@ +# ONBUILD instructions in base-image/Dockerfile are used to +# perform certain actions based on the presence of specific +# files (such as conda-linux-64.lock, start) in this repo. +# Refer to the base-image/Dockerfile for documentation. +ARG IMAGE_TAG=latest +ARG BASE_IMAGE_TAG=latest +ARG REPOSITORY=fornax-core/images + +FROM ghcr.io/${REPOSITORY}/base_image:${BASE_IMAGE_TAG} + +# set default conda env +ENV CONDA_ENV=notebook + + +LABEL org.opencontainers.image.ref.name="Fornax High Energy Astrophysics" +LABEL gov.nasa.smce.fornax.jupyterhub.image="${IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.base_image="${BASE_IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.repository="${REPOSITORY}" diff --git a/heasoft/README.md b/heasoft/README.md new file mode 100644 index 0000000..a4f38b8 --- /dev/null +++ b/heasoft/README.md @@ -0,0 +1,19 @@ +# Content +This directory contains the docker image for HEAsoft. + + +# Building locally +To build the image locally, run the following from the top level of `fornax-images`: +``` +version=6.32.1 +docker build -t heasoft:$version --network=host --build-arg heasoft_version=$version heasoft/ +``` + +# Building and pushing to the ECR Registry +This is done automatically by the CI once pushed to gitlab. +Note that this `README.md` file is required for the CI to work. + +# Versions +Version number is in `.image_tags` + +- 6.32.1: Heasoft image that starts with `base_image-v0.1` diff --git a/heasoft/apt.txt b/heasoft/apt.txt new file mode 100644 index 0000000..baa2a47 --- /dev/null +++ b/heasoft/apt.txt @@ -0,0 +1,13 @@ +g++ +libncurses-dev +libreadline-dev +libgsl-dev +file +make +ncurses-dev +perl-modules +tcsh +wget +xorg-dev +libcurl4 +libssl3 \ No newline at end of file diff --git a/heasoft/build-heasoft.sh b/heasoft/build-heasoft.sh new file mode 100644 index 0000000..5c93225 --- /dev/null +++ b/heasoft/build-heasoft.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +set -e +set -o pipefail + +pythonenv=notebook +heasoft_version=6.34 +install_dir=/opt/heasoft +caldb_dir=/opt/caldb + +# option to compile only part of heasoft when testing +# set to "yes" to activate +fast="no" + +export PYTHON=${CONDA_DIR}/envs/${pythonenv}/bin/python +heasoft_tarfile_suffix=src_no_xspec_modeldata +echo " --- Downloading heasoft ---" +wget -q https://heasarc.gsfc.nasa.gov/FTP/software/lheasoft/lheasoft${heasoft_version}/heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz +tar xzf heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz > tar.log.txt +rm -f heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz +cd heasoft-${heasoft_version} + +if [ $fast == "yes" ]; then + rm -r demo integral nicer suzaku Xspec calet heagen heasim hitomixrism ixpe maxi nustar swift + rm -r attitude ftools heasptools heatools tcltk +fi + +# ----------------------- # +# Handle large refdata that we don't want in the image +# we add them later +refdata=( + "ftools/xstar/data/atdb.fits" + "heasim/skyback/torus1006.fits" +) +# remove refdata from the image +for file in "${refdata[@]}"; do + rm -f $file +done +# ----------------------- # + +## Configure, make, and install ... +echo " --- Configure heasoft ---" +cd BUILD_DIR/ +# write stdout to a file (can be long), and leave stderr on screen +./configure --prefix=$install_dir --enable-collapse > config.log.txt + +echo " --- Build heasoft ---" +make > build.log.txt +make install > install.log.txt +make clean > clean.log.txt +gzip -9 *.log.txt && mv *.log.txt.gz $install_dir +cd .. +if [ ! $fast == "yes" ]; then + cp -p Xspec/BUILD_DIR/hmakerc $install_dir/x86_64*/bin/ + cp -p Xspec/BUILD_DIR/Makefile-std $install_dir/x86_64*/bin/ +fi +mv Release_Notes* $install_dir +cd .. && rm -rf heasoft-${heasoft_version} + +# Tweak Xspec settings for a no-X11 environment +if [ ! $fast == "yes" ]; then + printf "setplot splashpage off\ncpd /GIF\n" >> $install_dir/spectral/scripts/global_customize.tcl +fi + +# enable remote CALDB for now. +export CALDBCONFIG=$caldb_dir/caldb.config +export CALDBALIAS=$caldb_dir/alias_config.fits +export CALDB=/home/jovyan/efs/caldb +mkdir -p $caldb_dir +cd $caldb_dir +wget -q https://heasarc.gsfc.nasa.gov/FTP/caldb/software/tools/caldb.config +wget -q https://heasarc.gsfc.nasa.gov/FTP/caldb/software/tools/alias_config.fits + +# setup scripts so it runs when (heasoft) is activated +HEADAS=`ls -d $install_dir/x86_64*` +CALDB=https://heasarc.gsfc.nasa.gov/FTP/caldb + +# bash init script +cat < activate_heasoft.sh +export HEADAS=$HEADAS +export CALDB=$CALDB +source \$HEADAS/headas-init.sh +if [ -z \$CALDB ] ; then + echo "** No CALDB data. **" +elif [[ \$CALDB == http* ]]; then + echo "** Using Remote CALDB **" +elif [ -d \$CALDB ]; then + echo "** Using CALDB in \$CALDB **" + source \$CALDB/software/tools/caldbinit.sh +else + echo "** No CALDB data. **" +fi +EOF + +_activatedir=$CONDA_DIR/envs/${pythonenv}/etc/conda/activate.d/ +mkdir -p $_activatedir +mv activate_heasoft.sh $_activatedir + + +# ----------------------- # +# write a script to download the refdata +cat < $HEADAS/bin/download-refdata.sh +#!/usr/bin/env bash + +if test -z \$HEADAS; then + echo "\$HEADAS is not defined. Make sure heasoft is installed" +else + cd \$HEADAS/refdata + for file in ${refdata[@]}; do + if ! test -f \$(basename \$file); then + echo "downloading \$file ..." + wget -q https://heasarc.gsfc.nasa.gov/FTP/software/lheasoft/lheasoft${heasoft_version}/heasoft-${heasoft_version}/\$file + fi + done +fi +EOF +chmod +x $HEADAS/bin/download-refdata.sh \ No newline at end of file diff --git a/heasoft/conda-notebook-lock.yml b/heasoft/conda-notebook-lock.yml new file mode 100644 index 0000000..3801c15 --- /dev/null +++ b/heasoft/conda-notebook-lock.yml @@ -0,0 +1,395 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroquery=0.4.7=pyhd8ed1ab_0 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - gflags=2.2.2=h5888daf_1005 + - glog=0.7.1=hbabe93e_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - hpack=4.0.0=pyh9f0ad1d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=25_linux64_openblas + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numpy=2.1.3=py311h71ddf71_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.11.10=hc5c86c4_3_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_0 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zeromq=4.3.5=h3b0a872_7 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - gitdb==4.0.11 + - gitpython==3.1.43 + - jupyter-cpu-alive==0.1.2 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - smmap==5.0.1 +prefix: /opt/conda/envs/notebook diff --git a/heasoft/conda-notebook.yml b/heasoft/conda-notebook.yml new file mode 100644 index 0000000..f2ff1e1 --- /dev/null +++ b/heasoft/conda-notebook.yml @@ -0,0 +1,15 @@ +name: notebook +channels: + - conda-forge + - nodefaults +dependencies: + - python=3.11 + - numpy + - scipy>1.6 + - astropy>=4.0 + - matplotlib + - ipykernel + - pip + - pip: + - astroquery +prefix: /opt/conda/envs/notebook \ No newline at end of file diff --git a/src/README.rst b/src/README.rst new file mode 100644 index 0000000..84e1c0e --- /dev/null +++ b/src/README.rst @@ -0,0 +1,91 @@ +New Fornax Images Repo (alpha) +============================== + +This repo houses Dockerfiles and supporting files for the "base_image", and +"tractor" images within subdirectories of its root. (It also houses heasoft, +but that is currently disabled until we can package it in such a way that it +doesn't need to compile every time). + +It also includes three GitHub workflows: + +- A workflow which will build images for tractor and base_image to the branch + being checked into (e.g. "base_image:main" or "tractor:main"), when anything + in "tractor" or "base_image" changes. It also pushes the resulting images + into the container registry of this repository. + + This happens for every GitHub branch, not just "main". + + NB: when "base_image" changes, "tractor" will also be rebuilt against the new + base_image related to the branch. However, if only "tractor" changes, + "base_image" is not rebuilt. + +- A workflow that runs on release that tags the latest "main" image with the + release tag and a symbolic rolling tag named "stable". + +- A workflow that runs the tests of the building and tagging machinery (when + anything in "src" changes). + +For each image built, it is pushed to the GitHub container registry associated +with this repostory. + +See the "Packages" link on the right hand side of the main repository page for +a list of images in the container registry. + +See the "Actions" tab of the repository to see the results of each workflow. + +NB: The code in this directory has only been tested with Python3.11 and better. + +Notable Changes and Questions +============================= + +- We use GitHub Container Registry instead of Amazon's. + +- Instead of image names like ``fornax_images:base-image-XYZ``, and + ``fornax_images:heasoft-XYZ``, we produce images like ``base_image:XYZ`` and + ``tractor:XYZ`` as it is easy enough to do when we use the GitHub container + registry, and it's more "normal". + +- All of the logic to build and push (or not build or push) exists within + ``src/buildimages.py``, which is executed by the GitHub workflow actions + defined within ``.github/workflows/images.yml``. + +- The event that fires off the worfklow that produces the images is currently a + a push or pull request event. It happens on every push for every branch. + +- The event that creates "imagename:v0.1.1" tags is a GitHub release event. It + tags all "main" images as released using the release tag name supplied plus a + symbolic "stable" tag. + +- ``src/buildimages.py`` can be run standalone from any machine. It can only + push images if it is logged in to a GitHub account with an API token that + permitted to create "packages" using ``docker login`` . + +- There are several other Dockerfiles in the older GitLab fornax-images repo. + They seemed at a quick glance unrelated to the others because they didn't use + the base image. Can we confirm? + +- The limitations of a free GitHub account with respect to action runners + appears to be these: + + - Only one runner may be available at any time. + + - The active runner is executed on a run-of-the-mill machine. + + - The runner cannot be self-hosted. + +Findings +======== + +- With or without an enterprise subscripion, you can fork a repository into + another repository in the same organization (e.g. nasa/fornax-images to + nasa/fornax-images2). It requires the appropriate permissions granted to the + forking user from the organization manager. + https://github.blog/changelog/2022-06-27-improved-innersource-collaboration-and-enterprise-fork-policies/. + +- When you fork a repo with workflows in it, initially the workflows are + disabled. You can enable the workflows by visiting the Actions tab within + the fork and clicking the button to reenable them. Then the forked repo + actions work just like the original repo actions, and can contain their own + registries, etc. + + diff --git a/src/TODO.txt b/src/TODO.txt new file mode 100644 index 0000000..796bf3c --- /dev/null +++ b/src/TODO.txt @@ -0,0 +1,20 @@ +- Optimize by avoiding various image builds if their directories haven't + changed. + +Questions +--------- + +- What is your process for generating and checking in the conda lockfiles? I + suppose it's a run of build.py done locally. Are we trying to capture the + state of things after a generic conda update of all packages that's implied + by conda-env-install's mamba env update? At what point does it not update + such that QA can be done? (answer: manual) + +- My presumpution about a "release" was that we would generate all of the + images (base_image, tractor, and heasoft) together, and they would all have + the same version tag. Tractor and heasoft would depend on the base image + with the same version. This isn't how things work on GitLab currently. + (answer: use push trigger for dev, release trigger for stable) + +- dpkg pkg for Heasoft? (answer: abdu is developing a conda pkg) + diff --git a/src/buildimages.py b/src/buildimages.py new file mode 100644 index 0000000..2f5036b --- /dev/null +++ b/src/buildimages.py @@ -0,0 +1,263 @@ +import argparse +import glob +import logging +import os +import re +import subprocess +import sys + +# NB: we don't use docker-py because it doesn't support modern Dockerfiles + +order = ( + "base_image", + "tractor", + # "heasoft", # reenable heasoft once we have it packaged +) + + +class Base: + # this class only exists (instead of module-level functions) to make unit + # testing less painful; it is also used by "release.py" + def __init__(self, logger): + self.logger = logger + + def out(self, msg, severity=logging.INFO): + self.logger.log(severity, msg) + sys.stdout.flush() + + def run(self, command, timeout, **runargs): + self.out(f"Running {command} with timeout {timeout}") + result = subprocess.run( + command, + shell=True, + check=True, + text=True, + timeout=timeout, + **runargs, + ) + return result + + +class Builder(Base): + def build( + self, + repository, + path, + tag, + build_args=None, + build_pars=None, + ): + extra_args = [] + # cope with forks of the repository (see tractor/heasoft Dockerfiles) by + # setting a build arg + build_args = build_args or [] + default_tag = tag.rsplit(":", 1)[1] + if not any([x.startswith("REPOSITORY=") for x in build_args]): + build_args.append(f"REPOSITORY={repository}") + if not any([x.startswith("BASE_IMAGE_TAG=") for x in build_args]): + build_args.append(f"BASE_IMAGE_TAG={default_tag}") + if not any([x.startswith("IMAGE_TAG=") for x in build_args]): + build_args.append(f"IMAGE_TAG={default_tag}") + + for arg in build_args: + if not arg.count("=") == 1: + raise ValueError( + f"build_args should be of the form 'name=value'. " + f"Got '{arg}'." + ) + name, val = arg.split("=", 1) + nameandval = f"{name}={val}" + extra_args.append(f"--build-arg {nameandval}") + + if build_pars: + extra_args.append(build_pars) + extra_args = " ".join(extra_args) + buildcommand = f"docker build {extra_args} --tag {tag} {path}" + self.out(f"Building {path} via '{buildcommand}'") + result = self.run(buildcommand, timeout=10000) + self.out(result) + + def push(self, tag): + pushcommand = f"docker push {tag}" + self.out(f"Pushing {tag} via '{pushcommand}'") + result = self.run(pushcommand, timeout=1000) + self.out(result) + + def remove_lockfiles(self, path): + self.out(f"Removing the lock files for {path}") + lockfiles = glob.glob(f"{path}/conda-*lock.yml") + for lockfile in lockfiles: + self.out(f"Removing {path}/{lockfile}") + os.unlink(lockfile) + + def update_lockfiles(self, path, repository, tag): + tag = tag.rsplit(":", 1)[1] + self.out(f"Updating the lock files for {path}") + envfiles = glob.glob(f"{path}/conda-*.yml") + envfiles = [ + env for env in glob.glob(f"{path}/conda-*.yml") if "lock" not in env + ] + for env in envfiles: + match = re.match(rf"{path}/conda-(.*).yml", env) + if match: + env_name = match[1] + else: + env_name = "base" + cmd = ( + f'docker run --entrypoint="" --rm ' + f"ghcr.io/{repository}/{path}:{tag} " + f"mamba env export -n {env_name}" + ) + + result = self.run(cmd, 500, capture_output=True) + lines = [] + include = False + for line in result.stdout.split("\n"): + if "name:" in line: + include = True + if include: + lines.append(line) + with open(f"{path}/conda-{env_name}-lock.yml", "w") as fp: + fp.write("\n".join(lines)) + + def builds_necessary(self, repository, tag, images): + tobuild = [] + for name in images: + if not name in order: + self.out(f"Unknown image name {name}", logging.ERROR) + raise SystemExit(2) + + # tractor and heasoft depend on base_image, so ordering is important + # here (I think) + for name in order: + if name in images: + struct = ( + name, + f"ghcr.io/{repository}/{name}:{tag}", + ) + tobuild.append(struct) + + return tobuild + + def chdir(self, path): + os.chdir(path) + + +def main( + builder, + repository, + tag=None, + do_push=False, + update_lock=False, + no_build=False, + build_args=None, + images=order, + build_pars=None +): + if no_build and do_push: + builder.out( + "--no-build and --push cannot be used together", logging.ERROR + ) + raise SystemExit(2) + + if build_pars is not None and ('--tag' in build_pars or '--build-arg' in build_pars): + builder.out( + "--tag and --build-arg cannot be passed in build-pars", logging.ERROR + ) + raise SystemExit(2) + + builder.out(f"Repository {repository}, tag {tag}") + tobuild = builder.builds_necessary(repository, tag, images) + + # parent dir of the dir of this file (root of this checkout) + here = __file__ + if not here.startswith(os.path.sep): + here = os.path.join(os.getcwd(), here) + + root = os.path.dirname(os.path.dirname(here)) + builder.chdir(root) # indirection for testing sanity + + for dockerdir, tag in tobuild: + if not no_build: + if update_lock: + builder.remove_lockfiles(dockerdir) + builder.build( + repository, dockerdir, tag, build_args, build_pars=build_pars + ) + if update_lock: + builder.update_lockfiles(dockerdir, repository, tag) + if do_push: + builder.push(tag) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument( + "repository", help="GH repository name (e.g. 'fornax-core/images')" + ) + ap.add_argument("tag", help="Container registry tag name (e.g. 'mybranch')") + ap.add_argument( + "--push", + action="store_true", + help=( + "After building, push to container registry (incompatible with " + "--no-build)." + ), + default=False, + ) + ap.add_argument( + "--update-lock", + action="store_true", + help="Update conda lock files. Meant to be used when run to update conda lock files in local directories. A suitable command might be 'python3 src/buildimages.py nasa-fornax/fornax-images mybranch --update-lock'", + default=False, + ) + ap.add_argument( + "--no-build", + action="store_true", + help="don't actually build images (incompatible with --push)", + default=False, + ) + ap.add_argument( + "--build-args", + nargs="*", + help=( + "Extra --build-arg arguments passed to docker build e.g. 'a=b c=d'" + ), + ) + ap.add_argument( + "--images", + nargs="*", + help=("Image names separated by spaces e.g. 'base_image tractor'"), + default=order, + ) + ap.add_argument( + "--build-pars", + help="Arguments to be passed directly to `docker build`", + default=None, + ) + + args = ap.parse_args() + + # see https://github.com/docker/docker-py/issues/2230 for rationale + # as to why we set DOCKER_BUILDKIT (--chmod flag to COPY) + os.environ["DOCKER_BUILDKIT"] = "1" + logging.basicConfig( + format="%(asctime)s|%(levelname)5s| %(message)s", + datefmt="%Y-%m-%d|%H:%M:%S", + ) + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + builder = Builder(logger) + + main( + builder, + args.repository, + args.tag, + args.push, + args.update_lock, + args.no_build, + args.build_args, + args.images, + args.build_pars, + ) diff --git a/src/neededimages.py b/src/neededimages.py new file mode 100644 index 0000000..9e7e170 --- /dev/null +++ b/src/neededimages.py @@ -0,0 +1,68 @@ +import argparse +import logging +import json +import subprocess + +from buildimages import order +from buildimages import Base + +base_images = ('base_image',) + +class Needer(Base): + def needs(self, repository, gh_token, dirs_changed, branch): + if isinstance(dirs_changed, str): + dirs_changed = json.loads(dirs_changed) + images_changed = set() + for image_name in order: + for fn in dirs_changed: + if f"{fn}/".startswith(f"{image_name}/"): + images_changed.add(image_name) + needed = [] + for image_name in order: + try: + result = self.run( + f'curl -s -H "Authorization: Bearer {gh_token}" ' + f"https://ghcr.io/v2/{repository}/{image_name}/tags/list", + 500, + capture_output=True, + ) + struct = json.loads(result.stdout) + if not struct: + needed.append(image_name) + continue + tags = struct.get("tags", []) + if (image_name in images_changed) or (not branch in tags): + needed.append(image_name) + else: + # when a base image changes, we rebuild all images + if any([x in needed for x in base_images]): + needed.append(image_name) + except subprocess.CalledProcessError: + needed.append(image_name) + + return needed + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument( + "repository", help="GH repository name (e.g. 'fornax-core/images')" + ) + ap.add_argument("gh_token", help="Github token(already base64ed)") + ap.add_argument("dirs_changed", help="JSON of directories changed") + ap.add_argument("branch", help="Branch in repo") + logging.basicConfig( + format="%(asctime)s|%(levelname)5s| %(message)s", + datefmt="%Y-%m-%d|%H:%M:%S", + ) + args = ap.parse_args() + logger = logging.getLogger() + logger.setLevel(logging.INFO) + needer = Needer(logger) + print( + " ".join( + needer.needs( + args.repository, args.gh_token, args.dirs_changed, args.branch + ) + ) + ) diff --git a/src/release.py b/src/release.py new file mode 100644 index 0000000..f2b80bf --- /dev/null +++ b/src/release.py @@ -0,0 +1,51 @@ +import argparse +import logging + +from buildimages import Base +from buildimages import order + + +class Tagger(Base): + def tag(self, repository, release_name, source_tag, symbolic_tag): + for name in order: + source = f"ghcr.io/{repository}/{name}:{source_tag}" + self.run(f"docker pull {source}", 1000) + + releasetarget = f"ghcr.io/{repository}/{name}:{release_name}" + releasetagcommand = f"docker tag {source} {releasetarget}" + self.out(self.run(releasetagcommand, 500)) + self.out(self.run(f"docker push {releasetarget}", 1000)) + + symbolictarget = f"ghcr.io/{repository}/{name}:{symbolic_tag}" + symbolictagcommand = f"docker tag {source} {symbolictarget}" + self.out(self.run(symbolictagcommand, 500)) + self.out(self.run(f"docker push {symbolictarget}", 1000)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument( + "repository", help="GH repository name (e.g. 'fornax-core/images')" + ) + ap.add_argument("release_name", help="Release name (e.g. 'v0.0.1')") + ap.add_argument( + "--source_tag", + help="CR source tag name (e.g. 'main')", + default="main", + ) + ap.add_argument( + "--symbolic_tag", + help="CR symbolic target tag name (e.g. 'stable')", + default="stable", + ) + logging.basicConfig( + format="%(asctime)s|%(levelname)5s| %(message)s", + datefmt="%Y-%m-%d|%H:%M:%S", + ) + args = ap.parse_args() + logger = logging.getLogger() + logger.setLevel(logging.INFO) + tagger = Tagger(logger) + tagger.tag( + args.repository, args.release_name, args.source_tag, args.symbolic_tag + ) diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..f2cf19e --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1 @@ +# here to appease setup-python GH action diff --git a/src/tests.py b/src/tests.py new file mode 100644 index 0000000..bd59719 --- /dev/null +++ b/src/tests.py @@ -0,0 +1,496 @@ +import glob +import logging +import os +import pathlib +import subprocess +import tempfile +import unittest + +import buildimages +import neededimages +import release + +# python3.11 -m unittest discover -s src or python src/test.py + + +class DummyBuilder: + def __init__(self): + self.logged = [] + self.builds = [] + self.removed_lockfiles = [] + self.updated_lockfiles = [] + self.chdirs = [] + self.pushed = [] + + def builds_necessary(self, repository_name, release_tag, images): + return [("a", "tag")] + + def out(self, msg, severity=None): + self.logged.append(msg) + + def build( + self, repository, dockerdir, tags, build_args, build_pars=None, + ): + self.builds.append((repository, dockerdir, tags, build_args, build_pars)) + + def remove_lockfiles(self, dockerdir): + self.removed_lockfiles.append(dockerdir) + + def update_lockfiles(self, dockerdir, repository_name, tag): + self.updated_lockfiles.append((dockerdir, repository_name, tag)) + + def push(self, tags): + self.pushed.append(tags) + + def chdir(self, path): + self.chdirs.append(path) + + +class TestMain(unittest.TestCase): + def test_conflicting_params(self): + builder = DummyBuilder() + with self.assertRaises(SystemExit): + buildimages.main( + builder, + "repo_name", + "release_tag", + no_build=True, + do_push=True, + ) + self.assertEqual( + builder.logged, ["--no-build and --push cannot be used together"] + ) + + def test_conflicting_build_pars_tag(self): + builder = DummyBuilder() + with self.assertRaises(SystemExit): + buildimages.main( + builder, + "repo_name", + "release_tag", + build_pars="--tag" + ) + self.assertEqual( + builder.logged, ["--tag and --build-arg cannot be passed in build-pars"] + ) + + def test_conflicting_build_pars_arg(self): + builder = DummyBuilder() + with self.assertRaises(SystemExit): + buildimages.main( + builder, + "repo_name", + "release_tag", + build_pars="--build-arg" + ) + self.assertEqual( + builder.logged, ["--tag and --build-arg cannot be passed in build-pars"] + ) + + def test_main_nobuild(self): + builder = DummyBuilder() + buildimages.main( + builder, + "repo_name", + "release_tag", + no_build=True, + ) + self.assertEqual( + builder.logged, ["Repository repo_name, tag release_tag"] + ) + self.assertEqual(builder.removed_lockfiles, []) + self.assertEqual(builder.builds, []) + self.assertEqual(builder.updated_lockfiles, []) + self.assertEqual( + builder.chdirs, [ + os.path.realpath(os.path.dirname(os.path.dirname(__file__))) + ] + ) + self.assertEqual(builder.pushed, []) + + def test_main_update_lock(self): + builder = DummyBuilder() + buildimages.main( + builder, + "repo_name", + "release_tag", + update_lock=True, + ) + self.assertEqual( + builder.logged, ["Repository repo_name, tag release_tag"] + ) + self.assertEqual(builder.removed_lockfiles, ["a"]) + self.assertEqual( + builder.builds, [("repo_name", "a", "tag", None, None)] + ) + self.assertEqual(builder.updated_lockfiles, [("a", "repo_name", "tag")]) + self.assertEqual(builder.pushed, []) + + def test_main_do_push(self): + builder = DummyBuilder() + buildimages.main( + builder, + "repo_name", + "release_tag", + do_push=True, + ) + self.assertEqual( + builder.logged, ["Repository repo_name, tag release_tag"] + ) + self.assertEqual(builder.removed_lockfiles, []) + self.assertEqual( + builder.builds, [("repo_name", "a", "tag", None, None)] + ) + self.assertEqual(builder.updated_lockfiles, []) + self.assertEqual(builder.pushed, ["tag"]) + + +class DummyLogger: + def __init__(self): + self.messages = [] + + def log(self, severity, msg): + self.messages.append((severity, msg)) + + +class TestBuilder(unittest.TestCase): + def _makeOne(self): + logger = DummyLogger() + return buildimages.Builder(logger) + + def test_out(self): + builder = self._makeOne() + builder.out("foo", logging.ERROR) + self.assertEqual(builder.logger.messages, [(logging.ERROR, "foo")]) + + def test_run_success(self): + builder = self._makeOne() + result = builder.run("true", 200) + self.assertEqual( + builder.logger.messages, + [(logging.INFO, "Running true with timeout 200")], + ) + self.assertEqual(result.returncode, 0) + + def test_run_fail(self): + builder = self._makeOne() + with self.assertRaises(subprocess.CalledProcessError): + builder.run("/wont/exist", 200, capture_output=True) + self.assertEqual( + builder.logger.messages, + [(logging.INFO, "Running /wont/exist with timeout 200")], + ) + + def test_push(self): + builder = self._makeOne() + ran = [] + + def run(command, timeout): + ran.append((command, timeout)) + + builder.run = run + builder.push("a") + self.assertEqual(ran, [("docker push a", 1000)]) + + def test_remove_lockfiles(self): + with tempfile.TemporaryDirectory() as tmpdir: + files = ["conda-lock.yml", "conda-notebook-lock.yml", "conda-a.yml"] + for fn in files: + fn = os.path.join(tmpdir, fn) + pathlib.Path(fn).touch() + builder = self._makeOne() + builder.remove_lockfiles(tmpdir) + self.assertEqual(len(builder.logger.messages), 3) + self.assertTrue( + builder.logger.messages[0][1].startswith("Removing the lock") + ) + self.assertEqual(glob.glob(f"{tmpdir}/conda-*lock.yml"), []) + self.assertEqual( + glob.glob(f"{tmpdir}/conda-*"), + [os.path.join(tmpdir, "conda-a.yml")], + ) + + def test_chdir(self): + builder = self._makeOne() + oldwd = os.getcwd() + try: + with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + builder.chdir(tmpdir) + self.assertEqual(os.getcwd(), tmpdir) + finally: + os.chdir(oldwd) + + def test_builds_necessary_unknown_image(self): + builder = self._makeOne() + with self.assertRaises(SystemExit): + builder.builds_necessary("repo", "tag", ["nerp"]) + self.assertEqual( + builder.logger.messages, + [(logging.ERROR, "Unknown image name nerp")], + ) + + def test_builds_necessary(self): + builder = self._makeOne() + res = builder.builds_necessary("repo", "tag", ["tractor", "base_image"]) + self.assertEqual( + res, + [ + ( + "base_image", + "ghcr.io/repo/base_image:tag", + ), + ( + "tractor", + "ghcr.io/repo/tractor:tag", + ), + ], + ) + + def test_build_bad_build_arg(self): + builder = self._makeOne() + with self.assertRaises(ValueError): + builder.build("repo_name", "path", "a:main", build_args=["a"]) + + def test_build_bad_build_arg2(self): + builder = self._makeOne() + with self.assertRaises(ValueError): + builder.build("repo_name", "path", "a:main", build_args=["a==0"]) + + def test_build(self): + builder = self._makeOne() + ran = [] + + def run(command, timeout): + ran.append((command, timeout)) + + builder.run = run + builder.build( + "repo_name", "path", "a:main", build_args=["n=1"] + ) + self.assertEqual( + ran, + [ + ( + "docker build " + "--build-arg n=1 " + "--build-arg REPOSITORY=repo_name " + "--build-arg BASE_IMAGE_TAG=main " + "--build-arg IMAGE_TAG=main " + "--tag a:main path", + 10000, + ), + ], + ) + + def test_build_pars(self): + builder = self._makeOne() + ran = [] + + def run(command, timeout): + ran.append((command, timeout)) + + builder.run = run + builder.build( + "repo_name", "path", "a:main", build_args=["n=1"], build_pars="--no-cache=true --network=host" + ) + self.assertEqual( + ran, + [ + ( + "docker build " + "--build-arg n=1 " + "--build-arg REPOSITORY=repo_name " + "--build-arg BASE_IMAGE_TAG=main " + "--build-arg IMAGE_TAG=main " + "--no-cache=true " + "--network=host " + "--tag a:main path", + 10000, + ), + ], + ) + + def test_build_custom_repo_name(self): + builder = self._makeOne() + ran = [] + + def run(command, timeout): + ran.append((command, timeout)) + + builder.run = run + builder.build( + "repo_name", + "path", + "a:main", + build_args=["n=1", "REPOSITORY=another"], + ) + self.assertEqual( + ran, + [ + ( + "docker build --build-arg n=1 " + "--build-arg REPOSITORY=another " + "--build-arg BASE_IMAGE_TAG=main " + "--build-arg IMAGE_TAG=main " + "--tag a:main path", + 10000, + ), + ], + ) + + def test_update_lockfiles(self): + class DummyResult: + stdout = "woo\nyoo\nname:\nnextline" + + ran = [] + + def run(cmd, timeout, **kw): + ran.append((cmd, timeout)) + return DummyResult + + with tempfile.TemporaryDirectory() as tmpdir: + files = [ + "conda-notebook-lock.yml", + "conda-a.yml", + "conda-a-lock.yml", + ] + for fn in files: + fn = os.path.join(tmpdir, fn) + pathlib.Path(fn).touch() + builder = self._makeOne() + builder.run = run + builder.update_lockfiles(tmpdir, "repo_name", "main:tag") + with open(os.path.join(tmpdir, "conda-a-lock.yml"), "r") as f: + result = f.read() + self.assertEqual(len(builder.logger.messages), 1) + nowfiles = os.listdir(tmpdir) + self.assertEqual(len(nowfiles), 3) + self.assertEqual(result, "name:\nnextline") + + +class TestTagger(unittest.TestCase): + def test_tag(self): + logger = DummyLogger() + ran = [] + + def run(command, timeout): + ran.append((command, timeout)) + + tagger = release.Tagger(logger) + + tagger.run = run + tagger.tag("repo", "release", "source_tag", "symbolic_tag") + self.assertEqual( + ran, + [ + ("docker pull ghcr.io/repo/base_image:source_tag", 1000), + ( + "docker tag ghcr.io/repo/base_image:source_tag " + "ghcr.io/repo/base_image:release", + 500, + ), + ("docker push ghcr.io/repo/base_image:release", 1000), + ( + "docker tag ghcr.io/repo/base_image:source_tag " + "ghcr.io/repo/base_image:symbolic_tag", + 500, + ), + ("docker push ghcr.io/repo/base_image:symbolic_tag", 1000), + ("docker pull ghcr.io/repo/tractor:source_tag", 1000), + ( + "docker tag ghcr.io/repo/tractor:source_tag ghcr.io/repo/tractor:release", + 500, + ), + ("docker push ghcr.io/repo/tractor:release", 1000), + ( + "docker tag ghcr.io/repo/tractor:source_tag " + "ghcr.io/repo/tractor:symbolic_tag", + 500, + ), + ("docker push ghcr.io/repo/tractor:symbolic_tag", 1000), + ], + ) + + +class TestNeeder(unittest.TestCase): + def test_needs_all_due_to_no_tags(self): + logger = DummyLogger() + ran = [] + + class DummyProcessResult: + stdout = '{"tags": []}' + + def run(command, timeout, capture_output=True): + ran.append((command, timeout)) + return DummyProcessResult + + needer = neededimages.Needer(logger) + + needer.run = run + result = needer.needs("repo", "token", "[]", "main") + self.assertEqual(result, ["base_image", "tractor"]) + + def test_needs_one_due_to_no_tag(self): + logger = DummyLogger() + ran = [] + + class DummyProcessResult: + stdout = '{"tags": ["main"]}' + + class TractorDummyProcessResult: + stdout = '{"tags": []}' + + def run(command, timeout, capture_output=True): + ran.append((command, timeout)) + if len(ran) <= 1: + return DummyProcessResult + return TractorDummyProcessResult + + needer = neededimages.Needer(logger) + + needer.run = run + result = needer.needs("repo", "token", "[]", "main") + self.assertEqual(result, ["tractor"]) + + def test_needs_one_due_to_dirchange(self): + logger = DummyLogger() + ran = [] + + class DummyProcessResult: + stdout = '{"tags": ["main"]}' + + def run(command, timeout, capture_output=True): + ran.append((command, timeout)) + return DummyProcessResult + + needer = neededimages.Needer(logger) + + needer.run = run + result = needer.needs("repo", "token", '["tractor"]', "main") + self.assertEqual(result, ["tractor"]) + + def test_needs_all_due_to_baseimage_change(self): + logger = DummyLogger() + ran = [] + + class BaseImageProcessResult: + stdout = '{"tags": ["main"]}' + + class TractorDummyProcessResult: + stdout = '{"tags": ["main"]}' + + def run(command, timeout, capture_output=True): + ran.append((command, timeout)) + if len(ran) <= 1: + return BaseImageProcessResult + return TractorDummyProcessResult + + needer = neededimages.Needer(logger) + + needer.run = run + result = needer.needs("repo", "token", '["base_image"]', "main") + self.assertEqual(result, ["base_image", "tractor"]) + +if __name__ == "__main__": + unittest.main() diff --git a/tractor/Dockerfile b/tractor/Dockerfile new file mode 100644 index 0000000..de3e2bf --- /dev/null +++ b/tractor/Dockerfile @@ -0,0 +1,20 @@ +# ONBUILD instructions in base-image/Dockerfile are used to +# perform certain actions based on the presence of specific +# files (such as conda-linux-64.lock, start) in this repo. +# Refer to the base-image/Dockerfile for documentation. +ARG BASE_IMAGE_TAG=latest +ARG IMAGE_TAG=latest +ARG REPOSITORY=fornax-core/images +FROM ghcr.io/${REPOSITORY}/base_image:${BASE_IMAGE_TAG} + +LABEL org.opencontainers.image.ref.name="Fornax Default Astrophysics" +LABEL gov.nasa.smce.fornax.jupyterhub.base_image="${BASE_IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.image="${IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.repository="${REPOSITORY}" +# add notebook updater script to run when jupyter starts +USER root +COPY --chmod=0755 update-notebooks.sh /usr/local/bin/before-notebook.d +USER $NB_USER + +# For firefly +ENV FIREFLY_URL=https://irsa.ipac.caltech.edu/irsaviewer diff --git a/tractor/README.md b/tractor/README.md new file mode 100644 index 0000000..29fc495 --- /dev/null +++ b/tractor/README.md @@ -0,0 +1,16 @@ +# Content + +Main astrophysics demo image, containing tractor and related tools + +Includes Firefly and Aladin for visualization + +## To build and test + + $ docker build --rm --progress plain --force-rm -t tractor-image . + + $ docker run -it -p 9888:8888 --rm --name smce-tractor tractor-image:latest + +Then open your browser tab to the localhost URL shown, replacing the port +8888 with 9888. You may have to reload the browser tab to get Jupyterlab +fully working. + diff --git a/tractor/apt.txt b/tractor/apt.txt new file mode 100644 index 0000000..0a00f59 --- /dev/null +++ b/tractor/apt.txt @@ -0,0 +1,2 @@ +libcairo2-dev + diff --git a/tractor/build-tractor.sh b/tractor/build-tractor.sh new file mode 100644 index 0000000..2b9be26 --- /dev/null +++ b/tractor/build-tractor.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -o pipefail + +pythonenv=notebook +astrometry_commit=e868ccd +tractor_commit=8059ae0 + +# Install astrometry.net and tractor +cd /tmp +git clone https://github.com/dstndstn/astrometry.net.git +cd astrometry.net +git config --global --add safe.directory $PWD +git checkout $astrometry_commit +conda run -n $pythonenv make +conda run -n $pythonenv make py +conda run -n $pythonenv make extra +conda run -n $pythonenv make install INSTALL_DIR=${CONDA_DIR}/envs/${pythonenv} +mv ${CONDA_DIR}/envs/$pythonenv/lib/python/astrometry \ + ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/ + +cd /tmp +git clone https://github.com/dstndstn/tractor.git +cd tractor +git checkout $tractor_commit +conda run -n $pythonenv python setup.py build_ext --inplace --with-cython +conda run -n $pythonenv pip install --no-cache-dir . --target ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/ +cd $HOME +rm -rf /tmp/astrometry.net /tmp/tractor \ No newline at end of file diff --git a/tractor/conda-notebook-lock.yml b/tractor/conda-notebook-lock.yml new file mode 100644 index 0000000..eea643c --- /dev/null +++ b/tractor/conda-notebook-lock.yml @@ -0,0 +1,526 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - acstools=3.7.2=pyhd8ed1ab_0 + - affine=2.4.0=pyhd8ed1ab_0 + - aiobotocore=2.15.2=pyhd8ed1ab_0 + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aioitertools=0.12.0=pyhd8ed1ab_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - aom=3.9.1=hac33072_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - asciitree=0.3.3=py_2 + - asdf=4.0.0=pyhd8ed1ab_0 + - asdf-astropy=0.7.0=pyhd8ed1ab_0 + - asdf-coordinates-schemas=0.3.0=pyhd8ed1ab_0 + - asdf-standard=1.1.1=pyhd8ed1ab_0 + - asdf-transform-schemas=0.5.0=pyhd8ed1ab_0 + - asdf-wcs-schemas=0.4.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-healpix=1.0.3=py311h9f3472d_2 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroscrappy=1.2.0=py311h9ecbd09_1 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - autograd=1.7.0=pyhd8ed1ab_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - blosc=1.21.6=hef167b5_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - boto3=1.35.36=pyhd8ed1ab_0 + - botocore=1.35.36=pyge310_1234567_0 + - bottleneck=1.4.2=py311h9f3472d_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - brunsli=0.1=h9c3ff4c_0 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - c-blosc2=2.15.1=hc57e6cf_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - ccdproc=2.4.2=pyhd8ed1ab_0 + - cdshealpix=0.7.0=py311h9ecbd09_1 + - ceres-solver=2.2.0=ha77e7a2_4 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - cfitsio=4.4.1=ha728647_2 + - charls=2.4.2=h59595ed_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - click-plugins=1.1.1=py_0 + - cligj=0.7.2=pyhd8ed1ab_1 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cython=3.0.11=py311h55d416d_3 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dav1d=1.2.1=hd590300_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - deprecated=1.2.15=pyhff2d567_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - eigen=3.4.0=h00ab1b0_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - fasteners=0.17.3=pyhd8ed1ab_0 + - fbpca=1.0=py_0 + - firefly-client=3.1.0=pyhd8ed1ab_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - freexl=2.0.0=h743c826_0 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - future=1.0.0=pyhd8ed1ab_0 + - geos=3.13.0=h5888daf_0 + - geotiff=1.7.3=h77b800c_3 + - gflags=2.2.2=h5888daf_1005 + - ghostscript=10.04.0=h5888daf_0 + - giflib=5.2.2=hd590300_0 + - glog=0.7.1=hbabe93e_0 + - gmp=6.3.0=hac33072_2 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - gwcs=0.21.0=pyhd8ed1ab_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - healpy=1.18.0=py311hc30af9e_0 + - hipscat=0.3.5=pyhd8ed1ab_0 + - hpack=4.0.0=pyh9f0ad1d_0 + - hpgeom=1.4.0=py311h9f3472d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - imagecodecs=2024.9.22=py311h7d28041_0 + - imageio=2.36.0=pyh12aca89_1 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - jmespath=1.0.1=pyhd8ed1ab_0 + - joblib=1.4.2=pyhd8ed1ab_0 + - json-c=0.18=h6688a6e_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - jxrlib=1.1=hd590300_3 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lazy-loader=0.4=pyhd8ed1ab_1 + - lazy_loader=0.4=pyhd8ed1ab_1 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libaec=1.1.3=h59595ed_0 + - libarchive=3.7.7=hadbb8c3_0 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libavif16=1.1.1=h1909e37_2 + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libde265=1.0.15=h00ab1b0_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgdal-core=3.10.0=hef9eae6_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran-ng=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libheif=1.18.2=gpl_hffcb242_100 + - libhwloc=2.11.2=default_h0d58e46_1001 + - libhwy=1.1.0=h00ab1b0_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - libjxl=0.11.1=hdb8da77_0 + - libkml=1.3.0=hf539b9f_1021 + - liblapack=3.9.0=25_linux64_openblas + - libllvm14=14.0.6=hcd5def8_4 + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - librttopo=1.1.0=h97f6797_17 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libspatialite=5.1.0=h1b4f908_11 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - libzopfli=1.0.3=h9c3ff4c_0 + - lightkurve=2.5.0=pyhd8ed1ab_0 + - llvmlite=0.43.0=py311h9c9ff8c_1 + - locket=1.0.0=pyhd8ed1ab_0 + - lsdb=0.2.6=pyhd8ed1ab_0 + - lsst-sphgeom=27.2024.4400=py311h7db5c69_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=hd590300_1001 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - memoization=0.4.0=pyhd8ed1ab_1 + - metis=5.1.0=hd0bcaf9_1007 + - minizip=4.0.7=h401b404_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - mocpy=0.17.0=py311h9ecbd09_1 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - mpfr=4.2.1=h90cbb55_3 + - mpl_animators=1.2.0=pyhd8ed1ab_0 + - mpld3=0.5.10=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - ndcube=2.2.4=pyhd8ed1ab_0 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - netpbm=10.73.43=pl5321h6ae6222_4 + - networkx=3.4.2=pyh267e887_2 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numba=0.60.0=py311h4bc866e_0 + - numcodecs=0.14.1=py311h7db5c69_0 + - numpy=1.26.4=py311h64a7726_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - oktopus=0.1.2=py_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - perl=5.32.1=7_hd590300_perl5 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pgplot=5.2.2=hbeaba86_1009 + - photutils=2.0.2=py311h9f3472d_1 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkg-config=0.29.2=h4bc722e_1009 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - proj=9.5.0=h12925eb_0 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pynndescent=0.5.13=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.11.0=he550d4f_1_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pywavelets=1.7.0=py311h9f3472d_2 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - rasterio=1.4.2=py311h5394301_1 + - rav1e=0.6.6=he8a937b_2 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - regions=0.10=py311h9f3472d_0 + - reproject=0.14.1=py311h9f3472d_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - s3fs=2024.10.0=pyhd8ed1ab_0 + - s3transfer=0.10.4=pyhd8ed1ab_0 + - scikit-image=0.24.0=py311h7db5c69_3 + - scikit-learn=1.5.2=py311h57cc02b_1 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - semantic_version=2.10.0=pyhd8ed1ab_0 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_1 + - shapely=2.0.6=py311h2fdb869_2 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - snuggs=1.4.7=pyhd8ed1ab_1 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - specutils=1.19.0=pyhd8ed1ab_0 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - sqlite=3.47.0=h9eae976_1 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - suitesparse=7.8.3=hb42a789_1 + - svt-av1=2.3.0=h5888daf_0 + - swig=4.3.0=heed6a68_0 + - tbb=2022.0.0=hceb3a55_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - threadpoolctl=3.5.0=pyhc1e730c_0 + - tifffile=2024.9.20=pyhd8ed1ab_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - umap-learn=0.5.7=py311h38be061_0 + - uncertainties=3.2.2=pyhd8ed1ab_1 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - uriparser=0.9.8=hac33072_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcslib=8.2.2=ha708777_2 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - wrapt=1.17.0=py311h9ecbd09_0 + - x265=3.5=h924138e_3 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xerces-c=3.2.5=h988505b_2 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zarr=2.18.3=pyhd8ed1ab_0 + - zeromq=4.3.5=h3b0a872_7 + - zfp=1.0.1=h5888daf_2 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zlib-ng=2.2.2=h5888daf_0 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - alerce==1.2.0 + - anywidget==0.9.13 + - astroquery==0.4.8.dev9474 + - filelock==3.16.1 + - gdown==5.2.0 + - gitdb==4.0.11 + - gitpython==3.1.43 + - ipyaladin==0.5.2 + - jupyter-cpu-alive==0.1.2 + - jupyter-firefly-extensions==4.3.0 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - psygnal==0.11.1 + - smmap==5.0.1 + - specreduce==1.4.1 +prefix: /opt/conda/envs/notebook diff --git a/tractor/conda-notebook.yml b/tractor/conda-notebook.yml new file mode 100644 index 0000000..57a5912 --- /dev/null +++ b/tractor/conda-notebook.yml @@ -0,0 +1,51 @@ +name: notebook +channels: + - conda-forge + - nodefaults +dependencies: + - numpy + - pandas + - scipy + - pyarrow + - swig + - ipykernel + - cairo + - cython + - ceres-solver + - pkg-config + - setuptools + - wcslib + - libjpeg-turbo + - netpbm + - acstools + - pyvo + - boto3 + - firefly-client + - hpgeom + - lightkurve + - matplotlib + - seaborn + - mpld3 + - reproject + - s3fs + - scikit-learn + - scikit-image + - statsmodels + - astropy-healpix + - ccdproc + - photutils + - regions + - reproject + - specutils + - umap-learn + - statsmodels + - lsdb + - pip + - pip: + - specreduce + - astroquery>=0.4.8.dev0 + - alerce + - gdown + - ipyaladin + - jupyter_firefly_extensions +prefix: /opt/conda/envs/notebook \ No newline at end of file diff --git a/tractor/introduction.md b/tractor/introduction.md new file mode 100644 index 0000000..b69ba84 --- /dev/null +++ b/tractor/introduction.md @@ -0,0 +1,55 @@ +# +Fornax Science Console + +# Welcome to the Fornax Science Console! +--- + +# Documentation +The general documentation for using the Fornax Science Console are available in the +[documentation repository](fornax-documentation/README.md). These can also be browsed +in the [documentation page on github](https://nasa-fornax.github.io/fornax-demo-notebooks/#user-documentation). + +# Notebooks +The `notebooks` folder in the home directory contains notebooks that are actively being +developed. They currently include: +- [fornax-demo-notebooks](fornax-demo-notebooks/README.md): These are the main notbeooks developed + by the Fornax team. The rendered version is availale on the + [documentation page](https://nasa-fornax.github.io/fornax-demo-notebooks). +- [IVOA_2024_demo](IVOA_2024_demo/README.md): Notebooks developed in collaboration with the LINCC team, + demonstrating the use of Hispcat and LSDB for large catalog cross matching. +- Others will be added. + +The content of the `notebooks` folder in the home directory will be updated automatically +at the start of every new session. To disable these updates, add an empty file called +`.no-notebook-update.txt` in your home directory. + +--- +# Latest Changes +#### 11/26/2024 +- The primary conda environment is changed to `notebook`. It is the environment +where the notebooks should be run. With this change, the dask extension should +work naturally. +- Added the openvscode extension. +- Updates to prevent sessions with CPU activity from being stopped. The policy now is: + - If there is CPU activity, the notebook will not be stopped, even if the browser + is closed. + - If there is no activity (e.g. the notebook or browser tab is closed), + the session terminates after 15 min. +- The notebooks are updated automatically using `nbgitpuller` and they are +stored in the user's home directory. The update policy for `nbgitpuller`can be found +[here](https://nbgitpuller.readthedocs.io/en/latest/topic/automatic-merging.html#topic-automatic-merging). +The summary is: + - 1. A file that is changed in the remote repo but not in the local clone will be updated. + - 2. If different lines are changed by both the remote and local clone, the remote + changes will be merged similar to case 1. + - 3. If the same lines are changed by both the remote and local clone, the local + changes are kept and the remote changes are discarded. + - 4. If a file is deleted locally but still present in the remote repo, it will be restored. + - 5. If a new file is added in the locall clone, and the remote repo has a new file with + the same name, the local copy will be renamed by adding `_`, and the remote copy + will be used. +If the user has a file (it can be empty) called `.no-notebook-update.txt` in their home +directory, then `nbgitpuller` will not be used and the notebook folder in the home +directory will **not** be updated. +- Switched to using conda yaml files to keep track of the installed software. \ No newline at end of file diff --git a/tractor/update-notebooks.sh b/tractor/update-notebooks.sh new file mode 100644 index 0000000..e5512e9 --- /dev/null +++ b/tractor/update-notebooks.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set +e +timeout=10 + +notebook_repos=( + # main demo notebooks + https://github.com/nasa-fornax/fornax-demo-notebooks.git + # Documentation + https://github.com/nasa-fornax/fornax-documentation.git + # lsdb notbeooks + https://github.com/lincc-frameworks/IVOA_2024_demo.git +) + +if test -z $NOTEBOOK_DIR; then export NOTEBOOK_DIR=$HOME/notebooks; fi +mkdir -p $NOTEBOOK_DIR +cd $NOTEBOOK_DIR +# copy notebooks +echo "Cloning the notebooks to $notebook_dir ..." +for repo in ${notebook_repos[@]}; do + name=`echo $repo | sed 's#.*/\([^/]*\)\.git#\1#'` + if ! test -f $NOUPDATE; then + timeout $timeout python -m nbgitpuller.pull $repo main $name + fi +done +cd $HOME