work done towards github hosting

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: Run buildimages tests
+
+on:
+  push:
+    paths: ['src/**']
+  pull_request:
+    paths: ['src/**']
+
+jobs:
+
+  run-tests:
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Run builder tests
+        id: test
+        run: |
+           python3 src/tests.py

.github/workflows/merge.yml

@@ -0,0 +1,66 @@
+name: Maybe generate Docker images
+
+on: [ push ]
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+
+  build-and-push-images:
+
+    runs-on: ubuntu-latest
+
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in
+    # this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Debug Event Payload
+        env:
+          EVENT_PAYLOAD: ${{ toJson(github.event) }}
+        run: echo "$EVENT_PAYLOAD"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Git branch name
+        id: git-branch-name
+        run: echo "${{ github.head_ref || github.ref_name }}"
+
+      - uses: tchupp/actions-detect-directory-changes@v1
+        id: directory-changes
+        with:
+          included-paths: ""
+          included-extensions: ""
+          if-these-paths-change-return-all-included-paths: ""
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Maybe build and push Docker images to GHCR
+        id: push
+        env:
+          DIR_CHANGES: ${{ steps.directory-changes.outputs.changed }}
+        run: |
+           echo "$DIR_CHANGES"
+           export GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
+           export NEEDED="$(python src/neededimages.py "${{ github.repository }}" "$GHCR_TOKEN" "$DIR_CHANGES" "${{ github.head_ref || github.ref_name }}")"
+           echo "$NEEDED"
+           if [ ! -z "$NEEDED" ]; then
+             python src/buildimages.py "${{ github.repository }}" "${{ github.head_ref || github.ref_name }}" --push --images $NEEDED
+           fi

.github/workflows/release.yml

@@ -0,0 +1,46 @@
+name: Tag dev images as stable
+
+on:
+  release:
+    types: [published]
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+
+  build-and-push-images:
+
+    runs-on: ubuntu-latest
+
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in
+    # this job.
+    permissions:
+
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tag main images with release tags
+        id: push
+        run: |
+           python src/release.py "${{ github.repository }}" "${{ github.event.release.tag_name }}"  

base_image/Dockerfile

@@ -0,0 +1,144 @@
+ARG NOTEBOOK_TAG=2024-08-12
+FROM quay.io/jupyter/base-notebook:$NOTEBOOK_TAG as base
+FROM quay.io/jupyter/docker-stacks-foundation:$NOTEBOOK_TAG
+
+ARG IMAGE_TAG=latest
+LABEL gov.nasa.smce.fornax.jupyterhub.image="${IMAGE_TAG}"
+LABEL org.opencontainers.image.source=https://github.com/fornax-navo/fornax-images
+LABEL org.opencontainers.image.ref.name="Fornax Base Image"
+LABEL org.opencontainers.image.version=0.2
+LABEL maintainer="Fornax Project"
+
+# Bring some commands from jupyter/base-notebook
+# Skip the install of jupyterhub/lab etc
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV JUPYTER_PORT=8888
+EXPOSE $JUPYTER_PORT
+CMD ["start-notebook.py"]
+# Copy files from the image directly
+COPY --from=base /usr/local/bin/start-* /usr/local/bin/
+COPY --from=base /etc/jupyter/*py /etc/jupyter/
+USER root
+HEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \
+    CMD /etc/jupyter/docker_healthcheck.py || exit 1
+RUN fix-permissions /etc/jupyter/ /home/${NB_USER} /opt/
+USER $NB_USER
+WORKDIR $HOME
+# End of imports from jupyter/base-notebook #
+# ----------------------------------------- #
+
+
+ENV CONDA_ENV=notebook
+
+# Ask dask to read config from ${CONDA_DIR}/etc rather than
+# the default of /etc, since the non-root jovyan user can write
+# to ${CONDA_DIR}/etc but not to /etc
+ENV DASK_ROOT_CONFIG=${CONDA_DIR}/etc
+
+# COPY the current content to $HOME/build
+RUN mkdir -p $HOME/build/
+COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/
+COPY --chown=$NB_UID:$NB_GID overrides.json $HOME/build/
+
+USER root
+
+# Make /opt/ user writeable so it can be used by build-* scripts
+RUN fix-permissions /opt/
+
+# Install OS packages and then clean up
+COPY --chown=$NB_UID:$NB_GID scripts/*.sh /opt/scripts/
+# Read apt.txt line by line, and execute apt-get install for each line
+RUN cd build && bash /opt/scripts/apt-install.sh
+
+USER $NB_USER
+# setup conda environments
+RUN cd build && bash /opt/scripts/conda-env-install.sh \
+ # Change dispaly name of the default kernel
+ && mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV"
+
+# Any other build-* scripts #
+RUN cd $HOME/build \
+ ; for script in `ls build-*`; do \
+ echo "Found script ${script} ..." \
+ && chmod +x $script \
+ && ./$script \
+ ; done
+# --------------------------- #
+
+# cache location: use /tmp/ so we don't fill up $HOME
+ENV CACHE_DIR=/tmp/cache
+RUN mkdir -p $CACHE_DIR
+ENV XDG_CACHE_HOME=$CACHE_DIR
+ENV XDG_CACHE_DIR=$CACHE_DIR
+ENV XDG_CACHE_DIR=$CACHE_DIR
+
+# Set default config for pip and conda
+# change default pip cache from ~/.cache/pip
+ENV PIP_CACHE_DIR=$CACHE_DIR/pip
+RUN mkdir -p $PIP_CACHE_DIR
+
+COPY --chown=$NB_UID:$NB_GID condarc ${CONDA_DIR}/.condarc
+# ------------------------------------#
+
+# add a script for to run before the jupyter session
+USER root
+COPY --chmod=0755 pre-notebook.sh /usr/local/bin/before-notebook.d/20-pre-notebook.sh
+USER $NB_USER
+
+# Make $CONDA_ENV default; do it at the global level
+# because ~/.bashrc is not loaded when user space is mounted
+# Also, add it to before-notebook.d main script
+USER root
+ENV PATH=$CONDA_DIR/envs/$CONDA_ENV/bin:$PATH
+RUN cat $HOME/.bashrc >> /etc/bash.bashrc \
+ && printf "\nconda activate \$CONDA_ENV\n" >> /etc/bash.bashrc \
+ && printf "\nconda activate \$CONDA_ENV\n" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh \
+ && printf "" > $HOME/.bashrc
+USER $NB_USER
+
+# For vscode
+ENV CODE_EXECUTABLE=openvscode-server
+
+
+# For outside mount when using outside fornax
+RUN mkdir -p /opt/workspace
+VOLUME /opt/workspace
+
+# reset user and location
+RUN rm -r $HOME/build $HOME/work /tmp/*
+WORKDIR ${HOME}
+
+# Useful environment variables
+ENV NOTEBOOK_DIR=${HOME}/notebooks \
+    NOUPDATE=${HOME}/.no-notebook-update.txt
+
+# Install OS packages and then clean up
+ONBUILD RUN mkdir -p $HOME/build
+ONBUILD COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/
+ONBUILD USER root
+ONBUILD RUN mkdir -p build && cd build && bash /opt/scripts/apt-install.sh
+ONBUILD USER $NB_USER
+# ------------------------------------ #
+
+
+# setup conda environments
+ONBUILD RUN cd build && bash /opt/scripts/conda-env-install.sh
+# and ensure the correct display name
+ONBUILD RUN mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV"
+# ----------------------- #
+
+# Any other build-* scripts #
+ONBUILD RUN cd build \
+ ; for script in `ls build-*`; do \
+ echo "Found script ${script} ..." \
+ && chmod +x $script \
+ && ./$script \
+ ; done
+# --------------------------- #
+
+# landing page
+ONBUILD COPY --chown=$NB_UID:$NB_GID --chmod=644 introduction.md* /opt/scripts/
+
+ONBUILD RUN rm -r $HOME/build
+ONBUILD USER ${NB_USER}
+ONBUILD WORKDIR ${HOME}

base_image/README.md

@@ -0,0 +1,7 @@
+# Content
+
+
+# Versions
+Version number is in `.image_tags`
+
+- 0.1: A base image that starts from `jupyter/minimal-notebook:2023-04-17`

base_image/apt.txt

@@ -0,0 +1,14 @@
+fonts-liberation
+pandoc
+vim
+nano
+emacs-nox
+fuse
+bzip2
+git
+curl
+zip
+build-essential
+gcc
+make
+gfortran

base_image/build-config.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+env=notebook
+
+# jupyter overrides
+mkdir -p $CONDA_DIR/envs/$env/share/jupyter/lab/settings/
+mv ~/build/overrides.json $CONDA_DIR/envs/$env/share/jupyter/lab/settings/
+
+
+# disable the annoucement extension
+mamba run -n $env jupyter labextension disable "@jupyterlab/apputils-extension:announcements"