chore: update jupyter base

.github/workflows/build_and_push_to_docker.yml

@@ -24,8 +24,15 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/py --tag $DOCKER_NAME-py:3.7-$LABEL
-        docker push $DOCKER_NAME-py:3.7-$LABEL
+        docker build docker/py --tag $DOCKER_NAME-py:3.8-$LABEL
+        docker push $DOCKER_NAME-py:3.8-$LABEL
+
+        # on master push latest image
+        if [ "$REF" == "refs/heads/master" ]
+        then
+          docker tag $DOCKER_NAME-py:3.8-$LABEL $DOCKER_NAME-py:latest
+          docker push $DOCKER_NAME-py:latest
+        fi
       env:
         DOCKER_NAME: "renku/renkulab"
 
@@ -38,7 +45,7 @@ jobs:
       matrix:
         EXTENSIONS:
           - cuda9.2
-          - cuda10.0-tf1.14
+          - cuda-tf
 
     steps:
     - name: Docker Login
@@ -57,7 +64,7 @@ jobs:
         fi
 
         docker build docker/$EXTENSION \
-          --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.7-$LABEL" \
+          --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.8-$LABEL" \
           --tag $DOCKER_NAME-$EXTENSION:$LABEL
         docker push $DOCKER_NAME-$EXTENSION:$LABEL
       env:
@@ -107,7 +114,8 @@ jobs:
       fail-fast: true
       matrix:
         RVERSIONS:
-          - 4.0.0
+          - 4.0.3
+          - 4.0.4
     steps:
     - name: Docker Login
       uses: Azure/docker-login@v1
@@ -126,11 +134,17 @@ jobs:
 
         export DOCKER_TAG="$RVERSION-$LABEL"
         docker build docker/r \
-          --build-arg RENKU_BASE="$DOCKER_NAME-py:3.7-$LABEL" \
-          --build-arg BASE_IMAGE="rocker/verse:${RVERSION}-ubuntu18.04" \
+          --build-arg RENKU_BASE="$DOCKER_NAME-py:3.8-$LABEL" \
+          --build-arg BASE_IMAGE="rocker/verse:${RVERSION}" \
           --tag $DOCKER_NAME-r:$DOCKER_TAG
         docker push $DOCKER_NAME-r:$DOCKER_TAG
 
+        # on master push latest image
+        if [ "$REF" == "refs/heads/master" ] && [ "$RVERSION" == "4.0.4" ]
+        then
+          docker tag $DOCKER_NAME-r:$DOCKER_TAG $DOCKER_NAME-r:latest
+          docker push $DOCKER_NAME-r:latest
+        fi
       env:
         DOCKER_NAME: "renku/renkulab"
         # selected R versions available for rocker/rstudio
@@ -141,7 +155,7 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         RELEASES:
           - devel
@@ -165,11 +179,17 @@ jobs:
 
         export DOCKER_TAG="$RELEASE-$LABEL"
         docker build docker/r \
-          --build-arg RENKU_BASE="$DOCKER_NAME-py:3.7-$LABEL" \
+          --build-arg RENKU_BASE="$DOCKER_NAME-py:3.8-$LABEL" \
           --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${RELEASE} \
           --tag $DOCKER_NAME-bioc:$DOCKER_TAG
         docker push $DOCKER_NAME-bioc:$DOCKER_TAG
 
+        # on master push latest image
+        if [ "$REF" == "refs/heads/master" ] && [ "$RELEASE" == "devel" ]
+        then
+          docker tag $DOCKER_NAME-bioc:$DOCKER_TAG $DOCKER_NAME-bioc:latest
+          docker push $DOCKER_NAME-bioc:latest
+        fi
       env:
         DOCKER_NAME: "renku/renkulab"
         RELEASE: ${{ matrix.RELEASES }}

Makefile

@@ -20,7 +20,7 @@ extensions = \
 	bioc \
 	r \
 	cuda-9.2 \
-	cuda-10.0-tf-1.14
+	cuda-10.0-tf
 
 DOCKER_PREFIX?=renku/renkulab
 DOCKER_LABEL?=latest
@@ -33,10 +33,11 @@ ifdef RENKU_VERSION
 	RENKU_TAG=-renku$(RENKU_VERSION)
 endif
 
-RVERSION?=3.6.3
+RVERSION?=4.0.4
 BIOC_VERSION?=devel
 R_TAG=-r$(RVERSION)
-BIOC_TAG=-bioc$(BIOC_VERSION)
+BIOC_TAG=$(BIOC_VERSION)
+TENSORFLOW_VERSION?=2.2.0
 
 .PHONY: all
 
@@ -50,8 +51,8 @@ push:
 		if test "$$ext" != "" ; then \
 			ext=-$$ext; \
 		fi; \
-		docker push $(DOCKER_PREFIX)$$ext:$(DOCKER_LABEL)$(RENKU_TAG) ; \
-		docker push $(DOCKER_PREFIX)$$ext:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) ; \
+		docker push $(DOCKER_PREFIX)$$ext:$(DOCKER_LABEL) ; \
+		docker push $(DOCKER_PREFIX)$$ext:$(GIT_MASTER_HEAD_SHA) ; \
 	done
 
 pull:
@@ -64,14 +65,14 @@ pull:
 
 r: py
 	docker build docker/r \
-		--build-arg RENKU_BASE=renku/renkulab-py3.7:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
+		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RVERSION=$(RVERSION) \
 		-t $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) && \
 	docker tag $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) $(DOCKER_PREFIX)-r:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)$(R_TAG)
 
 bioc: py
 	docker build docker/bioc \
-		--build-arg RENKU_BASE=renku/renkulab-py3.7:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
+		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RELEASE=$(BIOC_VERSION) \
 		-t $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)$(RENKU_TAG)$(BIOC_TAG) && \
 	docker tag $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)$(RENKU_TAG)$(BIOC_TAG) $(DOCKER_PREFIX)-bioc:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)$(BIOC_TAG)
@@ -82,3 +83,11 @@ py:
 	docker build \
 		-t $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) . && \
 	docker tag $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-$@:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)
+
+cuda: py
+	docker build docker/cuda-tf \
+		--build-arg RENKU_PIP_SPEC=$(RENKU_PIP_SPEC) \
+		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA) \
+		--build-arg TENSORFLOW_VERSION=$(TENSORFLOW_VERSION) \
+		-t $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) && \
+	docker tag $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) $(DOCKER_PREFIX)-cuda-tf:$(GIT_MASTER_HEAD_SHA)

README.md

@@ -54,15 +54,15 @@ the image that's right for you:
 `renku/renkulab-[image flavor]:[image flavor version]-[tag|hash]`
 
 For example:
-`renku/renkulab-py:3.7-0.6.4`
+`renku/renkulab-py:3.8-0.7.0`
 
 * `renku/renkulab`: indicates this is an image you can use to spawn an environment
   from your project on RenkuLab.
 * `-py`: indicates this is a python image flavor; either the programming language
   installed in the environment, or the base image that extra dependencies are added to.
   See below for details about the available flavors.
-* `3.7`: indicates the version of python is 3.7
-* `0.6.4` (or `d572e9a`): the tag is a value given to a commit of the repository
+* `3.8`: indicates the version of python is 3.7
+* `0.7.0` (or `d572e9a`): the tag is a value given to a commit of the repository
   and indicates that the version is part of a release. If the version is not part of
   a release, this value is the first few chars of the git commit SHA from which the
   image is built.

docker/cuda-tf/Dockerfile

@@ -0,0 +1,98 @@
+ARG BASE_IMAGE=renku/renkulab-py:latest
+FROM $BASE_IMAGE
+
+LABEL maintainer="Swiss Data Science Center <[email protected]>"
+
+USER root
+# Install the CUDA "base"
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gnupg2 curl ca-certificates && \
+    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub | apt-key add - && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \
+    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list && \
+    apt-get purge --autoremove -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CUDA_VERSION 11.0.3
+
+# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    cuda-cudart-11-0=11.0.221-1 \
+    cuda-compat-11-0 \
+    && ln -s cuda-11.0 /usr/local/cuda && \
+    rm -rf /var/lib/apt/lists/*
+
+# Required for nvidia-docker v1
+RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
+    && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
+
+ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+
+# nvidia-container-runtime
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=11.0 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441 brand=tesla,driver>=450,driver<451"
+
+#
+# Install TensorFlow (from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile)
+#
+
+ARG CUDA=11.0
+ARG CUDNN=8.0.5.39
+ARG CUDNN_MAJOR_VERSION=8
+ARG LIB_DIR_PREFIX=x86_64
+
+# Needed for string substitution
+SHELL ["/bin/bash", "-c"]
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    cuda-command-line-tools-${CUDA/./-} \
+    libcublas-${CUDA/./-} \
+    cuda-nvrtc-${CUDA/./-} \
+    libcufft-${CUDA/./-} \
+    libcurand-${CUDA/./-} \
+    libcusolver-${CUDA/./-} \
+    libcusparse-${CUDA/./-} \
+    curl \
+    libcudnn8=${CUDNN}-1+cuda${CUDA} \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libzmq3-dev \
+    pkg-config \
+    software-properties-common \
+    unzip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+
+# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure
+# dynamic linker run-time bindings
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
+    && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
+    && ldconfig
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which python3) /usr/local/bin/python
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
+USER $NB_USER
+ARG TF_PACKAGE=tensorflow-gpu
+ARG TF_PACKAGE_VERSION=2.4
+RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
+
+# Tensorboard jupyter(lab) extension
+# RUN jupyter labextension install jupyterlab_tensorboard && \
+#     pip install jupyter-tensorboard==0.2.0

docker/py/Dockerfile

@@ -1,5 +1,5 @@
 # Run with the RenkuKubeSpawner via JupyterHub
-ARG BASE_IMAGE=jupyter/base-notebook:8fd7ada2a374
+ARG BASE_IMAGE=jupyter/base-notebook:aec555e49be6
 FROM $BASE_IMAGE as base
 
 LABEL maintainer="Swiss Data Science Center <[email protected]>"
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
     curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
     apt-get update && apt-get install -yq --no-install-recommends \
     git \
-    git-lfs=2.8.0 \
+    git-lfs=2.12.0 \
     gnupg \
     graphviz \
     less \

docker/py/requirements.txt

@@ -1,10 +1,10 @@
-jupyterhub==1.1.0
-jupyterlab-git==0.20.0
-jupyterlab_server~=1.0.0
-jupyter-server-proxy==1.5.0
-nbresuse==0.3.6
+jupyterhub==1.2.2
+jupyterlab-git==0.23.3
+jupyterlab_server<2.0
+jupyter-server-proxy==1.6.0
+nbresuse==0.4.0
 nbclient<0.5.1
-papermill==2.1.3
+papermill==2.3.1
 pipx>=0.15.0.0
 powerline-shell==0.7.0
 requests>=2.20.0

docker/r/Dockerfile

@@ -1,6 +1,6 @@
 # define build arguments
 ARG RENKU_BASE=renku/renkulab-py:latest
-ARG BASE_IMAGE=rocker/verse:4.0.0-ubuntu18.04
+ARG BASE_IMAGE=rocker/verse:4.0.4
 
 # define base images
 FROM $RENKU_BASE as renku_base