From 032ba064c630ee067ca411b805e47f40b49e4eca Mon Sep 17 00:00:00 2001 From: Greg Werner Date: Wed, 21 Aug 2024 07:08:15 -0400 Subject: [PATCH] Revert "fix: Update script to install packages (#25)" This reverts commit e8c3e5b32ee83a311762f36dff44a798d9577dd3. --- Makefile | 4 +- README.md | 21 ++-- python-notebook/requirements.txt | 18 ---- .../Dockerfile | 21 ++-- .../setup-scripts/install-julia-packages.bash | 77 +++++++++++++++ umich-notebook/setup-scripts/setup_julia.py | 97 +++++++++++++++++++ 6 files changed, 199 insertions(+), 39 deletions(-) delete mode 100755 python-notebook/requirements.txt rename {python-notebook => umich-notebook}/Dockerfile (51%) create mode 100644 umich-notebook/setup-scripts/install-julia-packages.bash create mode 100644 umich-notebook/setup-scripts/setup_julia.py diff --git a/Makefile b/Makefile index 3a0b59c..710e38e 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ HADOLINT="${HOME}/hadolint" help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html # http://github.com/jupyter/docker-stacks - @echo "illumidesk/illumidesk-stacks" + @echo "illumidesk/umich-stacks" @echo "=====================" - @echo "Replace % with a stack directory name (e.g., make build/python-notebook)" + @echo "Replace % with a stack directory name (e.g., make build/umich-notebook)" @echo @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 558cde7..03eddf7 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -[![Test Docker Image Status](https://github.com/illumidesk/illumidesk-stacks/workflows/Test%20and%20Push/badge.svg)](https://github.com/illumidesk/illumidesk-stacks/actions?query=branch%3Amain+workflow%3A%22Test+and+Push%22) +[![Test Docker Image Status](https://github.com/illumidesk/umich-stacks/workflows/Test%20and%20Push/badge.svg)](https://github.com/illumidesk/umich-stacks/actions?query=branch%3Amain+workflow%3A%22Test+and+Push%22) -# IllumiDesk Docker Stacks +# IllumiDesk Docker Stacks for the University of Michigan -This repo is used to manage IllumiDesk's standard docker images for the IllumiDesk learning environment to power Jupyter Server backends. +This repo is used to manage University of Michigan's docker images for the IllumiDesk learning environment integrated with the Canvas LMS. ## Pre Requisits @@ -28,7 +28,7 @@ make build-all Running the image standalone is helpful for testing: ```bash -docker run -p 8888:8888 illumidesk/python-notebook:latest +docker run -p 8888:8888 illumidesk/umich-notebook:latest ``` Then, navigate to `http://127.0.0.1:8888` to access your Jupyter Notebook server. @@ -37,24 +37,19 @@ Then, navigate to `http://127.0.0.1:8888` to access your Jupyter Notebook server ## Customize the Image -1. Add additional Python packages by editing the `./python-notebook/requirements.txt` file. +1. Add additional Julia packages by editing the `./umich-notebook/install-julia-packages.bash` file. 2. Rebuild end-user and grader images with `make build-all`. -3. Push images to AWS ECR - -- [You must first authenticate](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html) to push to AWS ECR repos using `docker` coomands. -- Run `docker push ` where `` represents the full path to your AWS ECR repository. - -1. (Optional) Push images to DockerHub +3. (Optional) Push images to DockerHub This step requires creating an Organization account in DockerHub or other docker image compatible registry. The `docker push ...` command will push the image to the DockerHub registry by default. Please refer to the official Docker documentation if you would like to push another registry. -For example, assuming the DockerHub organization is `illumidesk`, the source files are in the `python-notebook` folder, and the tag is `latest`, then the full namespace for the image would be `illumidesk/python-notebook:latest`. Assuming the image has been built, push the image to DockerHub or any other docker registry with the `docker push :` command: +For example, assuming the DockerHub organization is `illumidesk`, the source files are in the `umich-notebook` folder, and the tag is `latest`, then the full namespace for the image would be `illumidesk/umich-notebook:latest`. Assuming the image has been built, push the image to DockerHub or any other docker registry with the `docker push :` command: ```bash docker login -docker push illumidesk/python-notebook:latest +docker push illumidesk/umich-notebook:latest ``` ## Development and Testing diff --git a/python-notebook/requirements.txt b/python-notebook/requirements.txt deleted file mode 100755 index a051485..0000000 --- a/python-notebook/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -psycopg2-binary -gpt4all -jupyter_kernel_gateway -langchain -langchain.tools -langchain_openai -langchain_text_splitters -llmlingua -networkx -nltk -openai -pdfplumber -pymupdf -pypdf -pdf2image -poppler-utils -reportlab -requests \ No newline at end of file diff --git a/python-notebook/Dockerfile b/umich-notebook/Dockerfile similarity index 51% rename from python-notebook/Dockerfile rename to umich-notebook/Dockerfile index a260481..ad4fbde 100644 --- a/python-notebook/Dockerfile +++ b/umich-notebook/Dockerfile @@ -2,21 +2,30 @@ # https://github.com/jupyter/docker-stacks/blob/main/images/julia-notebook ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$REGISTRY/$OWNER/datascience-notebook -FROM ${BASE_CONTAINER} +ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook +FROM $BASE_CONTAINER # Fix: https://github.com/hadolint/hadolint/wiki/DL4006 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 SHELL ["/bin/bash", "-o", "pipefail", "-c"] +USER root + +# Julia dependencies +# install Julia packages in /opt/julia instead of ${HOME} +ENV JULIA_DEPOT_PATH=/opt/julia \ + JULIA_PKGDIR=/opt/julia + +# Setup Julia +RUN /opt/setup-scripts/setup_julia.py + USER ${NB_UID} # Setup IJulia kernel & other packages -COPY --chown=${NB_UID}:${NB_GID} requirements.txt "${HOME}/requirements.txt" -WORKDIR "${HOME}" -RUN pip install -r requirements.txt +RUN /opt/setup-scripts/setup-julia-packages.bash +RUN pip install jupyter_kernel_gateway psycopg2-binary WORKDIR "${HOME}" -CMD ["jupyter", "kernelgateway", "--KernelGatewayApp.ip=0.0.0.0", "--KernelGatewayApp.port=8888"] +CMD ["jupyter", "kernelgateway", "--KernelGatewayApp.ip=0.0.0.0", "--KernelGatewayApp.port=8888"] \ No newline at end of file diff --git a/umich-notebook/setup-scripts/install-julia-packages.bash b/umich-notebook/setup-scripts/install-julia-packages.bash new file mode 100644 index 0000000..e0ef68b --- /dev/null +++ b/umich-notebook/setup-scripts/install-julia-packages.bash @@ -0,0 +1,77 @@ +#!/bin/bash +set -exuo pipefail +# Requirements: +# - Run as a non-root user +# - The JULIA_PKGDIR environment variable is set +# - Julia is already set up, with the setup_julia.py command + +# replaces the default julia environment with the one we want +# ref: https://github.com/jupyter/docker-stacks/blob/main/images/minimal-notebook/setup-scripts/setup-julia-packages.bash +if [ "$(uname -m)" == "x86_64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +elif [ "$(uname -m)" == "aarch64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel" +fi + +# Install base Julia packages +julia -e ' +import Pkg; +Pkg.update(); +Pkg.add([ + "BenchmarkTools" + "Colors" + "CSV" + "CSVFiles" + "Compat" + "CoordinateTransformations" + "DataFrames" + "DelimitedFiles" + "DifferentialEquations" + "Distributions" + "FileIO" + "FiniteDiff" + "ForwardDiff" + "GeometryBasics" + "GMT" + "Images" + "ImageInTerminal" + "ImageFiltering" + "ImageFeatures" + "Interact" + "Interpolations" + "JLD2" + "LinearAlgebra" + "LaTeXStrings" + "Latexify" + "MeshCat" + "OSQP" + "Plots" + "Plotly" + "Printf" + "ProgressBars" + "PyPlot" + "Random" + "Rotations" + "Roots" + "SparseArrays" + "SymEngine" + "Symbolics" + "Statistics" + "StaticArrays" + "WebIO" + "WGLMakie" +]); +Pkg.precompile(); +' + +# Move the kernelspec out of ${HOME} to the system share location. +# Avoids problems with runtime UID change not taking effect properly +# on the .local folder in the jovyan home dir. +mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" +chmod -R go+rx "${CONDA_DIR}/share/jupyter" +rm -rf "${HOME}/.local" +fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" diff --git a/umich-notebook/setup-scripts/setup_julia.py b/umich-notebook/setup-scripts/setup_julia.py new file mode 100644 index 0000000..114e64c --- /dev/null +++ b/umich-notebook/setup-scripts/setup_julia.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import logging +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + +LOGGER = logging.getLogger(__name__) + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + LOGGER.info("Downloading Julia versions information") + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + # Compare versions semantically + latest_stable_version = max( + stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] + ) + latest_version_files = stable_versions[latest_stable_version]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + LOGGER.info(f"Latest version: {file_info['version']} url: {file_info['url']}") + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + LOGGER.info("Downloading and unpacking Julia") + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def configure_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + LOGGER.info("Configuring Julia") + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + configure_julia(julia_version=julia_version)