From 012c7299ef7aea8848830ec0c1809b6f286076bc Mon Sep 17 00:00:00 2001 From: Eli Holmes Date: Wed, 3 Apr 2024 12:42:59 -0700 Subject: [PATCH] fix link to jh-gcp --- _quarto.yml | 2 +- ci/arcgis/test.json | 22 ++++ ci/arcgis/test.txt | 1 + ci/arcgis/test2.txt | 1 + ci/py-rocket-gis/Dockerfile | 9 -- ci/py-rocket-gis/environment.yml | 82 ------------- ci/py-rocket-gis/instructions.md | 156 ------------------------- ci/py-rocket/Dockerfile | 2 +- docs/posts/JHub-User-Guide.html | 2 +- docs/posts/Set-up-centos-security.html | 2 +- docs/posts/Set-up-centos-tljh.html | 2 +- docs/posts/Set-up-centos.html | 6 +- docs/posts/Set-up-daskhub.html | 6 +- docs/posts/Setup-Notes.html | 2 +- docs/posts/set-up-authentication.html | 2 +- docs/posts/set-up-jh-gcp.html | 18 ++- docs/posts/set-up-vm.html | 2 +- docs/search.json | 55 +++++++-- 18 files changed, 98 insertions(+), 274 deletions(-) create mode 100644 ci/arcgis/test.json create mode 100644 ci/arcgis/test.txt create mode 100644 ci/arcgis/test2.txt delete mode 100644 ci/py-rocket-gis/Dockerfile delete mode 100644 ci/py-rocket-gis/environment.yml delete mode 100644 ci/py-rocket-gis/instructions.md diff --git a/_quarto.yml b/_quarto.yml index 4e6c888..b9df185 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -32,7 +32,7 @@ website: - text: "Set-up JHub on Azure" href: posts/Set-up-daskhub.Rmd - text: "Set-up JHub on Google Cloud" - href: posts/Set-up-jh-gcp.html + href: posts/set-up-jh-gcp.Rmd - text: "Set-up JHub on Centos" href: posts/Set-up-centos.Rmd - text: "Set-up https on Centos" diff --git a/ci/arcgis/test.json b/ci/arcgis/test.json new file mode 100644 index 0000000..e723f6c --- /dev/null +++ b/ci/arcgis/test.json @@ -0,0 +1,22 @@ +{ +"created": "2024-02-29T22:52:17.647410+00:00", +"modified": "2024-02-29T22:52:17.932383+00:00", +"id": 10729731, +"conceptrecid": "10719406", +"doi": "10.5281/zenodo.10729731", +"conceptdoi": "10.5281/zenodo.10719406", +"doi_url": "https://doi.org/10.5281/zenodo.10729731", +"metadata": { + "title": "Zenodo Release Action", + "doi": "10.5281/zenodo.10729731", + "publication_date": "2024-02-29", + "description": "This is a test", + "access_right": "open", + "creators": [{"name": "test", "affiliation": "test"}], + "keywords": ["test"], + "related_identifiers": [{"identifier": "https://github.com/Rgovys/test-integration/tree/test2", "relation": "isSupplementTo", "resource_type": "software", "scheme": "url"}], + "version": "test19", + "resource_type": {"title": "Software", "type": "software"}, + "license": {"id": "mit-license"}, + "relations": {"version": [{"index": 11, "is_last": true, "parent": {"pid_type": "recid", "pid_value": "10719406"}}]} +}, "title": "Zenodo Release Action", "links": {"self": "https://zenodo.org/api/records/10729731", "self_html": "https://zenodo.org/records/10729731", "self_doi": "https://zenodo.org/doi/10.5281/zenodo.10729731", "doi": "https://doi.org/10.5281/zenodo.10729731", "parent": "https://zenodo.org/api/records/10719406", "parent_html": "https://zenodo.org/records/10719406", "parent_doi": "https://zenodo.org/doi/10.5281/zenodo.10719406", "self_iiif_manifest": "https://zenodo.org/api/iiif/record:10729731/manifest", "self_iiif_sequence": "https://zenodo.org/api/iiif/record:10729731/sequence/default", "files": "https://zenodo.org/api/records/10729731/files", "media_files": "https://zenodo.org/api/records/10729731/media-files", "archive": "https://zenodo.org/api/records/10729731/files-archive", "archive_media": "https://zenodo.org/api/records/10729731/media-files-archive", "latest": "https://zenodo.org/api/records/10729731/versions/latest", "latest_html": "https://zenodo.org/records/10729731/latest", "draft": "https://zenodo.org/api/records/10729731/draft", "versions": "https://zenodo.org/api/records/10729731/versions", "access_links": "https://zenodo.org/api/records/10729731/access/links", "access_users": "https://zenodo.org/api/records/10729731/access/users", "access_request": "https://zenodo.org/api/records/10729731/access/request", "access": "https://zenodo.org/api/records/10729731/access", "reserve_doi": "https://zenodo.org/api/records/10729731/draft/pids/doi", "communities": "https://zenodo.org/api/records/10729731/communities", "communities-suggestions": "https://zenodo.org/api/records/10729731/communities-suggestions", "requests": "https://zenodo.org/api/records/10729731/requests"}, "updated": "2024-02-29T22:52:17.932383+00:00", "recid": "10729731", "revision": 4, "files": [{"id": "54fd2ae5-bc67-4d81-9b7d-06937230b945", "key": "test19.zip", "size": 1738, "checksum": "md5:b33eedda7a38a54d422ed61c2a8ec09a", "links": {"self": "https://zenodo.org/api/records/10729731/files/test19.zip/content"}}], "owners": [{"id": 1075630}], "status": "published", "stats": {"downloads": 5, "unique_downloads": 4, "views": 44, "unique_views": 24, "version_downloads": 0, "version_unique_downloads": 0, "version_unique_views": 1, "version_views": 1}, "state": "done", "submitted": true} \ No newline at end of file diff --git a/ci/arcgis/test.txt b/ci/arcgis/test.txt new file mode 100644 index 0000000..327c8d3 --- /dev/null +++ b/ci/arcgis/test.txt @@ -0,0 +1 @@ +"a `code` b" diff --git a/ci/arcgis/test2.txt b/ci/arcgis/test2.txt new file mode 100644 index 0000000..a07bf07 --- /dev/null +++ b/ci/arcgis/test2.txt @@ -0,0 +1 @@ +a\n\nb\n \ No newline at end of file diff --git a/ci/py-rocket-gis/Dockerfile b/ci/py-rocket-gis/Dockerfile deleted file mode 100644 index cf76bb9..0000000 --- a/ci/py-rocket-gis/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# This is a base geospatial R and Python image -FROM openscapes/python:6ee57a9 - -USER root -RUN apt-get update -RUN apt-get install -y libkrb5-dev -RUN conda install -c esri arcgis -USER ${NB_USER} - diff --git a/ci/py-rocket-gis/environment.yml b/ci/py-rocket-gis/environment.yml deleted file mode 100644 index b28dcb8..0000000 --- a/ci/py-rocket-gis/environment.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: openscapes -channels: - - conda-forge -dependencies: - - python=3.10 - - dask - - dask-gateway - - distributed - - rioxarray - - xarray>=2022.12 - - h5netcdf - - netcdf4>=1.6.4 - - h5py - - gdal~=3.8 - - geoviews - - holoviews=1.16.2 - - matplotlib-base - - seaborn - - intake - - hvplot - - datashader - - pyproj - - bqplot - - geopandas - - zarr - - cartopy - - pynco - - rasterio - - shapely - - pyresample - - icepyx - - joblib - - pystac-client - - pydap - - lxml - - ipyleaflet - # when jlab 4 gets released we can upgrade dask-labextension - # - dask-labextension - # - dask-labextension==6.2.0 - - dask-labextension - - jupyter-server-proxy - - ipywidgets - - jupyter-book - - jupyterlab>4 - - jupyterlab-myst - - jupyterhub-singleuser - - jupyterlab-geojson - - jupyterlab-favorites - - jupyterlab-git - - jupyter-resource-usage - - jupyterlab-h5web - - hdf5plugin - - ipympl - - conda-lock - - pooch - - earthaccess>=0.8.2 - - sliderule>=3.6.0 - # QGIS - - qgis~=3.28 - - pyopencl - - ocl-icd-system - - websockify>=0.10 - # /QGIS - - h5coro - - git >=2.39 - - itslive - - rasterstats - - kerchunk - - rechunker - - pqdm - - spectral - - scikit-image - - coiled - - s3fs - - nbgitpuller - - awscliv2 - - jupyter-server-proxy - - jupyter-vscode-proxy - - code-server >=3.2 - - pip -platforms: - - linux-64 \ No newline at end of file diff --git a/ci/py-rocket-gis/instructions.md b/ci/py-rocket-gis/instructions.md deleted file mode 100644 index ca23f9c..0000000 --- a/ci/py-rocket-gis/instructions.md +++ /dev/null @@ -1,156 +0,0 @@ -# This is based on the openscapes/py-rocket - - - -THIS WAS TESTING. NOT USED. - -# tldr; - -``` -cd ci/py-rocket-gis -DOCKER_TAG="20240129" -docker build --platform linux/amd64 -t eeholmes/py-rocket-gis:${DOCKER_TAG} . -docker push eeholmes/py-rocket-gis:${DOCKER_TAG} -``` - -Log into Azure portal, go to DaskHub, Connect, Cloud Shell, and edit `dconfig2.yaml` (`nano dconfig2.yaml`) to update the `py-rocket` tag. Then run this command. - -``` -helm upgrade --cleanup-on-fail --render-subchart-notes dhub dask/daskhub --namespace dhub --version=2023.1.0 --values dconfig2.yaml -``` - -Tip: if things fill up use -``` -docker system prune --all -``` - -# Requirements - -Docker installed. For example, if doing on a Mac or PC, you need Docker Desktop. On VMs, docker will already be installed. - -A DockerHub user account. The instructions are using EEH's. - -# Add new packages - -Add to Dockerfile something like -``` -RUN R -e 'install.packages("gtools", repos = "http://cran.us.r-project.org")' -``` - -# Rebuild and push the Docker image - -1. Make sure Docker app is running, not just installed. So if you are on a local computer, start up the app (open it). -1. Go to a terminal and cd to the directory with the Dockerfile. So to the `ci` directory in the `nmfs-jhub` repo. -``` -cd ci/py-rocket -``` -2. Update the docker tag to the date. -``` -DOCKER_TAG="20230901" -``` -2. Build the image. `.` means current directory. `eeholmes/py-rocket` is the name of the repo on DockerHub. `--platform` is added if you are building on an Mac with Apple chip. -``` -docker build --platform linux/amd64 -t eeholmes/py-rocket:${DOCKER_TAG} -t eeholmes/py-rocket:main . -``` - -3. Push the image up to DockerHub. Make sure you are logged into DockerHub in the Docker app otherwise you'll get "access denied". Open the Docker app and look that it shows that you are signed in. -``` -docker push eeholmes/py-rocket:${DOCKER_TAG} -docker push eeholmes/py-rocket:main -``` - -Notes: https://help.valohai.com/hc/en-us/articles/4421364087569-Build-your-own-Docker-image - - -# Stop any running the Jupyter Lab instances - -Log in. File > Hub Control > Stop my server - -# Run `helm upgrade` - -Log into Azure portal, go to DaskHub, Connect, Cloud Shell, and run this command. Note, this assumes that `eeholmes/iorocker:main` is still the image to use. If not, edit `dconfig2.yaml` (`nano dconfig2.yaml`) and then upgrade. - -``` -helm upgrade --cleanup-on-fail --render-subchart-notes dhub dask/daskhub --namespace dhub --version=2023.1.0 --values dconfig2.yaml -``` - - - - -## If an specific image tag is in config - -The JupyterHub has a config file that specifies what images are being used. If the image is say `eeholmes/iopython:hublatest`, then whenever the a image with tag `hublatest` is pushed, the hub will use that. If on the otherhand, you config file has a specific, an unique tag that you don't overwrite, then you'll have to update the file in the config file on the cluster (log into Azure, go to cluster, connect to cloud shell, `nano dconfig2.yaml`) and upgrade the installation of the JupyterHub. - -Why not `eeholmes/iopython:latest`? There is nothing special about `latest`. It is the default tag used if you don't specify `-t` and `:` in your build call. So it is a bit too easy to accidentally update "latest" and thus update the image for you hub when you didn't intend to do that. You just forgot to specify a tag. - -To update if you are using a specific tag, like `20230615` rather than one you keep updating like `hublatest` or `latest`: - -### Step 1 - -Edit the config file. Mine is called `dconfig2.yaml`. Yours is probably `config.yaml`. Name is unimportant. - -``` -nano dconfig2.yaml -``` - -Inside `dconfig2.yaml` is this info. This shows a fixed tag. So if I update, I need to change the `20230615` part. -``` - singleuser: - image: - name: eeholmes/iopython - tag: 20230615 -``` -Save the changes. In nano, it cmd-O, return, cmd-X. - -### run `helm upgrade` - -``` -helm upgrade --cleanup-on-fail --render-subchart-notes dhub dask/daskhub --namespace dhub --version=2023.1.0 --values dconfig2.yaml -``` - -### The helm upgrade command - -``` -helm upgrade --cleanup-on-fail --render-subchart-notes dhub dask/daskhub --namespace dhub --version=2023.1.0 --values dconfig2.yaml -``` - -A helm is what runs the commands to upgrade (and install in the beginning) our JupyterHub. `dask/daskhub` is point to the repo with the "helm chart" (the instructions). `--value dconfig2.yaml` is telling it where the config file is. - -* `upgrade` upgrade an existing installation with the values in `dconfig2.yaml` -* `--render-subchart-notes` the dask/daskhub helm chart has subcharts (jupyterhub) and you need to render these too. Not all helm charts have this. -* `dask/daskhub` the name of the repo that has the helm chart. The first time you reference this, you need to tell help about the repo by giving it the url. [Read how here](https://blog.dask.org/2020/08/31/helm_daskhub) -* `--version=2023.1.0` version of the helm chart. Update when the helm chart (instructions for installing the jupyterhub) changes. - -## Adding packages with newpackages.yml - -When the openscapes image is used, we are in a conda env called 'notebook'. We want to -update that with the packages in newpackages.yml but need to get that file into the container. For now, -I just hard code in the pip install commands. - -Add to Docker file -``` -# it can't find new.yml in home/joyvan/.kernels -# need to get that into the container somehow (git clone?) -# RUN conda env update --file new.yml - -# Asides - -## To set docker tag to latest commit - -``` -SHA7="$(git rev-parse --short HEAD)" -DOCKER_TAG=$SHA7 -``` -I am not doing that since this repo has lots of commits unrelated to the docker image. - -## To set up your own Docker repo - -1. Make an account on DockerHub. Free is fine. -2. Create a repo and give it a name. For example, for this project, my account is `eeholmes` and my repo is `iopython` (Indian Ocean Python) as it is specific a particular project I am working on. - -DockerHub will want to you to buy the premium account but you only need that if you are doing continuous integration, like using a GitHub Action to autobuild your image. If you are manually building, you don't need this. - -## Why is `--platform` needed in the build command - -You won't see this on docker build tutorials. But if you are on a Mac with Apple chip, then you'll build arm64 images and that's not going to work on Ubuntu VMs. The vanilla images you see are amd64 so we want to make sure we are building for that platform. This only matters if you are on a Mac with Apple chip, but it won't break things for unix and PC so I added to make the instructions more robust. -``` \ No newline at end of file diff --git a/ci/py-rocket/Dockerfile b/ci/py-rocket/Dockerfile index aea1f3b..c498641 100644 --- a/ci/py-rocket/Dockerfile +++ b/ci/py-rocket/Dockerfile @@ -1,4 +1,4 @@ -# This is a base geospatial R and Python image +# This is a base tidyverse R and Python image FROM rocker/geospatial:dev-osgeo ENV CONDA_ENV=notebook \ diff --git a/docs/posts/JHub-User-Guide.html b/docs/posts/JHub-User-Guide.html index 89087b3..6b81a59 100644 --- a/docs/posts/JHub-User-Guide.html +++ b/docs/posts/JHub-User-Guide.html @@ -132,7 +132,7 @@
  • - + Set-up JHub on Google Cloud
    • diff --git a/docs/posts/Set-up-centos-security.html b/docs/posts/Set-up-centos-security.html index 72d845b..6cb7565 100644 --- a/docs/posts/Set-up-centos-security.html +++ b/docs/posts/Set-up-centos-security.html @@ -166,7 +166,7 @@
  • - + Set-up JHub on Google Cloud
    • diff --git a/docs/posts/Set-up-centos-tljh.html b/docs/posts/Set-up-centos-tljh.html index b99b3de..d97a4c6 100644 --- a/docs/posts/Set-up-centos-tljh.html +++ b/docs/posts/Set-up-centos-tljh.html @@ -164,7 +164,7 @@
  • - + Set-up JHub on Google Cloud
    • diff --git a/docs/posts/Set-up-centos.html b/docs/posts/Set-up-centos.html index 6249352..e1dda0d 100644 --- a/docs/posts/Set-up-centos.html +++ b/docs/posts/Set-up-centos.html @@ -65,7 +65,7 @@ - + @@ -166,7 +166,7 @@
  • - + Set-up JHub on Google Cloud
    • @@ -1396,7 +1396,7 @@

    Summary