Merge pull request #498 from GoogleCloudPlatform/master

Rebase cl_sol on the current master
GoogleCloudPlatform · Jul 10, 2024 · 711c64c · 711c64c
2 parents 2450574 + bcb7085
commit 711c64c
Show file tree

Hide file tree

Showing 238 changed files with 9,115 additions and 1,932 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -9,7 +9,7 @@ ci:
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     -   id: check-ast
     -   id: check-case-conflict
@@ -38,19 +38,19 @@ repos:
     -   id: isort
         args: ["--profile", "black", "-l80"]
 -   repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
     -   id: black
         args: [-l 80]
     -   id: black-jupyter
         args: [-l 80]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.16.0
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
 -   repo: https://github.com/pycqa/pylint
-    rev: v3.1.0
+    rev: v3.2.4
     hooks:
     -   id: pylint
         args: ["--disable", "unexpected-keyword-arg,no-value-for-parameter,too-many-function-args"]
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -67,6 +67,18 @@ This repo's `.pre-commit-config.yaml` includes:
 
 ### FAQ
 
+#### _Local pre-commit results is different from the GitHub pre-commit.ci?_
+
+- Check the local pre-commit environment is up to date. If not, pull the latest repository and run `pre-commit install` at the repository root.
+
+- If not fixed yet, use the command below to reproduce the same environment as the pre-commit.ci and debug the issue.
+
+```
+docker run -v $PWD:/src:rw --workdir /src --rm \
+-ti ghcr.io/pre-commit-ci/runner-image \
+pre-commit install && pre-commit run --all-files
+```
+
 #### _How can I ask `pylint` to ignore something?_
 
 With comments like `# pylint: disable=[rule]`, see [the docs](https://pylint.pycqa.org/en/latest/user_guide/message-control.html).

diff --git a/Makefile b/Makefile
@@ -15,8 +15,6 @@
 all: clean install
 
 kernels: \
- reinforcement_learning_kernel \
- tf_recommenders_kernel \
  object_detection_kernel \
  pytorch_kfp_kernel \
  langchain_kernel
@@ -42,18 +40,14 @@ install:
 precommit:
 	@pre-commit run --all-files
 
+.PHONY: asl_kernel
+asl_kernel:
+	./kernels/asl_kernel.sh
+
 .PHONY: langchain_kernel
 langchain_kernel:
 	./kernels/langchain.sh
 
-.PHONY: reinforcement_learning_kernel
-reinforcement_learning_kernel:
-	./kernels/reinforcement_learning.sh
-
-.PHONY: tf_recommenders_kernel
-tf_recommenders_kernel:
-	./kernels/tf_recommenders.sh
-
 .PHONY: object_detection_kernel
 object_detection_kernel:
 	./kernels/object_detection.sh

diff --git a/README.md b/README.md
@@ -39,12 +39,11 @@ cd asl-ml-immersion
 ./scripts/setup_on_cloudshell.sh
 ```
 
-Second, follow the instruction of [the official documentation](https://cloud.google.com/vertex-ai/docs/workbench/user-managed/create-user-managed-notebooks-instance-console-quickstart) to set up a JupyterLab instance on [Vertex AI Workbench User Managed Notebooks](https://cloud.google.com/vertex-ai/docs/workbench/user-managed/introduction).
-The code in this repository is designed to run on [Vertex AI Workbench User Managed Notebooks](https://cloud.google.com/vertex-ai/docs/workbench/user-managed/introduction), and tested on the `Devian 11` -> `TensorFlow Enterprise 2.12` image.
+Second, follow the instruction of [the official documentation](https://cloud.google.com/vertex-ai/docs/workbench/instances/create-console-quickstart) to set up a JupyterLab instance on [Vertex AI Workbench Instance](https://cloud.google.com/vertex-ai/docs/workbench/instances/introduction).
 
 **Note:** Accelerators (GPU/TPU) are not required in most of the labs, but some notebooks recommend using them.
 
-After creating a Vertex Workbench User Managed Notebook instance, open the terminal *in your JupyterLab instance* and run the following commands:
+After creating a Vertex Workbench Instance, open the terminal *in your JupyterLab instance* and run the following commands:
 
 ```bash
 git clone https://github.com/GoogleCloudPlatform/asl-ml-immersion.git
@@ -55,6 +54,7 @@ make install
 
 **Note**: Some notebooks might require additional setup, please refer to the instructions in specific notebooks.
 
+After running these commands, you can open and execute a notebook on the base "Python 3" kernel.
 
 ## Contributions
 Currently, only Googlers can contribute to this repo. See [CONTRIBUTING.md](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/CONTRIBUTING.md) for more details on the contribution workflow.

diff --git a/kernels/langchain.sh b/kernels/langchain.sh
@@ -25,9 +25,13 @@ source $ENVNAME/bin/activate
 python -m ipykernel install --user --name=$ENVNAME
 
 pip install -q -U pip
-pip install langchain==0.0.217
-pip install wikipedia==1.4.0
+pip install langchain==0.0.323
+pip install langchain-google-vertexai==1.0.1
 pip install chromadb==0.3.26
-pip install google-cloud-aiplatform==1.26.1
+pip install pydantic==1.10.8
+pip install google-cloud-aiplatform==1.48.0
+pip install faiss-cpu==1.7.4
+pip install unstructured==0.14.4
+pip install wikipedia==1.4.0
 
 deactivate
diff --git a/kernels/object_detection.sh b/kernels/object_detection.sh
@@ -32,6 +32,8 @@ python -m ipykernel install --user --name=$ENVNAME
 
 # Install Object Detection API and its dependencies
 pip install -q -U pip
+pip install -q pyyaml==5.3.1
+pip install -q tensorflow==2.12.0 tensorflow_text==2.12.1 tf-models-official==2.12.0
 pip install -q .
 
 deactivate
diff --git a/kernels/reinforcement_learning.sh b/kernels/reinforcement_learning.sh
diff --git a/kernels/tf_recommenders.sh b/kernels/tf_recommenders.sh
diff --git a/notebooks/bigquery/labs/a_sample_explore_clean.ipynb b/notebooks/bigquery/labs/a_sample_explore_clean.ipynb
@@ -444,14 +444,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -463,7 +464,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/bigquery/labs/b_bqml.ipynb b/notebooks/bigquery/labs/b_bqml.ipynb
@@ -195,14 +195,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -214,7 +215,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/bigquery/labs/c_extract_and_benchmark.ipynb b/notebooks/bigquery/labs/c_extract_and_benchmark.ipynb
@@ -359,14 +359,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -378,7 +379,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/bigquery/solutions/a_sample_explore_clean.ipynb b/notebooks/bigquery/solutions/a_sample_explore_clean.ipynb
@@ -808,14 +808,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -827,7 +828,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/bigquery/solutions/b_bqml.ipynb b/notebooks/bigquery/solutions/b_bqml.ipynb
@@ -240,14 +240,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -259,7 +260,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/bigquery/solutions/c_extract_and_benchmark.ipynb b/notebooks/bigquery/solutions/c_extract_and_benchmark.ipynb
@@ -332,14 +332,15 @@
  ],
  "metadata": {
   "environment": {
-   "name": "tf2-gpu.2-5.m76",
+   "kernel": "conda-base-py",
+   "name": "workbench-notebooks.m121",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m76"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel) (Local)",
    "language": "python",
-   "name": "python3"
+   "name": "conda-base-py"
   },
   "language_info": {
    "codemirror_mode": {
@@ -351,7 +352,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,