add dev workflow notebook

CONTRIBUTING.md

@@ -23,8 +23,7 @@ This can be done by the following steps:
 This will create a `requirements.txt` file in the repository that contains the dependencies
 for the `finetuning` and `evaluation` job groups.
 2. In your Ray runtime environment, specify the following:
-    - `py_modules`: Local path to the `flamingo` module folder 
-    (located at `src/flamingo` in the workspace).
+    - `py_modules`: Local path to the `flamingo` module folder (located at `src/flamingo` in the workspace).
     - `pip`: Local path to the `requirements.txt` file generated above.
 3. Submit your job with an entrypoint command that invokes `flamingo` directly as a module,
 e.g., `python -m flamingo run finetuning --config cofig.yaml`.

README.md

@@ -19,13 +19,13 @@ poetry lock
 poetry install
 ```
 
-This will install the package in editable mode along with all of its optional dependency groups.
+This will install an editable version of the package along with all of its dependency groups.
 Poetry should recognize your active virtual environment during installation
 and install the package dependencies there.
 
 The `pyproject.toml` file defines dependency groups for the logical job types in the package.
-Individual package dependency groups can be installed by running 
-`poetry install --with <group1> <group2>` or `poetry install --only <group>`.
+Individual dependency groups can be installed by running 
+`poetry install --with <group1>,<group2>` or `poetry install --only <group>`.
 
 See the [contributing](CONTRIBUTING.md) guide for more information on development workflows.
 

examples/dev_workflow.ipynb

@@ -1,12 +1,95 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "123e34e9-70f8-42ab-b790-b59ddc01b1f3",
+   "metadata": {},
+   "source": [
+    "# Development Workflow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8c0f15ed-77dc-44ce-adb6-d1b59368f03c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Required imports (after flamingo is installed in your environment)\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "from ray.job_submission import JobSubmissionClient\n",
+    "\n",
+    "import flamingo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "969884e5-d815-42d9-9d4e-3b8f890657e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a submission client bound to a Ray cluster\n",
+    "# Note: You will likely have to update the cluster address shown below\n",
+    "client = JobSubmissionClient(f\"http://10.146.174.91:8265\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3258bb97-d3c6-4fee-aa0c-962c1411eaa7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "PosixPath('/Users/sfriedowitz/workspace/flamingo/src/flamingo')"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Determine local module path for the flamingo repo\n",
+    "flamingo_module = Path(flamingo.__file__).parent\n",
+    "flamingo_module"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b81b36be-35ce-4398-a6d4-ac1f719f5c95",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Construct the runtime environment for your job submission\n",
+    "# py_modules contains the path to the local flamingo module directory\n",
+    "# pip contains an export of the dependencies for the flamingo package (see CONTRIBUTING.md)\n",
+    "runtime_env = {\n",
+    "    \"working_dir\": \"/path/to/directory/with/finetunning_config.yaml\",\n",
+    "    \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]}, # If running a job that uses W&B\n",
+    "    \"py_modules\": [str(flamingo_module)],\n",
+    "    \"pip\": \"/path/to/flamingo/requirements.txt\"\n",
+    "}"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "ba73d5e4-640a-4aff-9ed5-17733e85c5b2",
+   "id": "4bd300f9-b863-4413-bd3a-430601656816",
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "# Submit the job to the Ray cluster\n",
+    "# Note: flamingo is invoked by 'python -m flamingo' since the CLI is not installed in the environment\n",
+    "client.submit_job(\n",
+    "    entrypoint=\"python -m flamingo run finetuning --config finetuning_config.yaml\",\n",
+    "    runtime_env=runtime_env\n",
+    ")"
+   ]
   }
  ],
  "metadata": {