From a3d3ac0c84f77423af803265d4983edd3620c5c5 Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 15:25:56 -0800 Subject: [PATCH 1/8] copy over wandb resume mode --- .../integrations/wandb/artifact_config.py | 18 ++++++++++++--- src/flamingo/integrations/wandb/run_utils.py | 15 ++++++++++++- src/flamingo/jobs/finetuning/entrypoint.py | 5 ++--- src/flamingo/jobs/lm_harness/entrypoint.py | 22 +++++++++++-------- .../wandb/test_artifact_config.py | 21 ++++++++++++++++++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/flamingo/integrations/wandb/artifact_config.py b/src/flamingo/integrations/wandb/artifact_config.py index 797cacde..93d03835 100644 --- a/src/flamingo/integrations/wandb/artifact_config.py +++ b/src/flamingo/integrations/wandb/artifact_config.py @@ -1,3 +1,5 @@ +import re + from flamingo.types import BaseFlamingoConfig @@ -5,12 +7,22 @@ class WandbArtifactConfig(BaseFlamingoConfig): """Configuration required to retrieve an artifact from W&B.""" name: str + project: str version: str = "latest" - project: str | None = None entity: str | None = None + @classmethod + def from_wandb_path(cls, path: str) -> "WandbArtifactConfig": + """Construct a configuration from the full W&B name.""" + match = re.search(r"((.*)\/)?(.*)\/(.*)\:(.*)", path) + if match is not None: + entity, project, name, version = match.groups()[1:] + return cls(name=name, project=project, version=version, entity=entity) + raise ValueError(f"Invalid artifact path: {path}") + def wandb_path(self) -> str: """String identifier for the asset on the W&B platform.""" - path = "/".join(x for x in [self.entity, self.project, self.name] if x is not None) - path = f"{path}:{self.version}" + path = f"{self.project}/{self.name}:{self.version}" + if self.entity is not None: + path = f"{self.entity}/{path}" return path diff --git a/src/flamingo/integrations/wandb/run_utils.py b/src/flamingo/integrations/wandb/run_utils.py index afd55630..83081986 100644 --- a/src/flamingo/integrations/wandb/run_utils.py +++ b/src/flamingo/integrations/wandb/run_utils.py @@ -1,4 +1,5 @@ import contextlib +from enum import Enum from typing import Any import wandb @@ -8,13 +9,25 @@ from flamingo.types import BaseFlamingoConfig +class WandbResumeMode(str, Enum): + """Enumeration of modes for resuming a W&B run. + + This set is not exhaustive, just the commonly used modes within flamingo. + See their documentation (https://docs.wandb.ai/ref/python/init) for adding more options. + """ + + ALLOW = "allow" + MUST = "must" + NEVER = "never" + + @contextlib.contextmanager def wandb_init_from_config( config: WandbRunConfig, *, parameters: BaseFlamingoConfig | None = None, + resume: WandbResumeMode | None = None, job_type: str | None = None, - resume: str | None = None, ): """Initialize a W&B run from the internal run configuration. diff --git a/src/flamingo/jobs/finetuning/entrypoint.py b/src/flamingo/jobs/finetuning/entrypoint.py index c25949b3..8ed6d128 100644 --- a/src/flamingo/jobs/finetuning/entrypoint.py +++ b/src/flamingo/jobs/finetuning/entrypoint.py @@ -13,6 +13,7 @@ from flamingo.integrations.wandb import ( ArtifactType, ArtifactURIScheme, + WandbResumeMode, default_artifact_name, log_directory_reference, wandb_init_from_config, @@ -64,9 +65,7 @@ def training_function(config_data: dict): config = FinetuningJobConfig(**config_data) if is_tracking_enabled(config): with wandb_init_from_config( - config.tracking, - job_type=FlamingoJobType.FINETUNING, - resume="never", + config.tracking, resume=WandbResumeMode.NEVER, job_type=FlamingoJobType.FINETUNING ): load_and_train(config) else: diff --git a/src/flamingo/jobs/lm_harness/entrypoint.py b/src/flamingo/jobs/lm_harness/entrypoint.py index 2a5a064f..ec2efadf 100644 --- a/src/flamingo/jobs/lm_harness/entrypoint.py +++ b/src/flamingo/jobs/lm_harness/entrypoint.py @@ -7,13 +7,18 @@ from peft import PeftConfig from flamingo.integrations.huggingface import resolve_loadable_path -from flamingo.integrations.wandb import ArtifactType, default_artifact_name, wandb_init_from_config +from flamingo.integrations.wandb import ( + ArtifactType, + WandbResumeMode, + default_artifact_name, + wandb_init_from_config, +) from flamingo.jobs.lm_harness import LMHarnessJobConfig from flamingo.jobs.utils import FlamingoJobType # TODO: Should this also be abstracted to a helper method like log_artifact_from_path? -def build_evaluation_artifact(run_name: str, results: dict[str, dict[str, Any]]) -> wandb.Artifact: +def log_evaluation_artifact(run_name: str, results: dict[str, dict[str, Any]]) -> wandb.Artifact: print("Building artifact for evaluation results...") artifact_name = default_artifact_name(run_name, ArtifactType.EVALUATION) artifact = wandb.Artifact(artifact_name, type=ArtifactType.EVALUATION) @@ -22,12 +27,12 @@ def build_evaluation_artifact(run_name: str, results: dict[str, dict[str, Any]]) task_data = [(k, v) for k, v in task_results.items() if isinstance(v, int | float)] task_table = wandb.Table(data=task_data, columns=["metric", "value"]) artifact.add(task_table, name=f"task-{task_name}") - return artifact + return wandb.log_artifact(artifact) def load_harness_model(config: LMHarnessJobConfig) -> HFLM: # Helper method to return lm-harness model wrapper - def _loader(pretrained: str, tokenizer: str, peft: str | None): + def loader(pretrained: str, tokenizer: str, peft: str | None): quantization_kwargs = config.quantization.dict() if config.quantization else {} return HFLM( pretrained=pretrained, @@ -44,7 +49,7 @@ def _loader(pretrained: str, tokenizer: str, peft: str | None): load_path, revision = resolve_loadable_path(config.model.load_from) try: peft_config = PeftConfig.from_pretrained(load_path, revision=revision) - return _loader( + return loader( pretrained=peft_config.base_model_name_or_path, tokenizer=peft_config.base_model_name_or_path, peft=load_path, @@ -54,7 +59,7 @@ def _loader(pretrained: str, tokenizer: str, peft: str | None): f"Unable to load model as adapter: {e}. " "This is expected if the checkpoint does not contain adapter weights." ) - return _loader(pretrained=load_path, tokenizer=load_path, peft=None) + return loader(pretrained=load_path, tokenizer=load_path, peft=None) def load_and_evaluate(config: LMHarnessJobConfig) -> dict[str, Any]: @@ -81,12 +86,11 @@ def evaluation_task(config: LMHarnessJobConfig) -> None: with wandb_init_from_config( config.tracking, parameters=config.evaluator, # Log eval settings in W&B run + resume=WandbResumeMode.ALLOW, job_type=FlamingoJobType.EVALUATION, - resume="allow", ) as run: eval_results = load_and_evaluate(config) - artifact = build_evaluation_artifact(run.name, eval_results) - run.log_artifact(artifact) + log_evaluation_artifact(run.name, eval_results) else: load_and_evaluate(config) diff --git a/tests/unit/integrations/wandb/test_artifact_config.py b/tests/unit/integrations/wandb/test_artifact_config.py index 0cdfdd73..adc5f923 100644 --- a/tests/unit/integrations/wandb/test_artifact_config.py +++ b/tests/unit/integrations/wandb/test_artifact_config.py @@ -19,3 +19,24 @@ def test_serde_round_trip(wandb_artifact_config): def test_wandb_path(wandb_artifact_config): assert wandb_artifact_config.wandb_path() == "team/research/artifact-name:latest" + + +def test_from_wandb_path(): + valid_path_with_entity = "entity/project/name:latest" + config_with_entity = WandbArtifactConfig.from_wandb_path(valid_path_with_entity) + assert config_with_entity.name == "name" + assert config_with_entity.project == "project" + assert config_with_entity.version == "latest" + assert config_with_entity.entity == "entity" + + valid_path_without_entity = "project/name:latest" + config_without_entity = WandbArtifactConfig.from_wandb_path(valid_path_without_entity) + assert config_without_entity.name == "name" + assert config_without_entity.project == "project" + assert config_without_entity.version == "latest" + assert config_without_entity.entity is None + + with pytest.raises(ValueError): + WandbArtifactConfig.from_wandb_path("entity/project/name") # No version + with pytest.raises(ValueError): + WandbArtifactConfig.from_wandb_path("entity/project/name/version") # Bad delimiter From 7fddf2aefb0b554a522068c07e92c8e9e6a3ddc4 Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 15:27:39 -0800 Subject: [PATCH 2/8] add to tempfile context manager --- src/flamingo/jobs/base.py | 31 ++++++++++++++++++++++++++ src/flamingo/jobs/finetuning/config.py | 3 ++- src/flamingo/jobs/lm_harness/config.py | 3 ++- src/flamingo/jobs/simple/config.py | 4 ++-- 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 src/flamingo/jobs/base.py diff --git a/src/flamingo/jobs/base.py b/src/flamingo/jobs/base.py new file mode 100644 index 00000000..a0faa044 --- /dev/null +++ b/src/flamingo/jobs/base.py @@ -0,0 +1,31 @@ +import contextlib +import tempfile +from pathlib import Path + +from flamingo.types import BaseFlamingoConfig + + +class BaseJobConfig(BaseFlamingoConfig): + """A `BaseFlamingoConfig` with some additional functionality to support job entrypoints. + + Currently, there is a 1:1 mapping between job config implementations and job entrypoints. + We may want to tighten these interfaces in the future to make implementing that + relationship more of a mandatory feature of the library. + """ + + @contextlib.contextmanager + def to_tempfile(self, *, name: str | None = None, dir: str | Path | None = None): + """Enter a context manager with the config written to a temporary YAML file. + + Args: + name (str, optional): Name of the config file in the temporary directory + dir (str | Path, optional): Root path of the temporary directory + + Returns: + Path to the temporary config file + """ + config_name = name or "config.yaml" + with tempfile.TemporaryDirectory(dir=dir) as tmpdir: + config_path = Path(tmpdir) / config_name + self.to_yaml_file(config_path) + yield config_path diff --git a/src/flamingo/jobs/finetuning/config.py b/src/flamingo/jobs/finetuning/config.py index c80a491a..759e5d6f 100644 --- a/src/flamingo/jobs/finetuning/config.py +++ b/src/flamingo/jobs/finetuning/config.py @@ -9,6 +9,7 @@ TrainerConfig, ) from flamingo.integrations.wandb import WandbRunConfig +from flamingo.jobs.base import BaseJobConfig from flamingo.types import BaseFlamingoConfig @@ -23,7 +24,7 @@ class FinetuningRayConfig(BaseFlamingoConfig): storage_path: str | None = None # TODO: This should be set globally somehow -class FinetuningJobConfig(BaseFlamingoConfig): +class FinetuningJobConfig(BaseJobConfig): """Configuration to submit an LLM finetuning job.""" model: AutoModelConfig diff --git a/src/flamingo/jobs/lm_harness/config.py b/src/flamingo/jobs/lm_harness/config.py index f5d33b86..55053743 100644 --- a/src/flamingo/jobs/lm_harness/config.py +++ b/src/flamingo/jobs/lm_harness/config.py @@ -4,6 +4,7 @@ from flamingo.integrations.huggingface import AutoModelConfig, QuantizationConfig from flamingo.integrations.wandb import WandbRunConfig +from flamingo.jobs.base import BaseJobConfig from flamingo.types import BaseFlamingoConfig @@ -24,7 +25,7 @@ class LMHarnessEvaluatorConfig(BaseFlamingoConfig): limit: int | float | None = None -class LMHarnessJobConfig(BaseFlamingoConfig): +class LMHarnessJobConfig(BaseJobConfig): """Configuration to run an lm-evaluation-harness evaluation job.""" model: AutoModelConfig diff --git a/src/flamingo/jobs/simple/config.py b/src/flamingo/jobs/simple/config.py index 3c629e25..0dc08c5a 100644 --- a/src/flamingo/jobs/simple/config.py +++ b/src/flamingo/jobs/simple/config.py @@ -1,7 +1,7 @@ -from flamingo.types import BaseFlamingoConfig +from flamingo.jobs.base import BaseJobConfig -class SimpleJobConfig(BaseFlamingoConfig): +class SimpleJobConfig(BaseJobConfig): """Simple job submission config.""" magic_number: int From 27aa56664801d70a9d0e8aa8e7bb29fb36537cbf Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 15:48:13 -0800 Subject: [PATCH 3/8] update example notevbook --- examples/dev_workflow.ipynb | 42 ++++++++++++++++++- tests/unit/conftest.py | 20 ++++----- .../huggingface/test_loading_utils.py | 2 +- tests/unit/jobs/test_finetuning_config.py | 7 ++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/examples/dev_workflow.ipynb b/examples/dev_workflow.ipynb index d7931b0e..ac47ea1b 100644 --- a/examples/dev_workflow.ipynb +++ b/examples/dev_workflow.ipynb @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8c0f15ed-77dc-44ce-adb6-d1b59368f03c", + "id": "9b26d777", "metadata": {}, "outputs": [], "source": [ @@ -97,6 +97,46 @@ " entrypoint=f\"python -m flamingo run simple --config {CONFIG_FILE}\", runtime_env=runtime_env\n", ")" ] + }, + { + "cell_type": "markdown", + "id": "425be140", + "metadata": {}, + "source": [ + "## Iterative Submission" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5715d09", + "metadata": {}, + "outputs": [], + "source": [ + "from flamingo.jobs.simple import SimpleJobConfig\n", + "\n", + "# Generate job configs programatically for sweeps over parameter ranges\n", + "magic_numbers = [0, 10, 20, 40]\n", + "\n", + "for number in magic_numbers:\n", + " config = SimpleJobConfig(magic_number=number)\n", + "\n", + " # `config_path` is the fully qualified path to the config file on your local filesystem\n", + " with config.to_tempfile(name=\"config.yaml\") as config_path:\n", + " # `config_path.parent` is the working directory\n", + " runtime_env = {\n", + " \"working_dir\": str(config_path.parent),\n", + " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]}, \n", + " \"py_modules\": [str(flamingo_module)],\n", + " \"pip\": \"/path/to/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", + " }\n", + "\n", + " # `config_path.name` is the file name within the working directory\n", + " client.submit_job(\n", + " entrypoint=f\"python -m flamingo run simple --config {config_path.name}\", \n", + " runtime_env=runtime_env\n", + " )" + ] } ], "metadata": { diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c341d223..934e5f1c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,39 +12,35 @@ @pytest.fixture -def model_config_with_path(): +def model_config_with_repo_id(): return AutoModelConfig(load_from="mistral-ai/mistral-7", trust_remote_code=True) @pytest.fixture def model_config_with_artifact(): - artifact = WandbArtifactConfig(name="model") + artifact = WandbArtifactConfig(name="model", project="project") return AutoModelConfig(load_from=artifact, trust_remote_code=True) @pytest.fixture -def tokenizer_config_with_path(): +def tokenizer_config_with_repo_id(): return AutoTokenizerConfig(load_from="mistral-ai/mistral-7", trust_remote_code=True) @pytest.fixture def tokenizer_config_with_artifact(): - artifact = WandbArtifactConfig(name="tokenizer") + artifact = WandbArtifactConfig(name="tokenizer", project="project") return AutoTokenizerConfig(load_from=artifact, trust_remote_code=True) @pytest.fixture -def dataset_config_with_path(): - return TextDatasetConfig( - load_from="databricks/dolly15k", - text_field="text", - split="train", - ) +def dataset_config_with_repo_id(): + return TextDatasetConfig(load_from="databricks/dolly15k", text_field="text", split="train") @pytest.fixture def dataset_config_with_artifact(): - artifact = WandbArtifactConfig(name="dataset") + artifact = WandbArtifactConfig(name="dataset", project="project") return TextDatasetConfig(load_from=artifact, split="train") @@ -66,4 +62,4 @@ def adapter_config(): @pytest.fixture def wandb_run_config(): - return WandbRunConfig(name="run", run_id="12345", project="research", entity="mozilla-ai") + return WandbRunConfig(name="run", run_id="12345", project="research", entity="mzai") diff --git a/tests/unit/integrations/huggingface/test_loading_utils.py b/tests/unit/integrations/huggingface/test_loading_utils.py index e47e87b6..8a3e3be9 100644 --- a/tests/unit/integrations/huggingface/test_loading_utils.py +++ b/tests/unit/integrations/huggingface/test_loading_utils.py @@ -19,7 +19,7 @@ def test_dataset_loading(resources_dir): "flamingo.integrations.huggingface.loading_utils.get_artifact_filesystem_path", return_value=xyz_dataset_path, ): - artifact = WandbArtifactConfig(name="xyz-dataset") + artifact = WandbArtifactConfig(name="xyz-dataset", project="project") dataset_config = DatasetConfig(load_from=artifact, test_size=0.2, seed=0) dataset = load_dataset_from_config(dataset_config) diff --git a/tests/unit/jobs/test_finetuning_config.py b/tests/unit/jobs/test_finetuning_config.py index 54b9c515..2525e9c6 100644 --- a/tests/unit/jobs/test_finetuning_config.py +++ b/tests/unit/jobs/test_finetuning_config.py @@ -51,6 +51,13 @@ def test_load_example_config(examples_dir): assert FinetuningJobConfig.parse_raw(config.json()) == config +def test_to_tempfile(finetuning_job_config): + config_name = "my-special-config.yaml" + with finetuning_job_config.to_tempfile(name=config_name) as path: + assert path.name == config_name + assert FinetuningJobConfig.from_yaml_file(path) == finetuning_job_config + + def test_argument_validation(): model_repo = HuggingFaceRepoConfig(repo_id="model_path") tokenizer_repo = HuggingFaceRepoConfig(repo_id="dataset_path") From 844f3db5ad0a9689de6ae0dc9922ff8d1a094a88 Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 15:49:44 -0800 Subject: [PATCH 4/8] update docs --- src/flamingo/integrations/wandb/run_utils.py | 2 +- src/flamingo/jobs/finetuning/entrypoint.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flamingo/integrations/wandb/run_utils.py b/src/flamingo/integrations/wandb/run_utils.py index 83081986..b57e66e8 100644 --- a/src/flamingo/integrations/wandb/run_utils.py +++ b/src/flamingo/integrations/wandb/run_utils.py @@ -34,7 +34,7 @@ def wandb_init_from_config( This method can be entered as a context manager similar to `wandb.init` as follows: ``` - with wandb_init_from_config(run_config, resume="must") as run: + with wandb_init_from_config(run_config, resume=WandbResumeMode.MUST) as run: # Use the initialized run here ... ``` diff --git a/src/flamingo/jobs/finetuning/entrypoint.py b/src/flamingo/jobs/finetuning/entrypoint.py index 8ed6d128..52c7f792 100644 --- a/src/flamingo/jobs/finetuning/entrypoint.py +++ b/src/flamingo/jobs/finetuning/entrypoint.py @@ -95,7 +95,7 @@ def run_finetuning(config: FinetuningJobConfig): # Register a model artifact if tracking is enabled and Ray saved a checkpoint if config.tracking and result.checkpoint: # Must resume from the just-completed training run - with wandb_init_from_config(config.tracking, resume="must") as run: + with wandb_init_from_config(config.tracking, resume=WandbResumeMode.MUST) as run: print("Logging artifact for model checkpoint...") log_directory_reference( dir_path=f"{result.checkpoint.path}/checkpoint", From e7b7d2bc19892ffacfda5ccbed600c142adf0553 Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 19:31:01 -0800 Subject: [PATCH 5/8] update workflow docs --- examples/dev_workflow.ipynb | 88 +++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/examples/dev_workflow.ipynb b/examples/dev_workflow.ipynb index ac47ea1b..f58ed159 100644 --- a/examples/dev_workflow.ipynb +++ b/examples/dev_workflow.ipynb @@ -8,9 +8,29 @@ "# Development Workflow" ] }, + { + "cell_type": "markdown", + "id": "9366fd9e", + "metadata": {}, + "source": [ + "## File-based Submission" + ] + }, + { + "cell_type": "markdown", + "id": "fcd5240e", + "metadata": {}, + "source": [ + "This demonstrates the basic workflow for submitting a Flamingo job to Ray\n", + "from a configuration stored as a local file.\n", + "\n", + "The job configuration is stored as a YAML file in a the local `configs` directory,\n", + "and that directory is specified as the working directory of the Ray runtime environment upon submission." + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "9b26d777", "metadata": {}, "outputs": [], @@ -19,10 +39,7 @@ "import os\n", "from pathlib import Path\n", "\n", - "from ray.job_submission import JobSubmissionClient\n", - "\n", - "# flamingo should be installed in your development environment\n", - "import flamingo" + "from ray.job_submission import JobSubmissionClient" ] }, { @@ -39,36 +56,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "3258bb97-d3c6-4fee-aa0c-962c1411eaa7", "metadata": {}, "outputs": [], "source": [ "# Determine local module path for the flamingo repo\n", - "flamingo_module = Path(flamingo.__file__).parent" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1db3b9aa-99a4-49d9-8773-7b91ccf89c85", - "metadata": {}, - "outputs": [], - "source": [ - "# Load and inspect the config file\n", - "# Not mandatory for job submission, but helpful when debugging\n", - "from flamingo.jobs.simple import SimpleJobConfig\n", - "\n", - "CONFIG_DIR = Path(\"configs\")\n", - "CONFIG_FILE = \"simple_config.yaml\"\n", + "# In theory this workflow is possible without having the flamingo package installed locally,\n", + "# but this is a convenient means to access the local module path\n", + "import flamingo\n", "\n", - "config = SimpleJobConfig.from_yaml_file(CONFIG_DIR / CONFIG_FILE)\n", - "config" + "flamingo_module = Path(flamingo.__file__).parent" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "b81b36be-35ce-4398-a6d4-ac1f719f5c95", "metadata": {}, "outputs": [], @@ -77,10 +80,10 @@ "# py_modules contains the path to the local flamingo module directory\n", "# pip contains an export of the dependencies for the flamingo package (see CONTRIBUTING.md for how to generate)\n", "runtime_env = {\n", - " \"working_dir\": str(CONFIG_DIR),\n", + " \"working_dir\": \"configs\",\n", " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]}, # If running a job that uses W&B\n", " \"py_modules\": [str(flamingo_module)],\n", - " \"pip\": \"/path/to/flamingo/requirements.txt\",\n", + " \"pip\": \"/path/to/flamingo/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", "}" ] }, @@ -94,7 +97,8 @@ "# Submit the job to the Ray cluster\n", "# Note: flamingo is invoked by 'python -m flamingo' since the CLI is not installed in the environment\n", "client.submit_job(\n", - " entrypoint=f\"python -m flamingo run simple --config {CONFIG_FILE}\", runtime_env=runtime_env\n", + " entrypoint=f\"python -m flamingo run simple --config simple_config.yaml\",\n", + " runtime_env=runtime_env,\n", ")" ] }, @@ -106,6 +110,22 @@ "## Iterative Submission" ] }, + { + "cell_type": "markdown", + "id": "e99ce273", + "metadata": {}, + "source": [ + "It is also possible to submit Flamingo jobs using a fully Python/Jupyter driven workflow.\n", + "\n", + "In this case, the Flamingo job configuration is instantiated in your Python script\n", + "and written to a temporary directory for submission. \n", + "\n", + "The Ray working directory is based off this temporary YAML file location.\n", + "\n", + "This approach is convenient if you want to run sweeps over parameter ranges\n", + "and use a Python script/Jupyter notebook as your local \"driver\" for the workflow." + ] + }, { "cell_type": "code", "execution_count": null, @@ -119,6 +139,8 @@ "magic_numbers = [0, 10, 20, 40]\n", "\n", "for number in magic_numbers:\n", + " # Instantitate config in your workflow script\n", + " # You may also want to read a \"base\" config from file with some suitable defaults\n", " config = SimpleJobConfig(magic_number=number)\n", "\n", " # `config_path` is the fully qualified path to the config file on your local filesystem\n", @@ -126,15 +148,15 @@ " # `config_path.parent` is the working directory\n", " runtime_env = {\n", " \"working_dir\": str(config_path.parent),\n", - " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]}, \n", + " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]},\n", " \"py_modules\": [str(flamingo_module)],\n", - " \"pip\": \"/path/to/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", + " \"pip\": \"/path/to/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", " }\n", "\n", - " # `config_path.name` is the file name within the working directory\n", + " # `config_path.name` is the file name within the working directory, i.e., \"config.yaml\"\n", " client.submit_job(\n", - " entrypoint=f\"python -m flamingo run simple --config {config_path.name}\", \n", - " runtime_env=runtime_env\n", + " entrypoint=f\"python -m flamingo run simple --config {config_path.name}\",\n", + " runtime_env=runtime_env,\n", " )" ] } From ddcf5059f20c761ab36d347cc7153b7004d5340c Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 19:41:01 -0800 Subject: [PATCH 6/8] fix up some docs and get rid of extra config class --- src/flamingo/integrations/wandb/run_utils.py | 4 +-- src/flamingo/jobs/base.py | 31 -------------------- src/flamingo/jobs/finetuning/config.py | 3 +- src/flamingo/jobs/lm_harness/config.py | 3 +- src/flamingo/jobs/simple/config.py | 4 +-- src/flamingo/types.py | 19 ++++++++++++ tests/unit/jobs/test_finetuning_config.py | 7 ----- tests/unit/test_types.py | 9 ++++++ 8 files changed, 34 insertions(+), 46 deletions(-) delete mode 100644 src/flamingo/jobs/base.py diff --git a/src/flamingo/integrations/wandb/run_utils.py b/src/flamingo/integrations/wandb/run_utils.py index b57e66e8..cd56ef17 100644 --- a/src/flamingo/integrations/wandb/run_utils.py +++ b/src/flamingo/integrations/wandb/run_utils.py @@ -12,8 +12,8 @@ class WandbResumeMode(str, Enum): """Enumeration of modes for resuming a W&B run. - This set is not exhaustive, just the commonly used modes within flamingo. - See their documentation (https://docs.wandb.ai/ref/python/init) for adding more options. + This is not an exahustive list of the values that can be passed to the W&B SDK + (Docs: https://docs.wandb.ai/ref/python/init), but just those commonly used within the package. """ ALLOW = "allow" diff --git a/src/flamingo/jobs/base.py b/src/flamingo/jobs/base.py deleted file mode 100644 index a0faa044..00000000 --- a/src/flamingo/jobs/base.py +++ /dev/null @@ -1,31 +0,0 @@ -import contextlib -import tempfile -from pathlib import Path - -from flamingo.types import BaseFlamingoConfig - - -class BaseJobConfig(BaseFlamingoConfig): - """A `BaseFlamingoConfig` with some additional functionality to support job entrypoints. - - Currently, there is a 1:1 mapping between job config implementations and job entrypoints. - We may want to tighten these interfaces in the future to make implementing that - relationship more of a mandatory feature of the library. - """ - - @contextlib.contextmanager - def to_tempfile(self, *, name: str | None = None, dir: str | Path | None = None): - """Enter a context manager with the config written to a temporary YAML file. - - Args: - name (str, optional): Name of the config file in the temporary directory - dir (str | Path, optional): Root path of the temporary directory - - Returns: - Path to the temporary config file - """ - config_name = name or "config.yaml" - with tempfile.TemporaryDirectory(dir=dir) as tmpdir: - config_path = Path(tmpdir) / config_name - self.to_yaml_file(config_path) - yield config_path diff --git a/src/flamingo/jobs/finetuning/config.py b/src/flamingo/jobs/finetuning/config.py index 759e5d6f..c80a491a 100644 --- a/src/flamingo/jobs/finetuning/config.py +++ b/src/flamingo/jobs/finetuning/config.py @@ -9,7 +9,6 @@ TrainerConfig, ) from flamingo.integrations.wandb import WandbRunConfig -from flamingo.jobs.base import BaseJobConfig from flamingo.types import BaseFlamingoConfig @@ -24,7 +23,7 @@ class FinetuningRayConfig(BaseFlamingoConfig): storage_path: str | None = None # TODO: This should be set globally somehow -class FinetuningJobConfig(BaseJobConfig): +class FinetuningJobConfig(BaseFlamingoConfig): """Configuration to submit an LLM finetuning job.""" model: AutoModelConfig diff --git a/src/flamingo/jobs/lm_harness/config.py b/src/flamingo/jobs/lm_harness/config.py index 55053743..f5d33b86 100644 --- a/src/flamingo/jobs/lm_harness/config.py +++ b/src/flamingo/jobs/lm_harness/config.py @@ -4,7 +4,6 @@ from flamingo.integrations.huggingface import AutoModelConfig, QuantizationConfig from flamingo.integrations.wandb import WandbRunConfig -from flamingo.jobs.base import BaseJobConfig from flamingo.types import BaseFlamingoConfig @@ -25,7 +24,7 @@ class LMHarnessEvaluatorConfig(BaseFlamingoConfig): limit: int | float | None = None -class LMHarnessJobConfig(BaseJobConfig): +class LMHarnessJobConfig(BaseFlamingoConfig): """Configuration to run an lm-evaluation-harness evaluation job.""" model: AutoModelConfig diff --git a/src/flamingo/jobs/simple/config.py b/src/flamingo/jobs/simple/config.py index 0dc08c5a..3c629e25 100644 --- a/src/flamingo/jobs/simple/config.py +++ b/src/flamingo/jobs/simple/config.py @@ -1,7 +1,7 @@ -from flamingo.jobs.base import BaseJobConfig +from flamingo.types import BaseFlamingoConfig -class SimpleJobConfig(BaseJobConfig): +class SimpleJobConfig(BaseFlamingoConfig): """Simple job submission config.""" magic_number: int diff --git a/src/flamingo/types.py b/src/flamingo/types.py index be8aa294..f7a7ea78 100644 --- a/src/flamingo/types.py +++ b/src/flamingo/types.py @@ -1,3 +1,5 @@ +import contextlib +import tempfile from pathlib import Path from typing import Any @@ -56,3 +58,20 @@ def from_yaml_file(cls, path: Path | str): def to_yaml_file(self, path: Path | str): to_yaml_file(path, self, exclude_none=True) + + @contextlib.contextmanager + def to_tempfile(self, *, name: str | None = None, dir: str | Path | None = None): + """Enter a context manager with the config written to a temporary YAML file. + + Args: + name (str, optional): Name of the config file in the temporary directory + dir (str | Path, optional): Root path of the temporary directory + + Returns: + Path to the temporary config file + """ + config_name = name or "config.yaml" + with tempfile.TemporaryDirectory(dir=dir) as tmpdir: + config_path = Path(tmpdir) / config_name + self.to_yaml_file(config_path) + yield config_path diff --git a/tests/unit/jobs/test_finetuning_config.py b/tests/unit/jobs/test_finetuning_config.py index 2525e9c6..54b9c515 100644 --- a/tests/unit/jobs/test_finetuning_config.py +++ b/tests/unit/jobs/test_finetuning_config.py @@ -51,13 +51,6 @@ def test_load_example_config(examples_dir): assert FinetuningJobConfig.parse_raw(config.json()) == config -def test_to_tempfile(finetuning_job_config): - config_name = "my-special-config.yaml" - with finetuning_job_config.to_tempfile(name=config_name) as path: - assert path.name == config_name - assert FinetuningJobConfig.from_yaml_file(path) == finetuning_job_config - - def test_argument_validation(): model_repo = HuggingFaceRepoConfig(repo_id="model_path") tokenizer_repo = HuggingFaceRepoConfig(repo_id="dataset_path") diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 1f03ad74..c4022384 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -1,6 +1,7 @@ import pytest import torch +from flamingo.jobs.simple import SimpleJobConfig from flamingo.types import TorchDtypeString @@ -17,3 +18,11 @@ def test_torch_dtype_validation(): TorchDtypeString.validate(5) with pytest.raises(ValueError): TorchDtypeString.validate("dogs") + + +def test_config_as_tempfile(): + config = SimpleJobConfig(magic_number=42) + config_name = "my-special-config.yaml" + with config.to_tempfile(name=config_name) as path: + assert path.name == config_name + assert SimpleJobConfig.from_yaml_file(path) == config From 5529b4ffa1df499cf0980eed978973c707cad0ac Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Mon, 29 Jan 2024 19:44:01 -0800 Subject: [PATCH 7/8] docstring --- src/flamingo/types.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/flamingo/types.py b/src/flamingo/types.py index f7a7ea78..fee4ef82 100644 --- a/src/flamingo/types.py +++ b/src/flamingo/types.py @@ -60,18 +60,17 @@ def to_yaml_file(self, path: Path | str): to_yaml_file(path, self, exclude_none=True) @contextlib.contextmanager - def to_tempfile(self, *, name: str | None = None, dir: str | Path | None = None): + def to_tempfile(self, *, name: str = "config.yaml", dir: str | Path | None = None): """Enter a context manager with the config written to a temporary YAML file. Args: - name (str, optional): Name of the config file in the temporary directory + name (str): Name of the config file in the tmp directory. Defaults to "config.yaml" dir (str | Path, optional): Root path of the temporary directory Returns: Path to the temporary config file """ - config_name = name or "config.yaml" with tempfile.TemporaryDirectory(dir=dir) as tmpdir: - config_path = Path(tmpdir) / config_name + config_path = Path(tmpdir) / name self.to_yaml_file(config_path) yield config_path From 2950b597a704e1c2b14672a8c9b17390058fc136 Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Tue, 30 Jan 2024 08:47:31 -0800 Subject: [PATCH 8/8] update configs, requirements, and tests --- examples/configs/finetuning.yaml | 50 +++++++++++++++++ examples/configs/finetuning_config.yaml | 42 -------------- examples/configs/lm_harness.yaml | 25 +++++++++ examples/configs/lm_harness_config.yaml | 28 ---------- .../{simple_config.yaml => simple.yaml} | 0 examples/dev_workflow.ipynb | 55 +++++++++++++++++-- pyproject.toml | 2 +- .../integrations/wandb/artifact_config.py | 6 +- tests/unit/jobs/test_finetuning_config.py | 9 ++- tests/unit/jobs/test_lm_harness_config.py | 9 ++- 10 files changed, 138 insertions(+), 88 deletions(-) create mode 100644 examples/configs/finetuning.yaml delete mode 100644 examples/configs/finetuning_config.yaml create mode 100644 examples/configs/lm_harness.yaml delete mode 100644 examples/configs/lm_harness_config.yaml rename examples/configs/{simple_config.yaml => simple.yaml} (100%) diff --git a/examples/configs/finetuning.yaml b/examples/configs/finetuning.yaml new file mode 100644 index 00000000..08e94414 --- /dev/null +++ b/examples/configs/finetuning.yaml @@ -0,0 +1,50 @@ +# Base model to load for finetuning +model: + load_from: + repo_id: "distilgpt2" + # Can also specify the asset to load as a W&B artifact + # load_from: + # name: "artifact-name" + # project: "artifact-project" + # version: "v0" + torch_dtype: "bfloat16" + +# Tokenizer section (when not defined, will default to the model value) +# tokenizer: "distilgpt2" + +# Text dataset to use for training +dataset: + load_from: + repo_id: "imdb" + split: "train[:100]" + test_size: 0.2 + text_field: "text" + +trainer: + max_seq_length: 512 + learning_rate: 0.001 + num_train_epochs: 2 + save_steps: 1 + save_strategy: "epochs" + logging_steps: 1 + logging_strategy: "steps" + +# Quantization section (not necessary when using LORA w/ built in LOFT-Q) +# quantization: + +adapter: + peft_type: "LORA" + task_type: "CAUSAL_LM" + r: 16 + lora_alpha: 32 + lora_dropout: 0.2 + +# Tracking info for where to log the run results +tracking: + name: "flamingo-example-finetuning" + project: "flamingo-examples" + entity: "mozilla-ai" + +ray: + use_gpu: True + num_workers: 2 diff --git a/examples/configs/finetuning_config.yaml b/examples/configs/finetuning_config.yaml deleted file mode 100644 index 99a581eb..00000000 --- a/examples/configs/finetuning_config.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Tokenizer defined by only a string repo ID -tokenizer: "mistral-ai/other-repo-with-special-tokenizer" - -# Model defined as an object with additional settings beyond the repo ID -model: - load_from: "mistral-ai/mistral-7b" - trust_remote_code: True - torch_dtype: "bfloat16" - -# Dataset defined as an object with a path linking to a W&B artifact -dataset: - load_from: - name: "dataset-artifact" - version: "latest" - project: "research-project" - split: "train" - test_size: 0.2 - -trainer: - max_seq_length: 512 - learning_rate: 0.1 - num_train_epochs: 2 - -quantization: - load_in_4bit: True - bnb_4bit_quant_type: "fp4" - -adapter: - peft_type: "LORA" - task_type: "CAUSAL_LM" - r: 16 - lora_alpha: 32 - lora_dropout: 0.2 - -tracking: - name: "location-to-log-results" - project: "another-project" - entity: "another-entity" - -ray: - use_gpu: True - num_workers: 4 diff --git a/examples/configs/lm_harness.yaml b/examples/configs/lm_harness.yaml new file mode 100644 index 00000000..1e8380ca --- /dev/null +++ b/examples/configs/lm_harness.yaml @@ -0,0 +1,25 @@ +# Model to evaluate +model: + load_from: "distilgpt2" + torch_dtype: "bfloat16" + +# Settings specific to lm_harness.evaluate +evaluator: + tasks: ["hellaswag"] + num_fewshot: 5 + limit: 10 + +quantization: + load_in_4bit: True + bnb_4bit_quant_type: "fp4" + +# Tracking info for where to log the run results +tracking: + name: "flamingo-example-lm-harness" + project: "flamingo-examples" + entity: "mozilla-ai" + +ray: + num_cpus: 1 + num_gpus: 1 + timeout: 3600 diff --git a/examples/configs/lm_harness_config.yaml b/examples/configs/lm_harness_config.yaml deleted file mode 100644 index 46df88c5..00000000 --- a/examples/configs/lm_harness_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Model to evaluate, specified as a W&B artifact -model: - load_from: - name: "training-run-model-artifact" - version: "v4" - project: "research-project" - entity: "mozilla.ai" - trust_remote_code: True - torch_dtype: "float16" - -# Settings specific to lm_harness.evaluate -evaluator: - tasks: ["task1", "task2", "...", "taskN"] - num_fewshot: 5 - -quantization: - load_in_4bit: True - bnb_4bit_quant_type: "fp4" - -tracking: - name: "location-to-log-results" - project: "another-project" - entity: "another-entity" - -ray: - num_cpus: 1 - num_gpus: 4 - timeout: 3600 diff --git a/examples/configs/simple_config.yaml b/examples/configs/simple.yaml similarity index 100% rename from examples/configs/simple_config.yaml rename to examples/configs/simple.yaml diff --git a/examples/dev_workflow.ipynb b/examples/dev_workflow.ipynb index f58ed159..5ee3bdf4 100644 --- a/examples/dev_workflow.ipynb +++ b/examples/dev_workflow.ipynb @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "9b26d777", "metadata": {}, "outputs": [], @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "3258bb97-d3c6-4fee-aa0c-962c1411eaa7", "metadata": {}, "outputs": [], @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "b81b36be-35ce-4398-a6d4-ac1f719f5c95", "metadata": {}, "outputs": [], @@ -83,7 +83,7 @@ " \"working_dir\": \"configs\",\n", " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]}, # If running a job that uses W&B\n", " \"py_modules\": [str(flamingo_module)],\n", - " \"pip\": \"/path/to/flamingo/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", + " \"pip\": \"requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", "}" ] }, @@ -97,7 +97,7 @@ "# Submit the job to the Ray cluster\n", "# Note: flamingo is invoked by 'python -m flamingo' since the CLI is not installed in the environment\n", "client.submit_job(\n", - " entrypoint=f\"python -m flamingo run simple --config simple_config.yaml\",\n", + " entrypoint=f\"python -m flamingo run simple --config simple.yaml\",\n", " runtime_env=runtime_env,\n", ")" ] @@ -126,6 +126,47 @@ "and use a Python script/Jupyter notebook as your local \"driver\" for the workflow." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cfccaa9", + "metadata": {}, + "outputs": [], + "source": [ + "# Required imports\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "from ray.job_submission import JobSubmissionClient" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a51235ed", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a submission client bound to a Ray cluster\n", + "# Note: You will likely have to update the cluster address shown below\n", + "client = JobSubmissionClient(\"http://10.147.154.77:8265\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1216c43", + "metadata": {}, + "outputs": [], + "source": [ + "# Determine local module path for the flamingo repo\n", + "# In theory this workflow is possible without having the flamingo package installed locally,\n", + "# but this is a convenient means to access the local module path\n", + "import flamingo\n", + "\n", + "flamingo_module = Path(flamingo.__file__).parent" + ] + }, { "cell_type": "code", "execution_count": null, @@ -133,6 +174,8 @@ "metadata": {}, "outputs": [], "source": [ + "import os\n", + "\n", "from flamingo.jobs.simple import SimpleJobConfig\n", "\n", "# Generate job configs programatically for sweeps over parameter ranges\n", @@ -150,7 +193,7 @@ " \"working_dir\": str(config_path.parent),\n", " \"env_vars\": {\"WANDB_API_KEY\": os.environ[\"WANDB_API_KEY\"]},\n", " \"py_modules\": [str(flamingo_module)],\n", - " \"pip\": \"/path/to/requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", + " \"pip\": \"requirements.txt\", # See CONTRIBUTING.md for how to generate this\n", " }\n", "\n", " # `config_path.name` is the file name within the working directory, i.e., \"config.yaml\"\n", diff --git a/pyproject.toml b/pyproject.toml index d729da83..1daea82d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ protobuf = "3.20.0" urllib3 = ">=1.26.18,<2" pydantic = "1.10.14" pydantic-yaml = "1.2.0" -ray = { version = "2.8.0", extras = ["default"] } +ray = { version = "2.9.1", extras = ["default"] } [tool.poetry.dev-dependencies] ruff = "0.1.7" diff --git a/src/flamingo/integrations/wandb/artifact_config.py b/src/flamingo/integrations/wandb/artifact_config.py index 93d03835..c38f115b 100644 --- a/src/flamingo/integrations/wandb/artifact_config.py +++ b/src/flamingo/integrations/wandb/artifact_config.py @@ -13,7 +13,11 @@ class WandbArtifactConfig(BaseFlamingoConfig): @classmethod def from_wandb_path(cls, path: str) -> "WandbArtifactConfig": - """Construct a configuration from the full W&B name.""" + """Construct an artifact configuration from the W&B name. + + The name should be of the form "//:" + with the "entity" field optional. + """ match = re.search(r"((.*)\/)?(.*)\/(.*)\:(.*)", path) if match is not None: entity, project, name, version = match.groups()[1:] diff --git a/tests/unit/jobs/test_finetuning_config.py b/tests/unit/jobs/test_finetuning_config.py index 54b9c515..0e776cf6 100644 --- a/tests/unit/jobs/test_finetuning_config.py +++ b/tests/unit/jobs/test_finetuning_config.py @@ -38,15 +38,14 @@ def test_serde_round_trip(finetuning_job_config): assert FinetuningJobConfig.parse_raw(finetuning_job_config.json()) == finetuning_job_config -def test_parse_yaml_file(finetuning_job_config, tmp_path_factory): - config_path = tmp_path_factory.mktemp("flamingo_tests") / "finetuning_config.yaml" - finetuning_job_config.to_yaml_file(config_path) - assert finetuning_job_config == FinetuningJobConfig.from_yaml_file(config_path) +def test_parse_yaml_file(finetuning_job_config): + with finetuning_job_config.to_tempfile() as config_path: + assert finetuning_job_config == FinetuningJobConfig.from_yaml_file(config_path) def test_load_example_config(examples_dir): """Load the example configs to make sure they stay up to date.""" - config_file = examples_dir / "configs" / "finetuning_config.yaml" + config_file = examples_dir / "configs" / "finetuning.yaml" config = FinetuningJobConfig.from_yaml_file(config_file) assert FinetuningJobConfig.parse_raw(config.json()) == config diff --git a/tests/unit/jobs/test_lm_harness_config.py b/tests/unit/jobs/test_lm_harness_config.py index fdbb2fa0..a2a07bdc 100644 --- a/tests/unit/jobs/test_lm_harness_config.py +++ b/tests/unit/jobs/test_lm_harness_config.py @@ -47,15 +47,14 @@ def test_serde_round_trip(lm_harness_job_config): assert LMHarnessJobConfig.parse_raw(lm_harness_job_config.json()) == lm_harness_job_config -def test_parse_yaml_file(lm_harness_job_config, tmp_path_factory): - config_path = tmp_path_factory.mktemp("flamingo_tests") / "lm_harness_config.yaml" - lm_harness_job_config.to_yaml_file(config_path) - assert lm_harness_job_config == LMHarnessJobConfig.from_yaml_file(config_path) +def test_parse_yaml_file(lm_harness_job_config): + with lm_harness_job_config.to_tempfile() as config_path: + assert lm_harness_job_config == LMHarnessJobConfig.from_yaml_file(config_path) def test_load_example_config(examples_dir): """Load the example configs to make sure they stay up to date.""" - config_file = examples_dir / "configs" / "lm_harness_config.yaml" + config_file = examples_dir / "configs" / "lm_harness.yaml" config = LMHarnessJobConfig.from_yaml_file(config_file) assert LMHarnessJobConfig.parse_raw(config.json()) == config