From 639c466bd8c4592d2b802dd59511a17d59d3c81a Mon Sep 17 00:00:00 2001 From: Sean Friedowitz Date: Wed, 10 Jan 2024 16:37:44 -0500 Subject: [PATCH] porting more stuff --- pyproject.toml | 2 +- src/flamingo/cli.py | 32 ++++- .../integrations/wandb/wandb_mixin.py | 22 --- src/flamingo/jobs/__init__.py | 14 +- src/flamingo/jobs/configs/__init__.py | 11 ++ .../jobs/{ => configs}/base_config.py | 0 .../jobs/{ => configs}/finetuning_config.py | 9 +- .../lm_harness_config.py} | 9 +- src/flamingo/jobs/configs/simple_config.py | 7 + src/flamingo/jobs/entrypoints/__init__.py | 5 + .../jobs/entrypoints/run_finetuning.py | 136 ++++++++++++++++++ .../jobs/entrypoints/run_lm_harness.py | 118 +++++++++++++++ src/flamingo/jobs/entrypoints/run_simple.py | 5 + src/flamingo/jobs/simple_config.py | 15 -- 14 files changed, 321 insertions(+), 64 deletions(-) delete mode 100644 src/flamingo/integrations/wandb/wandb_mixin.py create mode 100644 src/flamingo/jobs/configs/__init__.py rename src/flamingo/jobs/{ => configs}/base_config.py (100%) rename src/flamingo/jobs/{ => configs}/finetuning_config.py (85%) rename src/flamingo/jobs/{evaluation_config.py => configs/lm_harness_config.py} (93%) create mode 100644 src/flamingo/jobs/configs/simple_config.py create mode 100644 src/flamingo/jobs/entrypoints/__init__.py create mode 100644 src/flamingo/jobs/entrypoints/run_finetuning.py create mode 100644 src/flamingo/jobs/entrypoints/run_lm_harness.py create mode 100644 src/flamingo/jobs/entrypoints/run_simple.py delete mode 100644 src/flamingo/jobs/simple_config.py diff --git a/pyproject.toml b/pyproject.toml index e14aa8b4..a3777a99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,12 +11,12 @@ requires-python = ">=3.10,<3.11" dependencies = [ "click==8.1.7", - "ray[default]==2.7.0", "torch==2.1.0", "scipy==1.10.1", "wandb==0.16.1", "pydantic-yaml==1.2.0", "pydantic==1.10.8", + "ray[default]==2.7.0", ] [project.optional-dependencies] diff --git a/src/flamingo/cli.py b/src/flamingo/cli.py index 80743421..275ae2ea 100644 --- a/src/flamingo/cli.py +++ b/src/flamingo/cli.py @@ -1,14 +1,40 @@ import click +from flamingo.jobs import run_finetuning, run_lm_harness, run_simple +from flamingo.jobs.configs import FinetuningJobConfig, LMHarnessJobConfig, SimpleJobConfig + @click.group() -def main(): +def cli(): pass +@click.group("simple") +@click.option("--config", type=str) +def run_simple_cli(config: str) -> None: + config = SimpleJobConfig.from_yaml_file(config) + run_simple.main(config) + + +@click.group("finetune") +@click.option("--config", type=str) +def run_finetuning_cli(config: str) -> None: + config = FinetuningJobConfig.from_yaml_file(config) + run_finetuning.main(config) + + +@click.group("finetune") +@click.option("--config", type=str) +def run_finetuning_cli(config: str) -> None: + config = FinetuningJobConfig.from_yaml_file(config) + run_finetuning.main(config) + + # need to add the group / command function itself, not the module -main.add_command(simple.driver) +cli.add_command(run_simple.main) +cli.add_command(run_finetuning.main) +cli.add_command(run_lm_harness.main) if __name__ == "__main__": - main() + cli() diff --git a/src/flamingo/integrations/wandb/wandb_mixin.py b/src/flamingo/integrations/wandb/wandb_mixin.py deleted file mode 100644 index 56266b81..00000000 --- a/src/flamingo/integrations/wandb/wandb_mixin.py +++ /dev/null @@ -1,22 +0,0 @@ -from flamingo.integrations.wandb import WandbEnvironment -from flamingo.types import BaseFlamingoConfig - - -class WandbEnvironmentMixin(BaseFlamingoConfig): - """Mixin for a config that contains W&B environment settings.""" - - wandb_env: WandbEnvironment | None = None - - @property - def env_vars(self) -> dict[str, str]: - return self.wandb_env.env_vars if self.wandb_env else {} - - @property - def wandb_name(self) -> str | None: - """Return the W&B run name, if it exists.""" - return self.wandb_env.name if self.wandb_env else None - - @property - def wandb_project(self) -> str | None: - """Return the W&B project name, if it exists.""" - return self.wandb_env.project if self.wandb_env else None diff --git a/src/flamingo/jobs/__init__.py b/src/flamingo/jobs/__init__.py index 427b7422..a4e72c0d 100644 --- a/src/flamingo/jobs/__init__.py +++ b/src/flamingo/jobs/__init__.py @@ -1,11 +1,3 @@ -from .base_config import BaseJobConfig -from .evaluation_config import EvaluationJobConfig -from .finetuning_config import FinetuningJobConfig -from .simple_config import SimpleJobConfig - -__all__ = [ - "BaseJobConfig", - "SimpleJobConfig", - "FinetuningJobConfig", - "EvaluationJobConfig", -] +# ruff: noqa +from .configs import * +from .entrypoints import * diff --git a/src/flamingo/jobs/configs/__init__.py b/src/flamingo/jobs/configs/__init__.py new file mode 100644 index 00000000..dc5c0bfc --- /dev/null +++ b/src/flamingo/jobs/configs/__init__.py @@ -0,0 +1,11 @@ +from .base_config import BaseJobConfig +from .finetuning_config import FinetuningJobConfig +from .lm_harness_config import LMHarnessJobConfig +from .simple_config import SimpleJobConfig + +__all__ = [ + "BaseJobConfig", + "SimpleJobConfig", + "FinetuningJobConfig", + "LMHarnessJobConfig", +] diff --git a/src/flamingo/jobs/base_config.py b/src/flamingo/jobs/configs/base_config.py similarity index 100% rename from src/flamingo/jobs/base_config.py rename to src/flamingo/jobs/configs/base_config.py diff --git a/src/flamingo/jobs/finetuning_config.py b/src/flamingo/jobs/configs/finetuning_config.py similarity index 85% rename from src/flamingo/jobs/finetuning_config.py rename to src/flamingo/jobs/configs/finetuning_config.py index 51a733e4..2ffcb6c9 100644 --- a/src/flamingo/jobs/finetuning_config.py +++ b/src/flamingo/jobs/configs/finetuning_config.py @@ -4,12 +4,12 @@ from flamingo.integrations.huggingface import QuantizationConfig from flamingo.integrations.huggingface.utils import is_valid_huggingface_model_name -from flamingo.integrations.wandb import WandbEnvironmentMixin +from flamingo.integrations.wandb import WandbEnvironment from flamingo.jobs import BaseJobConfig from flamingo.types import SerializableTorchDtype -class FinetuningJobConfig(WandbEnvironmentMixin, BaseJobConfig): +class FinetuningJobConfig(BaseJobConfig): """Configuration to submit an LLM finetuning job.""" model: str @@ -32,6 +32,7 @@ class FinetuningJobConfig(WandbEnvironmentMixin, BaseJobConfig): logging_steps: float = 100 save_strategy: str = "steps" save_steps: int = 500 + wandb_env: WandbEnvironment | None = None # Lora/quantization lora_config: LoraConfig | None = None # TODO: Create our own config type quantization_config: QuantizationConfig | None = None @@ -45,7 +46,3 @@ def _validate_modelname(cls, v): # noqa: N805 return v else: raise (ValueError(f"`{v}` is not a valid HuggingFace model name.")) - - @property - def entrypoint_command(self) -> str: - return f"python run_finetuning.py --config_json '{self.json()}'" diff --git a/src/flamingo/jobs/evaluation_config.py b/src/flamingo/jobs/configs/lm_harness_config.py similarity index 93% rename from src/flamingo/jobs/evaluation_config.py rename to src/flamingo/jobs/configs/lm_harness_config.py index a2d8c85b..a7f6de38 100644 --- a/src/flamingo/jobs/evaluation_config.py +++ b/src/flamingo/jobs/configs/lm_harness_config.py @@ -8,7 +8,7 @@ from flamingo.integrations.huggingface import QuantizationConfig from flamingo.integrations.huggingface.utils import is_valid_huggingface_model_name -from flamingo.integrations.wandb import WandbEnvironment, WandbEnvironmentMixin +from flamingo.integrations.wandb import WandbEnvironment from flamingo.jobs import BaseJobConfig from flamingo.types import SerializableTorchDtype @@ -42,7 +42,7 @@ def __post_init__(self, checkpoint): raise (ValueError(f"{self.name} is not a valid checkpoint path or HF model name")) -class EvaluationJobConfig(WandbEnvironmentMixin, BaseJobConfig): +class LMHarnessJobConfig(BaseJobConfig): """Configuration to run an lm-evaluation-harness evaluation job. This job loads an existing checkpoint path @@ -66,6 +66,7 @@ class Config: trust_remote_code: bool = False torch_dtype: SerializableTorchDtype = None quantization_config: QuantizationConfig | None = None + wandb_env: WandbEnvironment | None = None num_cpus: int = 1 num_gpus: int = 1 timeout: datetime.timedelta | None = None @@ -112,7 +113,3 @@ def _validate_modelname_or_checkpoint(cls, values) -> Any: raise (ValueError(f"{mnp} is not a valid HuggingFaceModel or checkpoint path.")) return values - - @property - def entrypoint_command(self) -> str: - return f"python run_evaluation.py --config_json '{self.json()}'" diff --git a/src/flamingo/jobs/configs/simple_config.py b/src/flamingo/jobs/configs/simple_config.py new file mode 100644 index 00000000..373cb514 --- /dev/null +++ b/src/flamingo/jobs/configs/simple_config.py @@ -0,0 +1,7 @@ +from flamingo.jobs import BaseJobConfig + + +class SimpleJobConfig(BaseJobConfig): + """A simple job to demonstrate the submission interface.""" + + magic_number: int diff --git a/src/flamingo/jobs/entrypoints/__init__.py b/src/flamingo/jobs/entrypoints/__init__.py new file mode 100644 index 00000000..8fa9df64 --- /dev/null +++ b/src/flamingo/jobs/entrypoints/__init__.py @@ -0,0 +1,5 @@ +import run_finetuning +import run_lm_harness +import run_simple + +__all__ = ["run_finetuning", "run_lm_harness", "run_simple"] diff --git a/src/flamingo/jobs/entrypoints/run_finetuning.py b/src/flamingo/jobs/entrypoints/run_finetuning.py new file mode 100644 index 00000000..8b2b90e6 --- /dev/null +++ b/src/flamingo/jobs/entrypoints/run_finetuning.py @@ -0,0 +1,136 @@ +import json + +import torch +import wandb +from accelerate import Accelerator +from datasets import DatasetDict +from ray import train +from ray.train import CheckpointConfig, RunConfig +from ray.train.huggingface.transformers import RayTrainReportCallback, prepare_trainer +from ray.train.torch import TorchTrainer +from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedModel, TrainingArguments +from trl import SFTTrainer + +from flamingo.integrations.wandb import update_wandb_summary +from flamingo.jobs import FinetuningJobConfig + + +def is_wandb_enabled(config: FinetuningJobConfig): + # Only report to WandB on the rank 0 worker + # Reference: https://docs.ray.io/en/latest/train/user-guides/experiment-tracking.html + return config.wandb_env and train.get_context().get_world_rank() == 0 + + +def get_training_args(config: FinetuningJobConfig) -> TrainingArguments: + """Get TrainingArguments appropriate for the worker rank and job config.""" + return TrainingArguments( + output_dir="out", # Local checkpoint path on a worker + num_train_epochs=config.num_train_epochs, + learning_rate=config.learning_rate, + per_device_train_batch_size=config.batch_size, + per_device_eval_batch_size=config.batch_size, + gradient_accumulation_steps=config.gradient_accumulation_steps, + gradient_checkpointing=config.gradient_checkpointing, + weight_decay=config.weight_decay, + evaluation_strategy=config.evaluation_strategy, + eval_steps=config.eval_steps, + logging_strategy=config.logging_strategy, + logging_steps=config.logging_steps, + save_strategy=config.save_strategy, + save_steps=config.save_steps, + run_name=config.wandb_name, + report_to="wandb" if is_wandb_enabled(config) else "none", + no_cuda=not config.scaling_config.use_gpu, + push_to_hub=False, + disable_tqdm=True, + logging_dir=None, + ) + + +def get_datasets(config: FinetuningJobConfig) -> DatasetDict: + # TODO: Refactor me somehow + ... + + +def get_model(config: FinetuningJobConfig) -> PreTrainedModel: + device_map, bnb_config = None, None + if config.quantization_config: + bnb_config = config.quantization_config.as_huggingface() + # When quantization is enabled, model must all be on same GPU to work with DDP + # If a device_map is not specified we will get accelerate errors downstream + # Reference: https://github.com/huggingface/accelerate/issues/1840#issuecomment-1683105994 + current_device = Accelerator().local_process_index if torch.cuda.is_available() else "cpu" + device_map = {"": current_device} + print(f"Setting model device_map = {device_map} to enable quantization") + + return AutoModelForCausalLM.from_pretrained( + config.model, + trust_remote_code=config.trust_remote_code, + torch_dtype=config.torch_dtype, + quantization_config=bnb_config, + device_map=device_map, + ) + + +def get_tokenizer(config: FinetuningJobConfig): + tokenizer = AutoTokenizer.from_pretrained( + config.tokenizer or config.model, + trust_remote_code=config.trust_remote_code, + use_fast=True, + ) + if not tokenizer.pad_token_id: + # Pad token required for generating consistent batch sizes + tokenizer.pad_token_id = tokenizer.eos_token_id + return tokenizer + + +def train_func(config_data: dict): + config = FinetuningJobConfig(**config_data) + model = get_model(config) + tokenizer = get_tokenizer(config) + + datasets = get_datasets(config) + training_args = get_training_args(config) + + trainer = SFTTrainer( + model=model, + tokenizer=tokenizer, + args=training_args, + peft_config=config.lora_config, + max_seq_length=config.max_seq_length, + train_dataset=datasets["train"], + eval_dataset=datasets["test"], + dataset_text_field="text", + ) + trainer.add_callback(RayTrainReportCallback()) + trainer = prepare_trainer(trainer) + trainer.train() + + # Force WandB finish on rank 0 worker + if is_wandb_enabled(config): + wandb.finish() + + +def main(config: FinetuningJobConfig): + print(f"Received job configuration: {config}") + + run_config = RunConfig( + name=config.wandb_name, + storage_path=config.storage_path, + checkpoint_config=CheckpointConfig(num_to_keep=1), + ) + trainer = TorchTrainer( + train_loop_per_worker=train_func, + train_loop_config=json.loads(config.json()), + scaling_config=config.scaling_config, + run_config=run_config, + ) + result = trainer.fit() + print(f"Training result: {result}") + + # Log additional training metrics to completed WandB run + if config.wandb_env: + result_paths = {"ray/result_path": result.path} + if result.checkpoint: + result_paths["ray/checkpoint_path"] = f"{result.checkpoint.path}/checkpoint" + update_wandb_summary(config.wandb_env, result_paths) diff --git a/src/flamingo/jobs/entrypoints/run_lm_harness.py b/src/flamingo/jobs/entrypoints/run_lm_harness.py new file mode 100644 index 00000000..cc2286b6 --- /dev/null +++ b/src/flamingo/jobs/entrypoints/run_lm_harness.py @@ -0,0 +1,118 @@ +from pathlib import Path + +import lm_eval +import ray +from lm_eval.models.huggingface import HFLM +from peft import PeftConfig +from tuner.integrations.wandb import get_wandb_summary, update_wandb_summary + +from flamingo.jobs import LMHarnessJobConfig, ModelNameOrCheckpointPath + + +def resolve_model_or_path(config: LMHarnessJobConfig) -> str: + mn_or_path = None + match config.model_name_or_path: + case None: + print("Attempting to resolve checkpoint path from existing W&B run...") + run_summary = get_wandb_summary(config.wandb_env) + cp = Path(run_summary["ray/checkpoint_path"]) + print(f"Using checkpoint path from wandb run: {cp}") + if not cp.exists(): + raise (FileNotFoundError(f"{mn_or_path} cannot be found.")) + mn_or_path = str(cp) + case ModelNameOrCheckpointPath(checkpoint=None) as x: + print("No checkpoint; will attempt to load model from HuggingFace") + mn_or_path = x.name + case ModelNameOrCheckpointPath(checkpoint=ckpt): + print(f"Checkpoint found; will attempt to load model from {ckpt}") + mn_or_path = ckpt + case _: + raise ( + ValueError( + "Something is wrong with the passed " + f"model_name_or_path: {config.model_name_or_path}" + ) + ) + return mn_or_path + + +def load_harness_model(config: LMHarnessJobConfig, model_to_load: str) -> HFLM: + # We don't know if the checkpoint is adapter weights or merged model weights + # Try to load as an adapter and fall back to the checkpoint containing the full model + try: + adapter_config = PeftConfig.from_pretrained(model_to_load) + pretrained = adapter_config.base_model_name_or_path + peft = model_to_load + except ValueError as e: + print( + f"Unable to load model as adapter: {e}. " + "This is expected if the checkpoint does not contain adapter weights." + ) + pretrained = model_to_load + peft = None + + # Return lm-harness model wrapper class + quantization_kwargs = config.quantization_config.dict() if config.quantization_config else {} + return HFLM( + pretrained=pretrained, + tokenizer=pretrained, + peft=peft, + device="cuda" if config.num_gpus > 0 else None, + trust_remote_code=config.trust_remote_code, + dtype=config.torch_dtype if config.torch_dtype else "auto", + **quantization_kwargs, + ) + + +@ray.remote +def run_evaluation(config: LMHarnessJobConfig, model_to_load: str) -> None: + print("Initializing lm-harness tasks...") + lm_eval.tasks.initialize_tasks() + + print("Running lm-harness evaluation inside remote function...") + llm = load_harness_model(config, model_to_load) + raw_results = lm_eval.simple_evaluate( + model=llm, + tasks=config.tasks, + num_fewshot=config.num_fewshot, + batch_size=config.batch_size, + limit=config.limit, + log_samples=False, + ) + print("Finished lm-harness evaluation inside remote function") + + formatted_results = {} + for task_name, metrics in raw_results["results"].items(): + task_metrics = { + f"{task_name}/{metric.replace(',', '_')}": value for metric, value in metrics.items() + } + formatted_results.update(task_metrics) + print(f"Obtained evaluation results: {formatted_results}") + + if config.wandb_env: + print("Logging results to W&B...") + update_wandb_summary(config.wandb_env, formatted_results) + + +def main(config: LMHarnessJobConfig): + print(f"Received job configuration: {config}") + + # Resolve path and ensure exists + model_to_load = resolve_model_or_path(config) + + # Using .options() to dynamically specify resource requirements + eval_func = run_evaluation.options(num_cpus=config.num_cpus, num_gpus=config.num_gpus) + eval_future = eval_func.remote(config, model_to_load) + + timeout_seconds = config.timeout.seconds if config.timeout else None + try: + print("Waiting on evaluation task...") + ray.get(eval_future, timeout=timeout_seconds) + print("Evaluation successfully completed") + except TimeoutError: + print( + f"Evaluation task timed out after {timeout_seconds} sec. " + "If the evaluation runner finished but the task failed to shut down, " + "please check if your results were still generated and persisted." + ) + raise diff --git a/src/flamingo/jobs/entrypoints/run_simple.py b/src/flamingo/jobs/entrypoints/run_simple.py new file mode 100644 index 00000000..897002ca --- /dev/null +++ b/src/flamingo/jobs/entrypoints/run_simple.py @@ -0,0 +1,5 @@ +from flamingo.jobs.configs import SimpleJobConfig + + +def main(config: SimpleJobConfig): + print(f"The magic number is {config.magic_number}") diff --git a/src/flamingo/jobs/simple_config.py b/src/flamingo/jobs/simple_config.py deleted file mode 100644 index 04cb270b..00000000 --- a/src/flamingo/jobs/simple_config.py +++ /dev/null @@ -1,15 +0,0 @@ -from flamingo.jobs import BaseJobConfig - - -class SimpleJobConfig(BaseJobConfig): - """A simple job to demonstrate the submission interface.""" - - magic_number: int - - @property - def env_vars(self) -> dict[str, str]: - return {} - - @property - def entrypoint_command(self) -> str: - return f"python simple.py --magic_number '{self.magic_number}'"