From bae01c16e1e09596745110ee6506df84e45997a0 Mon Sep 17 00:00:00 2001 From: Davide Eynard Date: Tue, 6 Aug 2024 15:32:03 +0100 Subject: [PATCH 1/8] Updated pinned libs in pyproject.toml --- pyproject.toml | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fb5bac0..02b26d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,47 +26,45 @@ dependencies = [ "wandb>=0.16.3", "protobuf>=3.20.2", "urllib3>=1.26.18,<2", - "pydantic>=2.6.4", - "pydantic-yaml>=1.2.0", "pydantic==2.6.4", "pydantic-yaml==1.2.0", "ray[default]==2.30.0", "loguru==0.7.2", - "s3fs", + "s3fs==2024.5.0", ] [project.optional-dependencies] -ruff = ["ruff==0.2.1"] +ruff = ["ruff==0.5.6"] jobs = [ # HuggingFace / pytorch - "torch==2.3.1", - "datasets>=2.17.1", - "transformers==4.38.0", - "accelerate==0.26.1", - "peft==0.7.1", - "trl==0.7.10", + "torch==2.4.0", + "datasets>=2.20.0", + "transformers==4.43.4", + "accelerate==0.33.0", + "peft==0.12.0", + "trl==0.9.6", "bitsandbytes==0.42.0", "bert_score==0.3.13", # Evaluation frameworks - "lm-eval==0.4.2", - "einops==0.7.0", - "openai==1.14.3", - "ragas==0.1.5", - "langchain-community==0.0.29", - "langchain_openai==0.1.1", + "lm-eval==0.4.3", + "einops==0.8.0", + "openai==1.38.0", + "ragas==0.1.12", + "langchain-community==0.2.11", + "langchain_openai==0.1.20", "sentencepiece==0.2.0", "evaluate==0.4.2", "mistralai==0.4.2", ] -test = ["pytest==7.4.3", "pytest-cov==4.1.0"] +test = ["pytest==8.3.2", "pytest-cov==5.0.0"] docs = [ - "Sphinx==7.2.6", - "nbsphinx==0.9.3", - "sphinx-codeautolink==0.15.0", - "myst-parser==2.0.0", - "furo==2024.1.29", + "Sphinx==7.4.7", + "nbsphinx==0.9.4", + "sphinx-codeautolink==0.15.2", + "myst-parser==3.0.1", + "furo==2024.7.18", ] -dev = ["lm-buddy[jobs,ruff,test,docs]", "pre-commit==3.6.0", "jupyter==1.0.0"] +dev = ["lm-buddy[jobs,ruff,test,docs]", "pre-commit==3.8.0", "jupyter==1.0.0"] [project.scripts] lm-buddy = "lm_buddy.cli:cli" @@ -82,7 +80,7 @@ addopts = "-v --cov src --no-cov-on-fail --disable-warnings" testpaths = ["tests"] [tool.ruff] -target-version = "py310" +target-version = "py311" exclude = [ ".bzr", From 2a7609a2b8f7aebbf0d68db953eaff365b66352c Mon Sep 17 00:00:00 2001 From: Davide Eynard Date: Tue, 6 Aug 2024 15:49:59 +0100 Subject: [PATCH 2/8] Updated ruff in configs --- .github/workflows/merge_checks.yaml | 2 +- .pre-commit-config.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge_checks.yaml b/.github/workflows/merge_checks.yaml index 7970174..3eba1d9 100644 --- a/.github/workflows/merge_checks.yaml +++ b/.github/workflows/merge_checks.yaml @@ -40,7 +40,7 @@ jobs: id: ruff run: | . .venv/bin/activate - ruff --output-format=github . + ruff --output-format=github check . continue-on-error: false tests: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65b30f1..988c40a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,12 @@ repos: - id: requirements-txt-fixer - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.2.1 + rev: v0.5.6 hooks: - id: ruff args: [--exit-non-zero-on-fix] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.2.1 + rev: v0.5.6 hooks: - id: ruff-format From dc1d3196296c22792ea35c3ce6750f5fe99d2787 Mon Sep 17 00:00:00 2001 From: Davide Eynard Date: Tue, 6 Aug 2024 15:52:16 +0100 Subject: [PATCH 3/8] Updated ruff cmdline --- .github/workflows/merge_checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_checks.yaml b/.github/workflows/merge_checks.yaml index 3eba1d9..8ed8be0 100644 --- a/.github/workflows/merge_checks.yaml +++ b/.github/workflows/merge_checks.yaml @@ -40,7 +40,7 @@ jobs: id: ruff run: | . .venv/bin/activate - ruff --output-format=github check . + ruff check --output-format=github . continue-on-error: false tests: From 4fe9823886f1a7bcf54fa1c6a46baac6b218240e Mon Sep 17 00:00:00 2001 From: Davide Eynard Date: Tue, 6 Aug 2024 16:05:55 +0100 Subject: [PATCH 4/8] Bumped version, fixed ruff E721 --- pyproject.toml | 2 +- src/lm_buddy/jobs/evaluation/hf_evaluate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 02b26d1..d37edaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lm-buddy" -version = "0.12.1" +version = "0.13.0" authors = [ { name = "Sean Friedowitz", email = "sean@mozilla.ai" }, { name = "Aaron Gonzales", email = "aaron@mozilla.ai" }, diff --git a/src/lm_buddy/jobs/evaluation/hf_evaluate.py b/src/lm_buddy/jobs/evaluation/hf_evaluate.py index 71de901..dcd4a84 100644 --- a/src/lm_buddy/jobs/evaluation/hf_evaluate.py +++ b/src/lm_buddy/jobs/evaluation/hf_evaluate.py @@ -102,7 +102,7 @@ def run_eval(config: HuggingFaceEvalJobConfig) -> Path: dataset_iterable = tqdm(input_samples) if config.evaluation.enable_tqdm else input_samples # Choose which model client to use - if type(config.model) == VLLMCompletionsConfig: + if isinstance(config.model, VLLMCompletionsConfig): model_name = config.model.inference.base_url output_model_name = config.model.inference.engine if "mistral" in model_name: From 53c0d6e3069d7b2448d3e0d1ed6fa88c2dad8fce Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 6 Aug 2024 11:44:25 -0600 Subject: [PATCH 5/8] updating remote code passthrough location --- pyproject.toml | 8 ++++---- src/lm_buddy/configs/jobs/lm_harness.py | 1 + src/lm_buddy/jobs/evaluation/lm_harness.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d37edaa..6bfd9a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,15 +26,15 @@ dependencies = [ "wandb>=0.16.3", "protobuf>=3.20.2", "urllib3>=1.26.18,<2", - "pydantic==2.6.4", - "pydantic-yaml==1.2.0", + "pydantic>=2.6.4", + "pydantic-yaml>=1.2.0", "ray[default]==2.30.0", "loguru==0.7.2", "s3fs==2024.5.0", ] [project.optional-dependencies] -ruff = ["ruff==0.5.6"] +ruff = ["ruff==0.5.5"] jobs = [ # HuggingFace / pytorch "torch==2.4.0", @@ -64,7 +64,7 @@ docs = [ "myst-parser==3.0.1", "furo==2024.7.18", ] -dev = ["lm-buddy[jobs,ruff,test,docs]", "pre-commit==3.8.0", "jupyter==1.0.0"] +dev = ["lm-buddy[jobs,ruff,test,docs]", "pre-commit==3.8.0", "jupyter>=1.0.0"] [project.scripts] lm-buddy = "lm_buddy.cli:cli" diff --git a/src/lm_buddy/configs/jobs/lm_harness.py b/src/lm_buddy/configs/jobs/lm_harness.py index 41830b9..82cf62a 100644 --- a/src/lm_buddy/configs/jobs/lm_harness.py +++ b/src/lm_buddy/configs/jobs/lm_harness.py @@ -47,6 +47,7 @@ class LMHarnessJobConfig(JobConfig): model: AutoModelConfig | LocalChatCompletionsConfig evaluation: LMHarnessEvaluationConfig quantization: QuantizationConfig | None = None + trust_remote_code: bool = False def asset_paths(self) -> list[AssetPath]: match self.model: diff --git a/src/lm_buddy/jobs/evaluation/lm_harness.py b/src/lm_buddy/jobs/evaluation/lm_harness.py index ec6f63f..b78202b 100644 --- a/src/lm_buddy/jobs/evaluation/lm_harness.py +++ b/src/lm_buddy/jobs/evaluation/lm_harness.py @@ -50,7 +50,7 @@ def load_harness_model(config: LMHarnessJobConfig) -> HFLM | OpenaiCompletionsLM tokenizer=model_path, peft=peft_path, device="cuda" if torch.cuda.device_count() > 0 else "cpu", - trust_remote_code=config.model.trust_remote_code, + trust_remote_code=config.trust_remote_code, dtype=config.model.torch_dtype if config.model.torch_dtype else "auto", **quantization_kwargs, ) From e0ea16eadb610b8c85e5e176c308ea07775f3db7 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 6 Aug 2024 11:55:26 -0600 Subject: [PATCH 6/8] changing eval test task to GLUE --- tests/integration/test_lm_harness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lm_harness.py b/tests/integration/test_lm_harness.py index be7e279..191b724 100644 --- a/tests/integration/test_lm_harness.py +++ b/tests/integration/test_lm_harness.py @@ -11,7 +11,7 @@ def job_config(llm_model_path) -> LMHarnessJobConfig: model_config = AutoModelConfig(path=format_file_path(llm_model_path)) tracking_config = WandbRunConfig(project="test-project") - evaluation_config = LMHarnessEvaluationConfig(tasks=["hellaswag"], limit=5) + evaluation_config = LMHarnessEvaluationConfig(tasks=["glue"], limit=5) return LMHarnessJobConfig( name="test-job", model=model_config, @@ -23,5 +23,5 @@ def job_config(llm_model_path) -> LMHarnessJobConfig: def test_lm_harness_job(job_config): buddy = LMBuddy() result = buddy.evaluate(job_config) - assert len(result.tables) == 1 # One table for hellaswag + assert len(result.tables) == 10 assert len(result.artifacts) == 1 # One table artifact From 943f9823e38c8fcf5b141be0f4714b0014181981 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 6 Aug 2024 12:49:24 -0600 Subject: [PATCH 7/8] reverting trust_remote_code changes to address in a distinct pr --- src/lm_buddy/configs/huggingface.py | 2 +- src/lm_buddy/configs/jobs/lm_harness.py | 1 - src/lm_buddy/jobs/evaluation/lm_harness.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lm_buddy/configs/huggingface.py b/src/lm_buddy/configs/huggingface.py index 1a5a85a..18e9ac1 100644 --- a/src/lm_buddy/configs/huggingface.py +++ b/src/lm_buddy/configs/huggingface.py @@ -100,7 +100,7 @@ def validate_adapter_args(cls, config: "AdapterConfig"): # Filter fields to those found on the PeftConfig extra_fields = config.model_fields_set.difference(allowed_fields) if extra_fields: - raise ValueError(f"Unknowon arguments for {peft_type} adapter: {extra_fields}") + raise ValueError(f"Unknown arguments for {peft_type} adapter: {extra_fields}") return config diff --git a/src/lm_buddy/configs/jobs/lm_harness.py b/src/lm_buddy/configs/jobs/lm_harness.py index 82cf62a..41830b9 100644 --- a/src/lm_buddy/configs/jobs/lm_harness.py +++ b/src/lm_buddy/configs/jobs/lm_harness.py @@ -47,7 +47,6 @@ class LMHarnessJobConfig(JobConfig): model: AutoModelConfig | LocalChatCompletionsConfig evaluation: LMHarnessEvaluationConfig quantization: QuantizationConfig | None = None - trust_remote_code: bool = False def asset_paths(self) -> list[AssetPath]: match self.model: diff --git a/src/lm_buddy/jobs/evaluation/lm_harness.py b/src/lm_buddy/jobs/evaluation/lm_harness.py index b78202b..334a160 100644 --- a/src/lm_buddy/jobs/evaluation/lm_harness.py +++ b/src/lm_buddy/jobs/evaluation/lm_harness.py @@ -50,7 +50,7 @@ def load_harness_model(config: LMHarnessJobConfig) -> HFLM | OpenaiCompletionsLM tokenizer=model_path, peft=peft_path, device="cuda" if torch.cuda.device_count() > 0 else "cpu", - trust_remote_code=config.trust_remote_code, + trust_remote_code=model_config.trust_remote_code, dtype=config.model.torch_dtype if config.model.torch_dtype else "auto", **quantization_kwargs, ) From cf07462840bff79d8a0e2fdc73f88a667a7a8ce4 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 6 Aug 2024 12:57:04 -0600 Subject: [PATCH 8/8] updated contributing guide --- CONTRIBUTING.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82f3683..bdecb48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,23 @@ development dependencies. You can execute Ruff by calling `ruff --fix .` or `ruff format .` from the workspace root. Ruff will pick up the configuration defined in the `pyproject.toml` file automatically. + +## Updating deps +Update deps in `pyproject.toml` and ensure you can `pip-compile` them as follows: + +pip-tools: +``` +pip install pip-tools +pip-compile -o requirements.lock --all-extras pyproject.toml +``` + +uv: +``` +pip install uv +uv pip compile --python-version=3.11.9 --all-extras --universal -o requirements.lock pyproject.toml +``` + + ## Testing a development branch on Ray LM Buddy is intended to be installed as a pip requirement in the runtime environment of a Ray job. @@ -40,11 +57,11 @@ as shown below: ``` # pip-tools pip install pip-tools - pip-compile -o requirements.txt pyproject.toml + pip-compile -o requirements.txt --all-extras pyproject.toml # uv pip install uv - uv pip compile -o requirements.txt pyproject.toml + uv pip compile --python-version=3.11.9 --all-extras --universal -o requirements.txt pyproject.toml ``` 2. When submitting a job to a Ray cluster, specify in the Ray runtime environment the following: