Merge pull request #85 from mozilla-ai/update_harness

Update lm-eval version and API

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lm-buddy"
-version = "0.3.0"
+version = "0.3.1"
 authors = [
     { name = "Sean Friedowitz", email = "[email protected]" },
     { name = "Aaron Gonzales", email = "[email protected]" },
@@ -29,14 +29,14 @@ dependencies = [
     "pydantic-yaml==1.2.0",
     "ray[default]==2.9.3",
     # HuggingFace
-    "datasets>=2.16.1",
+    "datasets>=2.17.1",
     "transformers==4.36.2",
     "accelerate==0.26.1",
     "peft==0.7.1",
     "trl==0.7.10",
     "bitsandbytes==0.42.0",
     # Evaluation frameworks
-    "lm-eval==0.4.1",
+    "lm-eval==0.4.2",
     "einops==0.7.0",
     "fschat==0.2.36",
     "openai==1.3.9",

src/lm_buddy/jobs/_entrypoints/lm_harness.py

@@ -86,8 +86,6 @@ def load_and_evaluate(
     config: LMHarnessJobConfig,
     artifact_loader: ArtifactLoader,
 ) -> dict[str, list[tuple[str, float]]]:
-    print("Initializing lm-harness tasks...")
-    lm_eval.tasks.initialize_tasks()
 
     llm = load_harness_model(config, artifact_loader)
     eval_results = lm_eval.simple_evaluate(