log eval settings in wandb run

src/flamingo/integrations/wandb/utils.py

@@ -6,19 +6,27 @@
 from wandb.apis.public import Run as ApiRun
 
 from flamingo.integrations.wandb import ArtifactType, WandbArtifactConfig, WandbRunConfig
+from flamingo.types import BaseFlamingoConfig
 
 
 @contextlib.contextmanager
-def wandb_init_from_config(config: WandbRunConfig, *, resume: str | None = None):
+def wandb_init_from_config(
+    config: WandbRunConfig,
+    *,
+    resume: str | None = None,
+    parameters: BaseFlamingoConfig | None = None,
+):
     """Initialize a W&B run from the internal run configuration."""
     init_kwargs = dict(
         id=config.run_id,
         name=config.name,
         project=config.project,
         entity=config.entity,
         group=config.run_group,
+        config=parameters.dict() if parameters else None,
+        resume=resume,
     )
-    with wandb.init(**init_kwargs, resume=resume) as run:
+    with wandb.init(**init_kwargs) as run:
         yield run
 
 

src/flamingo/jobs/lm_harness/entrypoint.py

@@ -78,7 +78,11 @@ def load_and_evaluate(config: LMHarnessJobConfig) -> dict[str, Any]:
 @ray.remote
 def evaluation_task(config: LMHarnessJobConfig) -> None:
     if config.tracking is not None:
-        with wandb_init_from_config(config.tracking, resume="never") as run:
+        with wandb_init_from_config(
+            config.tracking,
+            resume="allow",
+            parameters=config.evaluator,  # Log eval settings in W&B run
+        ) as run:
             eval_results = load_and_evaluate(config)
             artifact = build_evaluation_artifact(run.name, eval_results)
             run.log_artifact(artifact)