fix merge

stanford-crfm · Feb 9, 2024 · ddba7fd · ddba7fd
1 parent c5fb7a6
commit ddba7fd
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 46 deletions.
diff --git a/config/gpt2_large_sophia_g.yaml b/config/gpt2_large_sophia_g.yaml
diff --git a/config/gpt2_small_fast_sophia_g.yaml b/config/gpt2_small_fast_sophia_g.yaml
diff --git a/src/levanter/doremi.py b/src/levanter/doremi.py
@@ -185,7 +185,7 @@ def doremi_step(state: DoremiState, ref, batch, domains):
                 "train/mean_proxy_loss": mean_proxy_loss,
                 **{f"alpha/{domain}": weight for domain, weight in alpha_dict.items()},
                 # just skip domains with no excess loss
-                **{f"train/{domain}/excess_loss": loss for domain, loss in per_domain_dict.items() if loss > 0},
+                **{f"train/{domain}/excess_loss": loss for domain, loss in per_domain_dict.items()},
             },
             step=state._step,
         )