Merge pull request #592 from sillsdev/config_model_cleanup

Clean up use of config.model
sillsdev · Nov 14, 2024 · 3fd6b81 · 3fd6b81
2 parents 5d935da + b475b12
commit 3fd6b81
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 13 deletions.
diff --git a/silnlp/nmt/analyze_project_pairs.py b/silnlp/nmt/analyze_project_pairs.py
@@ -106,12 +106,8 @@ def get_corpus_stats(config: Config, force_align: bool = False, deutero: bool =
 
             src_script = predict_script_code("".join(corpus["source"][: min(len(corpus["source"]), 3000)]))
             trg_script = predict_script_code("".join(corpus["target"][: min(len(corpus["target"]), 3000)]))
-            try:
-                src_script_in_model = is_represented(src_script, config.model)
-                trg_script_in_model = is_represented(trg_script, config.model)
-            except:
-                src_script_in_model = None
-                trg_script_in_model = None
+            src_script_in_model = is_represented(src_script, config.model)
+            trg_script_in_model = is_represented(trg_script, config.model)
 
             stats_df.loc[project_pair, :] = [
                 pair_count,
@@ -210,12 +206,8 @@ def get_extra_alignments(config: Config, deutero: bool = False) -> List[str]:
             parallel_count = len(align_corpus.index)
             src_script = predict_script_code("".join(align_corpus["source"][: min(len(align_corpus["source"]), 3000)]))
             trg_script = predict_script_code("".join(align_corpus["target"][: min(len(align_corpus["target"]), 3000)]))
-            try:
-                src_script_in_model = is_represented(src_script, config.model)
-                trg_script_in_model = is_represented(trg_script, config.model)
-            except:
-                src_script_in_model = None
-                trg_script_in_model = None
+            src_script_in_model = is_represented(src_script, config.model)
+            trg_script_in_model = is_represented(trg_script, config.model)
 
             stats_df.loc[project_pair, :] = [
                 pair_count,

diff --git a/silnlp/nmt/config_utils.py b/silnlp/nmt/config_utils.py
@@ -18,5 +18,4 @@ def load_config(exp_name: str) -> Config:
 
 
 def create_config(exp_dir: Path, config: dict) -> Config:
-    model_name: Optional[str] = config.get("model")
     return HuggingFaceConfig(exp_dir, config)
diff --git a/silnlp/nmt/hugging_face_config.py b/silnlp/nmt/hugging_face_config.py
@@ -319,6 +319,7 @@ def __init__(self, exp_dir: Path, config: dict) -> None:
                     "lr_scheduler_type": "cosine",
                     "attention_implementation": "eager",
                 },
+                "model": "facebook/nllb-200-distilled-1.3B",
             },
             config,
         )
Original file line number	Diff line number	Diff line change
Expand Up		@@ -18,5 +18,4 @@ def load_config(exp_name: str) -> Config:


		def create_config(exp_dir: Path, config: dict) -> Config:
		model_name: Optional[str] = config.get("model")
		return HuggingFaceConfig(exp_dir, config)
-Original file line number
+Diff line change
@@ Expand Up / @@ -319,6 +319,7 @@ def __init__(self, exp_dir: Path, config: dict) -> None: @@
                         "lr_scheduler_type": "cosine",
                         "attention_implementation": "eager",
                     },
+                    "model": "facebook/nllb-200-distilled-1.3B",
                 },
                 config,
             )
@@ Expand Down @@