Deduplicate kbts (#551)

* Deduplicate kbts * Switch to tuple
sillsdev · Dec 3, 2024 · aa7f31f · aa7f31f
1 parent 28f3c44
commit aa7f31f
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
@@ -121,7 +121,7 @@ public async Task RunAsync_EnableKeyTerms()
             Assert.That(src1Count, Is.EqualTo(14));
             Assert.That(src2Count, Is.EqualTo(0));
             Assert.That(trgCount, Is.EqualTo(1));
-            Assert.That(termCount, Is.EqualTo(166));
+            Assert.That(termCount, Is.EqualTo(144));
         });
     }
 

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs
@@ -101,7 +101,11 @@ public async Task PreprocessAsync(
                     IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpora
                         .ChooseRandom(Seed)
                         .AlignRows(targetTermCorpora.ChooseFirst());
-                    foreach (ParallelTextRow row in parallelKeyTermsCorpus)
+                    foreach (
+                        ParallelTextRow row in parallelKeyTermsCorpus.DistinctBy(row =>
+                            (row.SourceText, row.TargetText)
+                        )
+                    )
                     {
                         await train(new Row(row.TextId, row.Refs, row.SourceText, row.TargetText, 1));
                     }