A better fix for #516.

sillsdev · Oct 28, 2024 · b270eb5 · b270eb5
1 parent ac1193f
commit b270eb5
Show file tree

Hide file tree

Showing 3 changed files with 339 additions and 25 deletions.
diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs
@@ -248,7 +248,13 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
                     Corpora =
                     {
                         parallelCorpora.Select(c =>
-                            Map(c, trainOn?.GetValueOrDefault(c.Id), pretranslate?.GetValueOrDefault(c.Id))
+                            Map(
+                                c,
+                                trainOn?.GetValueOrDefault(c.Id),
+                                pretranslate?.GetValueOrDefault(c.Id),
+                                trainOn is null,
+                                pretranslate is null
+                            )
                         )
                     }
                 };
@@ -276,7 +282,13 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
                     Corpora =
                     {
                         corpora.Select(c =>
-                            Map(c, trainOn?.GetValueOrDefault(c.Id), pretranslate?.GetValueOrDefault(c.Id))
+                            Map(
+                                c,
+                                trainOn?.GetValueOrDefault(c.Id),
+                                pretranslate?.GetValueOrDefault(c.Id),
+                                trainOn is null,
+                                pretranslate is null
+                            )
                         )
                     }
                 };
@@ -613,7 +625,13 @@ private Models.WordGraphArc Map(V1.WordGraphArc source)
         };
     }
 
-    private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, PretranslateCorpus? pretranslateCorpus)
+    private V1.ParallelCorpus Map(
+        Corpus source,
+        TrainingCorpus? trainingCorpus,
+        PretranslateCorpus? pretranslateCorpus,
+        bool noTrainingCorpusDefined,
+        bool noPretranslateCorpusDefined
+    )
     {
         IEnumerable<V1.CorpusFile> sourceFiles = source.SourceFiles.Select(Map);
         IEnumerable<V1.CorpusFile> targetFiles = source.TargetFiles.Select(Map);
@@ -622,12 +640,15 @@ private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, Pre
         V1.MonolingualCorpus targetCorpus =
             new() { Language = source.TargetLanguage, Files = { source.TargetFiles.Select(Map) } };
 
-        if (trainingCorpus is null || (trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null))
+        if (
+            noTrainingCorpusDefined
+            || (trainingCorpus is not null && trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null)
+        )
         {
             sourceCorpus.TrainOnAll = true;
             targetCorpus.TrainOnAll = true;
         }
-        else
+        else if (trainingCorpus is not null)
         {
             if (trainingCorpus.TextIds is not null && trainingCorpus.ScriptureRange is not null)
             {
@@ -663,14 +684,18 @@ private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, Pre
             }
         }
         if (
-            pretranslateCorpus is null
-            || (pretranslateCorpus.TextIds is null && pretranslateCorpus.ScriptureRange is null)
+            noPretranslateCorpusDefined
+            || (
+                pretranslateCorpus is not null
+                && pretranslateCorpus.TextIds is null
+                && pretranslateCorpus.ScriptureRange is null
+            )
         )
         {
             sourceCorpus.PretranslateAll = true;
             targetCorpus.PretranslateAll = true;
         }
-        else
+        else if (pretranslateCorpus is not null)
         {
             if (pretranslateCorpus.TextIds is not null && pretranslateCorpus.ScriptureRange is not null)
             {
@@ -713,7 +738,9 @@ pretranslateCorpus is null
     private V1.ParallelCorpus Map(
         Models.ParallelCorpus source,
         TrainingCorpus? trainingCorpus,
-        PretranslateCorpus? pretranslateCorpus
+        PretranslateCorpus? pretranslateCorpus,
+        bool noTrainingCorpusDefined,
+        bool noPretranslateCorpusDefined
     )
     {
         string? referenceFileLocation =
@@ -731,7 +758,10 @@ private V1.ParallelCorpus Map(
                         sc,
                         trainingCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(),
                         pretranslateCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(),
-                        referenceFileLocation
+                        referenceFileLocation,
+                        noTrainingCorpusDefined || (trainingCorpus is not null && trainingCorpus.SourceFilters is null),
+                        noPretranslateCorpusDefined
+                            || (pretranslateCorpus is not null && pretranslateCorpus.SourceFilters is null)
                     )
                 )
             },
@@ -742,7 +772,9 @@ private V1.ParallelCorpus Map(
                         tc,
                         trainingCorpus?.TargetFilters?.Where(sf => sf.CorpusRef == tc.Id).FirstOrDefault(),
                         null,
-                        referenceFileLocation
+                        referenceFileLocation,
+                        noTrainingCorpusDefined || (trainingCorpus is not null && trainingCorpus.TargetFilters is null),
+                        noPretranslateCorpusDefined || pretranslateCorpus is not null // there is no pretranslate Target filter.
                     )
                 )
             }
@@ -753,7 +785,9 @@ private V1.MonolingualCorpus Map(
         Models.MonolingualCorpus source,
         ParallelCorpusFilter? trainingFilter,
         ParallelCorpusFilter? pretranslateFilter,
-        string? referenceFileLocation
+        string? referenceFileLocation,
+        bool trainOnAll,
+        bool pretranslateOnAll
     )
     {
         Dictionary<string, ScriptureChapters>? trainOnChapters = null;
@@ -801,7 +835,10 @@ pretranslateFilter is not null
             Files = { source.Files.Select(Map) }
         };
 
-        if (trainingFilter is null || (trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null))
+        if (
+            trainOnAll
+            || (trainingFilter is not null && trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null)
+        )
         {
             corpus.TrainOnAll = true;
         }
@@ -814,8 +851,12 @@ pretranslateFilter is not null
         }
 
         if (
-            pretranslateFilter is null
-            || (pretranslateFilter.TextIds is null && pretranslateFilter.ScriptureRange is null)
+            pretranslateOnAll
+            || (
+                pretranslateFilter is not null
+                && pretranslateFilter.TextIds is null
+                && pretranslateFilter.ScriptureRange is null
+            )
         )
         {
             corpus.PretranslateAll = true;

diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs
@@ -125,11 +125,16 @@ public async Task NmtBatch()
         _helperClient.TranslationBuildConfig.Pretranslate = [new() { CorpusId = cId2, TextIds = ["2JN.txt"] }];
         await _helperClient.BuildEngineAsync(engineId);
         await Task.Delay(1000);
-        IList<Pretranslation> lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync(
+        IList<Pretranslation> lTrans1 = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync(
+            engineId,
+            cId1
+        );
+        Assert.That(lTrans1, Has.Count.EqualTo(0)); // should be nothing
+        IList<Pretranslation> lTrans2 = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync(
             engineId,
             cId2
         );
-        Assert.That(lTrans, Has.Count.EqualTo(13)); // just 2 John
+        Assert.That(lTrans2, Has.Count.EqualTo(13)); // just 2 John
     }
 
     [Test]