Make write pretranslate async

sillsdev · Nov 27, 2024 · 4a0e5cb · 4a0e5cb
1 parent 244d66f
commit 4a0e5cb
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 5 deletions.
diff --git a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs
@@ -97,6 +97,7 @@ await _parallelCorpusPreprocessingService.Preprocess(
                                     Translation = row.SourceSegment
                                 }
                             );
+                            return Task.CompletedTask;
                         },
                         false
                     );

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
@@ -119,7 +119,7 @@ await _parallelCorpusPreprocessingService.Preprocess(
                 if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
                     trainCount++;
             },
-            (row, corpus) =>
+            async (row, corpus) =>
             {
                 if (row.SourceSegment.Length > 0 && row.TargetSegment.Length == 0)
                 {
@@ -135,7 +135,7 @@ await _parallelCorpusPreprocessingService.Preprocess(
                     pretranslateCount++;
                 }
                 if (pretranslateWriter.BytesPending > 1024 * 1024)
-                    pretranslateWriter.FlushAsync();
+                    await pretranslateWriter.FlushAsync();
             },
             (bool?)buildOptionsObject?["use_key_terms"] ?? true
         );

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs
@@ -7,7 +7,7 @@ public interface IParallelCorpusPreprocessingService
     Task Preprocess(
         IReadOnlyList<ParallelCorpus> corpora,
         Func<Row, Task> train,
-        Action<Row, ParallelCorpus> pretranslate,
+        Func<Row, ParallelCorpus, Task> pretranslate,
         bool useKeyTerms = false
     );
 }
diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs
@@ -28,7 +28,7 @@ internal int Seed
     public async Task Preprocess(
         IReadOnlyList<ParallelCorpus> corpora,
         Func<Row, Task> train,
-        Action<Row, ParallelCorpus> pretranslate,
+        Func<Row, ParallelCorpus, Task> pretranslate,
         bool useKeyTerms = false
     )
     {
@@ -106,7 +106,7 @@ public async Task Preprocess(
 
             foreach (Row row in CollapseRanges(pretranslateCorpus.ToArray()))
             {
-                pretranslate(row, corpus);
+                await pretranslate(row, corpus);
             }
         }
     }

diff --git a/...iceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs b/...iceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs
@@ -85,6 +85,7 @@ await processor.Preprocess(
             {
                 if (row.SourceSegment.Length > 0 && row.TargetSegment.Length == 0)
                     pretranslateCount++;
+                return Task.CompletedTask;
             },
             false
         );
-Original file line number
+Diff line change
@@ Expand Up / @@ -97,6 +97,7 @@ await _parallelCorpusPreprocessingService.Preprocess( @@
                                         Translation = row.SourceSegment
                                     }
                                 );
+                                return Task.CompletedTask;
                             },
                             false
                         );
@@ Expand Down @@