Add ability to use localizations for key terms

src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs

@@ -36,14 +36,14 @@
         return corpora;
     }
 
-    public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files)
+    public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files, string? languageCode = null)
     {
         foreach (CorpusFile file in files)
         {
             switch (file.Format)
             {
                 case FileFormat.Paratext:
-                    yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
+                    yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], languageCode: languageCode);
                     break;
             }
         }

src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs

@@ -3,5 +3,5 @@
 public interface ICorpusService
 {
     IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
-    IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files);
+    IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files, string? languageCode = null);
 }

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -50,13 +50,20 @@ protected override async Task DoWorkAsync(
         CancellationToken cancellationToken
     )
     {
+        TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
+        if (engine is null)
+            throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
+
+        bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
+        bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);
+
         (int trainCount, int pretranslateCount) = await WriteDataFilesAsync(
             buildId,
             data,
             buildOptions,
+            (engine.SourceLanguage, engine.TargetLanguage),
             cancellationToken
         );
-
         // Log summary of build data
         JsonObject buildPreprocessSummary =
             new()
@@ -65,16 +72,10 @@ CancellationToken cancellationToken
                 { "EngineId", engineId },
                 { "BuildId", buildId },
                 { "NumTrainRows", trainCount },
-                { "NumPretranslateRows", pretranslateCount }
+                { "NumPretranslateRows", pretranslateCount },
+                { "SourceLanguageResolved", srcLang },
+                { "TargetLanguageResolved", trgLang }
             };
-        TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken);
-        if (engine is null)
-            throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
-
-        bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang);
-        buildPreprocessSummary.Add("SourceLanguageResolved", srcLang);
-        bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang);
-        buildPreprocessSummary.Add("TargetLanguageResolved", trgLang);
         Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());
 
         if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel))
@@ -105,6 +106,7 @@ CancellationToken cancellationToken
         string buildId,
         IReadOnlyList<Corpus> corpora,
         string? buildOptions,
+        (string, string) languageCodes,
         CancellationToken cancellationToken
     )
     {
@@ -164,8 +166,13 @@ CancellationToken cancellationToken
 
             if ((bool?)buildOptionsObject?["use_key_terms"] ?? true)
             {
-                ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault();
-                ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault();
+                (string sourceLanguageCode, string targetLanguageCode) = languageCodes;
+                ITextCorpus? sourceTermCorpus = _corpusService
+                    .CreateTermCorpora(corpus.SourceFiles, sourceLanguageCode)
+                    .FirstOrDefault();
+                ITextCorpus? targetTermCorpus = _corpusService
+                    .CreateTermCorpora(corpus.TargetFiles, targetLanguageCode)
+                    .FirstOrDefault();
                 if (sourceTermCorpus is not null && targetTermCorpus is not null)
                 {
                     IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus);