Skip to content

Commit

Permalink
Refactor logic; add test; make sure previously failing tests are passing
Browse files Browse the repository at this point in the history
  • Loading branch information
Enkidu93 committed Oct 25, 2024
1 parent af1aa56 commit a4dfb6e
Show file tree
Hide file tree
Showing 2 changed files with 385 additions and 60 deletions.
101 changes: 41 additions & 60 deletions src/Serval/src/Serval.Translation/Services/EngineService.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using MassTransit.Mediator;
using MassTransit.Mediator;
using Serval.Translation.V1;

namespace Serval.Translation.Services;
Expand Down Expand Up @@ -227,31 +227,46 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
StartBuildRequest request;
if (engine.ParallelCorpora.Any())
{
var trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!);
var pretranslate = build.Pretranslate?.ToDictionary(c => c.ParallelCorpusRef!);
Dictionary<string, TrainingCorpus>? trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!);
Dictionary<string, PretranslateCorpus>? pretranslate = build.Pretranslate?.ToDictionary(c =>
c.ParallelCorpusRef!
);
IReadOnlyList<Models.ParallelCorpus> parallelCorpora = engine
.ParallelCorpora.Where(pc =>
trainOn == null
|| trainOn.ContainsKey(pc.Id)
|| pretranslate == null
|| pretranslate.ContainsKey(pc.Id)
)
.ToList();

request = new StartBuildRequest
{
EngineType = engine.Type,
EngineId = engine.Id,
BuildId = build.Id,
Corpora =
{
engine.ParallelCorpora.Select(c =>
Map(
c,
trainOn?.GetValueOrDefault(c.Id),
pretranslate?.GetValueOrDefault(c.Id),
trainOn is null,
pretranslate is null
)
parallelCorpora.Select(c =>
Map(c, trainOn?.GetValueOrDefault(c.Id), pretranslate?.GetValueOrDefault(c.Id))
)
}
};
}
else
{
var pretranslate = build.Pretranslate?.ToDictionary(c => c.CorpusRef!);
var trainOn = build.TrainOn?.ToDictionary(c => c.CorpusRef!);
Dictionary<string, TrainingCorpus>? trainOn = build.TrainOn?.ToDictionary(c => c.CorpusRef!);
Dictionary<string, PretranslateCorpus>? pretranslate = build.Pretranslate?.ToDictionary(c =>
c.CorpusRef!
);
IReadOnlyList<Corpus> corpora = engine
.Corpora.Where(c =>
trainOn == null
|| trainOn.ContainsKey(c.Id)
|| pretranslate == null
|| pretranslate.ContainsKey(c.Id)
)
.ToList();

request = new StartBuildRequest
{
Expand All @@ -260,14 +275,8 @@ pretranslate is null
BuildId = build.Id,
Corpora =
{
engine.Corpora.Select(c =>
Map(
c,
trainOn?.GetValueOrDefault(c.Id),
pretranslate?.GetValueOrDefault(c.Id),
trainOn is null,
pretranslate is null
)
corpora.Select(c =>
Map(c, trainOn?.GetValueOrDefault(c.Id), pretranslate?.GetValueOrDefault(c.Id))
)
}
};
Expand Down Expand Up @@ -604,13 +613,7 @@ private Models.WordGraphArc Map(V1.WordGraphArc source)
};
}

private V1.ParallelCorpus Map(
Corpus source,
TrainingCorpus? trainingCorpus,
PretranslateCorpus? pretranslateCorpus,
bool noTrainingCorpusDefined,
bool noPretranslateCorpusDefined
)
private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, PretranslateCorpus? pretranslateCorpus)
{
IEnumerable<V1.CorpusFile> sourceFiles = source.SourceFiles.Select(Map);
IEnumerable<V1.CorpusFile> targetFiles = source.TargetFiles.Select(Map);
Expand All @@ -619,10 +622,7 @@ bool noPretranslateCorpusDefined
V1.MonolingualCorpus targetCorpus =
new() { Language = source.TargetLanguage, Files = { source.TargetFiles.Select(Map) } };

if (
noTrainingCorpusDefined
|| (trainingCorpus is not null && trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null)
)
if (trainingCorpus is null || (trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null))
{
sourceCorpus.TrainOnAll = true;
targetCorpus.TrainOnAll = true;
Expand Down Expand Up @@ -663,12 +663,8 @@ bool noPretranslateCorpusDefined
}
}
if (
noPretranslateCorpusDefined
|| (
pretranslateCorpus is not null
&& pretranslateCorpus.TextIds is null
&& pretranslateCorpus.ScriptureRange is null
)
pretranslateCorpus is null
|| (pretranslateCorpus.TextIds is null && pretranslateCorpus.ScriptureRange is null)
)
{
sourceCorpus.PretranslateAll = true;
Expand Down Expand Up @@ -717,9 +713,7 @@ pretranslateCorpus is not null
private V1.ParallelCorpus Map(
Models.ParallelCorpus source,
TrainingCorpus? trainingCorpus,
PretranslateCorpus? pretranslateCorpus,
bool noTrainingCorpusDefined,
bool noPretranslateCorpusDefined
PretranslateCorpus? pretranslateCorpus
)
{
string? referenceFileLocation =
Expand All @@ -737,9 +731,7 @@ bool noPretranslateCorpusDefined
sc,
trainingCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(),
pretranslateCorpus?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(),
referenceFileLocation,
noTrainingCorpusDefined,
noPretranslateCorpusDefined
referenceFileLocation
)
)
},
Expand All @@ -750,9 +742,7 @@ bool noPretranslateCorpusDefined
tc,
trainingCorpus?.TargetFilters?.Where(sf => sf.CorpusRef == tc.Id).FirstOrDefault(),
null,
referenceFileLocation,
noTrainingCorpusDefined,
noPretranslateCorpusDefined
referenceFileLocation
)
)
}
Expand All @@ -763,9 +753,7 @@ private V1.MonolingualCorpus Map(
Models.MonolingualCorpus source,
ParallelCorpusFilter? trainingFilter,
ParallelCorpusFilter? pretranslateFilter,
string? referenceFileLocation,
bool noTrainingCorpusDefined,
bool noPretranslateCorpusDefined
string? referenceFileLocation
)
{
Dictionary<string, ScriptureChapters>? trainOnChapters = null;
Expand Down Expand Up @@ -813,10 +801,7 @@ pretranslateFilter is not null
Files = { source.Files.Select(Map) }
};

if (
noTrainingCorpusDefined
|| (trainingFilter is not null && trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null)
)
if (trainingFilter is null || (trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null))
{
corpus.TrainOnAll = true;
}
Expand All @@ -829,12 +814,8 @@ pretranslateFilter is not null
}

if (
noPretranslateCorpusDefined
|| (
pretranslateFilter is not null
&& pretranslateFilter.TextIds is null
&& pretranslateFilter.ScriptureRange is null
)
pretranslateFilter is null
|| (pretranslateFilter.TextIds is null && pretranslateFilter.ScriptureRange is null)
)
{
corpus.PretranslateAll = true;
Expand Down
Loading

0 comments on commit a4dfb6e

Please sign in to comment.