From ac8bc211e12ff23a9bc5c06e0b4a85141127a98e Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 5 Nov 2024 16:59:13 -0500 Subject: [PATCH] it compiles. --- .../Services/ClearMLMonitorService.cs | 50 +++++++++++-------- .../Services/HangfireBuildJobRunner.cs | 9 ++-- .../Services/IWordAlignmentEngineService.cs | 1 + .../Services/NmtEngineService.cs | 4 +- .../Services/NmtHangfireBuildJobFactory.cs | 14 ++++-- .../Services/PostprocessBuildJob.cs | 11 +++- .../Services/PreprocessBuildJob.cs | 2 +- .../ServalWordAlignmentEngineServiceV1.cs | 1 + .../Services/SmtTransferEngineService.cs | 4 +- .../SmtTransferHangfireBuildJobFactory.cs | 12 +++-- .../SmtTransferPostprocessBuildJob.cs | 2 +- .../Services/SmtTransferTrainBuildJob.cs | 2 +- .../Services/StatisticalEngineService.cs | 39 +++++++++++---- .../Services/NmtEngineServiceTests.cs | 4 +- .../Services/PreprocessBuildJobTests.cs | 8 +-- .../Services/SmtTransferEngineServiceTests.cs | 9 ++-- .../Services/PretranslationService.cs | 18 +++---- .../Configuration/IServalBuilderExtensions.cs | 19 ++----- .../Services/ScriptureDataFileServiceTests.cs | 2 +- 19 files changed, 119 insertions(+), 92 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs index 4f04e5d1..644320aa 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -40,11 +40,10 @@ public int GetQueueSize(EngineType engineType) protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) { - await MonitorClearMLTasksPerDomain(scope, cancellationToken); + await MonitorClearMLTasksPerDomain(scope, cancellationToken); } - private async Task MonitorClearMLTasksPerDomain(IServiceScope scope, CancellationToken cancellationToken) - where TEngine : Enum + private async Task MonitorClearMLTasksPerDomain(IServiceScope scope, CancellationToken cancellationToken) { try { @@ -55,22 +54,26 @@ private async Task MonitorClearMLTasksPerDomain(IServiceScope scope, Ca IBuildJobService >(); - Dictionary> trainingEngines = ( + Dictionary> engineToBuildServiceDict = ( await translationBuildJobService.GetBuildingEnginesAsync(BuildJobRunnerType.ClearML, cancellationToken) - ).ToDictionary(e => e, e => translationBuildJobService as IBuildJobService); + ).ToDictionary(e => e, e => translationBuildJobService); - trainingEngines.AddRange( - await wordAlignmentBuildJobService.GetBuildingEnginesAsync( + foreach ( + var engine in await wordAlignmentBuildJobService.GetBuildingEnginesAsync( BuildJobRunnerType.ClearML, cancellationToken ) - ); - if (trainingEngines.Count == 0) + ) + { + engineToBuildServiceDict[engine] = (IBuildJobService)wordAlignmentBuildJobService; + } + + if (engineToBuildServiceDict.Count == 0) return; Dictionary tasks = ( await _clearMLService.GetTasksByIdAsync( - trainingEngines.Select(e => e.CurrentBuild!.JobId), + engineToBuildServiceDict.Select(e => e.Key.CurrentBuild!.JobId), cancellationToken ) ).ToDictionary(t => t.Id); @@ -80,8 +83,11 @@ await _clearMLService.GetTasksByIdAsync( { var tasksPerEngineType = tasks .Where(kvp => - trainingEngines.Where(te => te.CurrentBuild?.JobId == kvp.Key).FirstOrDefault()?.Type.ToString() - == engineType + engineToBuildServiceDict + .Where(te => te.Key.CurrentBuild?.JobId == kvp.Key) + .FirstOrDefault() + .Key?.Type + .ToString() == engineType ) .Select(kvp => kvp.Value) .UnionBy(await _clearMLService.GetTasksForQueueAsync(queueName, cancellationToken), t => t.Id) @@ -98,7 +104,7 @@ await _clearMLService.GetTasksByIdAsync( var dataAccessContext = scope.ServiceProvider.GetRequiredService(); var platformService = scope.ServiceProvider.GetRequiredService(); - foreach (ITrainingEngine engine in trainingEngines) + foreach (ITrainingEngine engine in engineToBuildServiceDict.Keys) { if (engine.CurrentBuild is null || !tasks.TryGetValue(engine.CurrentBuild.JobId, out ClearMLTask? task)) continue; @@ -131,7 +137,7 @@ or ClearMLTaskStatus.Completed { bool canceled = !await TrainJobStartedAsync( dataAccessContext, - buildJobService, + engineToBuildServiceDict[engine], platformService, engine.EngineId, engine.CurrentBuild.BuildId, @@ -170,7 +176,7 @@ await UpdateTrainJobStatus( cancellationToken ); bool canceling = !await TrainJobCompletedAsync( - buildJobService, + engineToBuildServiceDict[engine], engine.Type, engine.EngineId, engine.CurrentBuild.BuildId, @@ -183,7 +189,7 @@ await UpdateTrainJobStatus( { await TrainJobCanceledAsync( dataAccessContext, - buildJobService, + engineToBuildServiceDict[engine], platformService, engine.EngineId, engine.CurrentBuild.BuildId, @@ -197,7 +203,7 @@ await TrainJobCanceledAsync( { await TrainJobCanceledAsync( dataAccessContext, - buildJobService, + engineToBuildServiceDict[engine], platformService, engine.EngineId, engine.CurrentBuild.BuildId, @@ -210,7 +216,7 @@ await TrainJobCanceledAsync( { await TrainJobFaultedAsync( dataAccessContext, - buildJobService, + engineToBuildServiceDict[engine], platformService, engine.EngineId, engine.CurrentBuild.BuildId, @@ -231,7 +237,7 @@ await TrainJobFaultedAsync( private async Task TrainJobStartedAsync( IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, IPlatformService platformService, string engineId, string buildId, @@ -254,7 +260,7 @@ private async Task TrainJobStartedAsync( } private async Task TrainJobCompletedAsync( - IBuildJobService buildJobService, + IBuildJobService buildJobService, EngineType engineType, string engineId, string buildId, @@ -285,7 +291,7 @@ CancellationToken cancellationToken private async Task TrainJobFaultedAsync( IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, IPlatformService platformService, string engineId, string buildId, @@ -318,7 +324,7 @@ await buildJobService.BuildJobFinishedAsync( private async Task TrainJobCanceledAsync( IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, IPlatformService platformService, string engineId, string buildId, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs index b414e985..fc91536a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs @@ -1,18 +1,18 @@ namespace Serval.Machine.Shared.Services; -public class HangfireBuildJobRunner( +public class HangfireBuildJobRunner( IBackgroundJobClient jobClient, IEnumerable buildJobFactories ) : IBuildJobRunner - where TEngine : ITrainingEngine { - public static Job CreateJob( + public static Job CreateJob( string engineId, string buildId, string queue, object? data, string? buildOptions ) + where TEngine : ITrainingEngine where TJob : HangfireBuildJob { ArgumentNullException.ThrowIfNull(data); @@ -23,7 +23,8 @@ public static Job CreateJob( ); } - public static Job CreateJob(string engineId, string buildId, string queue, string? buildOptions) + public static Job CreateJob(string engineId, string buildId, string queue, string? buildOptions) + where TEngine : ITrainingEngine where TJob : HangfireBuildJob { // Token "None" is used here because hangfire injects the proper cancellation token diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs index 867857b3..c436897a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs @@ -14,6 +14,7 @@ Task CreateAsync( Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); Task GetBestPhraseAlignmentAsync( + string engineId, string sourceSegment, string targetSegment, CancellationToken cancellationToken = default diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs index 3b1cfec4..56845e1e 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs @@ -4,7 +4,7 @@ public class NmtEngineService( IPlatformService platformService, IDataAccessContext dataAccessContext, IRepository engines, - IBuildJobService buildJobService, + IBuildJobService buildJobService, ILanguageTagService languageTagService, IClearMLQueueService clearMLQueueService, ISharedFileService sharedFileService @@ -13,7 +13,7 @@ ISharedFileService sharedFileService private readonly IPlatformService _platformService = platformService; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IRepository _engines = engines; - private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; private readonly ILanguageTagService _languageTagService = languageTagService; private readonly ISharedFileService _sharedFileService = sharedFileService; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs index 35fcdc1c..0c54d65b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs @@ -1,4 +1,6 @@ -namespace Serval.Machine.Shared.Services; +using static Serval.Machine.Shared.Services.HangfireBuildJobRunner; + +namespace Serval.Machine.Shared.Services; public class NmtHangfireBuildJobFactory : IHangfireBuildJobFactory { @@ -9,7 +11,7 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? return stage switch { BuildStage.Preprocess - => CreateJob>( + => CreateJob>( engineId, buildId, "nmt", @@ -17,7 +19,13 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? buildOptions ), BuildStage.Postprocess - => CreateJob(engineId, buildId, "nmt", data, buildOptions), + => CreateJob( + engineId, + buildId, + "nmt", + data, + buildOptions + ), _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), }; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs index c9922e41..794487c5 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs @@ -4,11 +4,18 @@ public class PostprocessBuildJob( IPlatformService platformService, IRepository engines, IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService, IOptionsMonitor options -) : HangfireBuildJob<(int, double)>(platformService, engines, dataAccessContext, buildJobService, logger) +) + : HangfireBuildJob( + platformService, + engines, + dataAccessContext, + buildJobService, + logger + ) { protected ISharedFileService SharedFileService { get; } = sharedFileService; private readonly BuildJobOptions _buildJobOptions = options.CurrentValue; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 54a99ea1..8453cf51 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -49,7 +49,7 @@ protected override async Task DoWorkAsync( CancellationToken cancellationToken ) { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + TEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); if (engine is null) throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs index 21d208df..5ef409ab 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs @@ -42,6 +42,7 @@ ServerCallContext context try { result = await engineService.GetBestPhraseAlignmentAsync( + request.EngineId, request.SourceSegment, request.TargetSegment, context.CancellationToken diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs index 88f388ac..670934cb 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs @@ -7,7 +7,7 @@ public class SmtTransferEngineService( IRepository engines, IRepository trainSegmentPairs, SmtTransferEngineStateService stateService, - IBuildJobService buildJobService, + IBuildJobService buildJobService, IClearMLQueueService clearMLQueueService ) : ITranslationEngineService { @@ -17,7 +17,7 @@ IClearMLQueueService clearMLQueueService private readonly IRepository _engines = engines; private readonly IRepository _trainSegmentPairs = trainSegmentPairs; private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; public EngineType Type => EngineType.SmtTransfer; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs index 18fda0fc..65e13e24 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs @@ -11,7 +11,7 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? return stage switch { BuildStage.Preprocess - => CreateJob>( + => CreateJob>( engineId, buildId, "smt_transfer", @@ -19,14 +19,20 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? buildOptions ), BuildStage.Postprocess - => CreateJob( + => CreateJob( engineId, buildId, "smt_transfer", data, buildOptions ), - BuildStage.Train => CreateJob(engineId, buildId, "smt_transfer", buildOptions), + BuildStage.Train + => CreateJob( + engineId, + buildId, + "smt_transfer", + buildOptions + ), _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), }; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs index 8b30dc6d..9e26fcdb 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs @@ -4,7 +4,7 @@ public class SmtTransferPostprocessBuildJob( IPlatformService platformService, IRepository engines, IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService, IDistributedReaderWriterLockFactory lockFactory, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs index 657198e1..81db5131 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs @@ -4,7 +4,7 @@ public class SmtTransferTrainBuildJob( IPlatformService platformService, IRepository engines, IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, + IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService, ITruecaserFactory truecaserFactory, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs index 8ccb46c4..5649dc55 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs @@ -6,7 +6,7 @@ public class StatisticalEngineService( IDataAccessContext dataAccessContext, IRepository engines, SmtTransferEngineStateService stateService, - IBuildJobService buildJobService, + IBuildJobService buildJobService, IClearMLQueueService clearMLQueueService ) : IWordAlignmentEngineService { @@ -15,7 +15,7 @@ IClearMLQueueService clearMLQueueService private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IRepository _engines = engines; private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IBuildJobService _buildJobService = buildJobService; private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; public EngineType Type => EngineType.Statistical; @@ -50,6 +50,33 @@ public async Task CreateAsync( return wordAlignmentEngine; } + public async Task GetBestPhraseAlignmentAsync( + string engineId, + string sourceSegment, + string targetSegment, + CancellationToken cancellationToken = default + ) + { + WordAlignmentEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + SmtTransferEngineState state = _stateService.Get(engineId); + + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + TranslationResult result = await @lock.ReaderLockAsync( + async ct => + { + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision, ct); + // there is no way to cancel this call + return hybridEngine.GetBestPhraseAlignment(sourceSegment, targetSegment); + }, + cancellationToken: cancellationToken + ); + + state.Touch(); + return result; + + throw new NotImplementedException(); + } + public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) { await CancelBuildJobAsync(engineId, cancellationToken); @@ -127,14 +154,6 @@ await _dataAccessContext.WithTransactionAsync( return buildId is not null; } - public Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) { WordAlignmentEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs index d6baa2a4..88989842 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs @@ -149,7 +149,7 @@ public TestEnvironment() ); BuildJobService = new BuildJobService( [ - new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]), + new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]), new ClearMLBuildJobRunner( ClearMLService, [ @@ -184,7 +184,7 @@ public TestEnvironment() public IPlatformService PlatformService { get; } public IClearMLService ClearMLService { get; } public ISharedFileService SharedFileService { get; } - public IBuildJobService BuildJobService { get; } + public IBuildJobService BuildJobService { get; } public void StopServer() { diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index d0f01020..fed897c2 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -505,7 +505,7 @@ private class TestEnvironment : DisposableBase public MemoryRepository Engines { get; } public MemoryRepository TrainSegmentPairs { get; } public IDistributedReaderWriterLockFactory LockFactory { get; } - public IBuildJobService BuildJobService { get; } + public IBuildJobService BuildJobService { get; } public IClearMLService ClearMLService { get; } public IOptionsMonitor BuildJobOptions { get; } @@ -756,7 +756,7 @@ public TestEnvironment() SharedFileService = new SharedFileService(Substitute.For()); BuildJobService = new BuildJobService( [ - new HangfireBuildJobRunner( + new HangfireBuildJobRunner( Substitute.For(), [new NmtHangfireBuildJobFactory()] ), @@ -776,7 +776,7 @@ [new NmtHangfireBuildJobFactory()] ); } - public PreprocessBuildJob GetBuildJob(EngineType engineType) + public PreprocessBuildJob GetBuildJob(EngineType engineType) { switch (engineType) { @@ -802,7 +802,7 @@ public PreprocessBuildJob GetBuildJob(EngineType engineType) PlatformService, Engines, new MemoryDataAccessContext(), - Substitute.For>(), + Substitute.For>(), BuildJobService, SharedFileService, CorpusService, diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs index 14828231..20859115 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs @@ -321,10 +321,7 @@ public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerTyp ); BuildJobService = new BuildJobService( [ - new HangfireBuildJobRunner( - _jobClient, - [new SmtTransferHangfireBuildJobFactory()] - ), + new HangfireBuildJobRunner(_jobClient, [new SmtTransferHangfireBuildJobFactory()]), new ClearMLBuildJobRunner( ClearMLService, [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], @@ -355,7 +352,7 @@ [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], public ISharedFileService SharedFileService { get; } - public IBuildJobService BuildJobService { get; } + public IBuildJobService BuildJobService { get; } public async Task CommitAsync(TimeSpan inactiveTimeout) { @@ -687,7 +684,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, new MemoryDataAccessContext(), - Substitute.For>(), + Substitute.For>(), _env.BuildJobService, _env.SharedFileService, Substitute.For(), diff --git a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs index 1bf552fb..516e634e 100644 --- a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs +++ b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs @@ -100,8 +100,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) textId, pretranslations.ToList(), fullName: targetSettings.FullName, - stripAllText: false, - preferExistingText: true + behavior: UpdateUsfmBehavior.PreferExisting ) ?? ""; break; case PretranslationUsfmTextOrigin.PreferPretranslated: @@ -110,8 +109,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) textId, pretranslations.ToList(), fullName: targetSettings.FullName, - stripAllText: false, - preferExistingText: false + behavior: UpdateUsfmBehavior.PreferNew ) ?? ""; break; case PretranslationUsfmTextOrigin.OnlyExisting: @@ -120,8 +118,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) textId, [], // don't put any pretranslations, we only want the existing text. fullName: targetSettings.FullName, - stripAllText: false, - preferExistingText: false + behavior: UpdateUsfmBehavior.PreferNew ) ?? ""; break; case PretranslationUsfmTextOrigin.OnlyPretranslated: @@ -130,8 +127,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) textId, pretranslations.ToList(), fullName: targetSettings.FullName, - stripAllText: true, - preferExistingText: false + behavior: UpdateUsfmBehavior.StripExisting ) ?? ""; break; } @@ -155,16 +151,14 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) textId, pretranslations.ToList(), fullName: targetSettings.FullName, - stripAllText: true, - preferExistingText: true + behavior: UpdateUsfmBehavior.StripExisting ) ?? ""; case PretranslationUsfmTextOrigin.OnlyExisting: return updater.UpdateUsfm( textId, [], // don't pass the pretranslations, we only want the existing text. fullName: targetSettings.FullName, - stripAllText: true, - preferExistingText: true + behavior: UpdateUsfmBehavior.StripExisting ) ?? ""; } } diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs index 94ee2e99..f48813a7 100644 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs @@ -5,21 +5,10 @@ namespace Microsoft.Extensions.DependencyInjection; public static class IServalBuilderExtensions { - public static IServalBuilder AddWordAlignment( - this IServalBuilder builder, - Action? configure = null - ) + public static IServalBuilder AddWordAlignment(this IServalBuilder builder) { - if (builder.Configuration is null) - { - builder.AddApiOptions(o => { }); - builder.AddDataFileOptions(o => { }); - } - else - { - builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); - builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); - } + builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); + builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -27,8 +16,6 @@ public static IServalBuilder AddWordAlignment( var wordAlignmentOptions = new WordAlignmentOptions(); builder.Configuration?.GetSection(WordAlignmentOptions.Key).Bind(wordAlignmentOptions); - if (configure is not null) - configure(wordAlignmentOptions); foreach (EngineInfo engine in wordAlignmentOptions.Engines) { diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs index b4dc6841..42d70339 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs @@ -17,7 +17,7 @@ public void GetZipParatextProjectTextUpdater() TestEnvironment env = new(); using ZipParatextProjectTextUpdater updater = env.Service.GetZipParatextProjectTextUpdater("file1.zip"); Assert.That( - updater.UpdateUsfm("MAT", [], preferExistingText: true).ReplaceLineEndings("\n"), + updater.UpdateUsfm("MAT", [], behavior: UpdateUsfmBehavior.PreferExisting).ReplaceLineEndings("\n"), Is.EqualTo( $@"\id MAT - PROJ \h {Canon.BookIdToEnglishName("MAT")}