From 346dd30e1539f0c98886012c5545c52dc0bed787 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 7 Jun 2024 16:43:15 -0400 Subject: [PATCH] Change SMT model save location to be in build - always do it and auto-save when delete. --- src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs | 4 ++-- .../Services/SmtTransferClearMLBuildJobFactory.cs | 3 --- .../Services/SmtTransferPostprocessBuildJob.cs | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs index f5d7c2c98..1bf7a4389 100644 --- a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs @@ -30,7 +30,7 @@ CancellationToken cancellationToken await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { - int additionalCorpusSize = await SaveModelAsync(engineId); + int additionalCorpusSize = await SaveModelAsync(engineId, buildId); await PlatformService.BuildCompletedAsync( buildId, corpusSize + additionalCorpusSize, @@ -43,7 +43,7 @@ await PlatformService.BuildCompletedAsync( Logger.LogInformation("Build completed ({0}).", buildId); } - protected virtual Task SaveModelAsync(string engineId) + protected virtual Task SaveModelAsync(string engineId, string buildId) { return Task.FromResult(0); } diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs index 367bdb2d2..6a1f42aeb 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs @@ -37,9 +37,6 @@ public async Task CreateJobScriptAsync( + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") - // buildRevision + 1 because the build revision is incremented after the build job - // is finished successfully but the file should be saved with the new revision number - + (engine.IsModelPersisted ? $" 'save_model': '{engineId}',\n" : $"") + $" 'clearml': True\n" + "}\n" + "run(args)\n"; diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs index 435d82263..38c65de64 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs @@ -18,11 +18,11 @@ IOptionsMonitor options private readonly IRepository _trainSegmentPairs = trainSegmentPairs; private readonly IOptionsMonitor _options = options; - protected override async Task SaveModelAsync(string engineId) + protected override async Task SaveModelAsync(string engineId, string buildId) { await using ( Stream engineStream = await SharedFileService.OpenReadAsync( - $"models/{engineId}.zip", + $"builds/{buildId}/model.zip", CancellationToken.None ) )