Update from other changes

.vscode/settings.json

@@ -13,6 +13,7 @@
     "hmac",
     "HMACSHA",
     "inferencing",
+    "Initializable",
     "keyterms",
     "MATSRC",
     "MATTRG",

src/Serval/src/Serval.Translation/Services/EngineService.cs

@@ -568,8 +568,8 @@ public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cance
                     c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId)
                 )
                 || e.ParallelCorpora.Any(c =>
-                    c.SourceCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId))
-                    || c.TargetCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId))
+                    c.SourceCorpora.Any(sc => sc.Files.Any(f => f.Id == dataFileId))
+                    || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId))
                 ),
             u =>
             {
@@ -600,8 +600,8 @@ public Task UpdateDataFileFilenameFilesAsync(
                     c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId)
                 )
                 || e.ParallelCorpora.Any(c =>
-                    c.SourceCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId))
-                    || c.TargetCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId))
+                    c.SourceCorpora.Any(sc => sc.Files.Any(f => f.Id == dataFileId))
+                    || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId))
                 ),
             u =>
             {
@@ -643,7 +643,7 @@ public Task UpdateCorpusFilesAsync(
         return Entities.UpdateAllAsync(
             e =>
                 e.ParallelCorpora.Any(c =>
-                    c.SourceCorpora.Any(mc => mc.Id == corpusId) || c.TargetCorpora.Any(mc => mc.Id == corpusId)
+                    c.SourceCorpora.Any(sc => sc.Id == corpusId) || c.TargetCorpora.Any(tc => tc.Id == corpusId)
                 ),
             u =>
             {

src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs

@@ -7,6 +7,8 @@ this IMediatorRegistrationConfigurator configurator
     )
     {
         configurator.AddConsumer<DataFileDeletedConsumer>();
+        configurator.AddConsumer<DataFileUpdatedConsumer>();
+        configurator.AddConsumer<CorpusUpdatedConsumer>();
         return configurator;
     }
 }

src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs

@@ -15,14 +15,22 @@ this IMongoDataAccessConfigurator configurator
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Engine>(Builders<Engine>.IndexKeys.Ascending(e => e.Owner))
                 );
+                await c.Indexes.CreateOrUpdateAsync(
+                    new CreateIndexModel<Engine>(Builders<Engine>.IndexKeys.Ascending(e => e.DateCreated))
+                );
             }
         );
         configurator.AddRepository<Build>(
             "word_alignment.builds",
-            init: c =>
-                c.Indexes.CreateOrUpdateAsync(
+            init: async c =>
+            {
+                await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Build>(Builders<Build>.IndexKeys.Ascending(b => b.EngineRef))
-                )
+                );
+                await c.Indexes.CreateOrUpdateAsync(
+                    new CreateIndexModel<Build>(Builders<Build>.IndexKeys.Ascending(b => b.DateCreated))
+                );
+            }
         );
         configurator.AddRepository<WordAlignment>(
             "word_alignment.word_alignments",

src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs

@@ -7,13 +7,16 @@ public static class IServalBuilderExtensions
 {
     public static IServalBuilder AddWordAlignment(this IServalBuilder builder)
     {
-        builder.AddApiOptions(builder.Configuration!.GetSection(ApiOptions.Key));
+        builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key));
         builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key));
 
         builder.Services.AddScoped<IBuildService, BuildService>();
         builder.Services.AddScoped<IWordAlignmentService, WordAlignmentService>();
         builder.Services.AddScoped<IEngineService, EngineService>();
 
+        builder.Services.AddSingleton<EngineCleanupService>();
+        builder.Services.AddSingleton<BuildCleanupService>();
+
         var wordAlignmentOptions = new WordAlignmentOptions();
         builder.Configuration.GetSection(WordAlignmentOptions.Key).Bind(wordAlignmentOptions);
 

src/Serval/src/Serval.WordAlignment/Consumers/CorpusUpdatedConsumer.cs

@@ -0,0 +1,26 @@
+namespace Serval.WordAlignment.Consumers;
+
+public class CorpusUpdatedConsumer(IEngineService engineService) : IConsumer<CorpusUpdated>
+{
+    private readonly IEngineService _engineService = engineService;
+
+    public async Task Consume(ConsumeContext<CorpusUpdated> context)
+    {
+        await _engineService.UpdateCorpusFilesAsync(
+            context.Message.CorpusId,
+            context.Message.Files.Select(Map).ToList(),
+            context.CancellationToken
+        );
+    }
+
+    private static CorpusFile Map(CorpusFileResult corpusFile)
+    {
+        return new CorpusFile
+        {
+            Id = corpusFile.File.DataFileId,
+            TextId = corpusFile.TextId ?? corpusFile.File.Name,
+            Filename = corpusFile.File.Filename,
+            Format = corpusFile.File.Format,
+        };
+    }
+}

src/Serval/src/Serval.WordAlignment/Consumers/DataFileUpdatedConsumer.cs

@@ -0,0 +1,15 @@
+namespace Serval.WordAlignment.Consumers;
+
+public class DataFileUpdatedConsumer(IEngineService engineService) : IConsumer<DataFileUpdated>
+{
+    private readonly IEngineService _engineService = engineService;
+
+    public async Task Consume(ConsumeContext<DataFileUpdated> context)
+    {
+        await _engineService.UpdateDataFileFilenameFilesAsync(
+            context.Message.DataFileId,
+            context.Message.Filename,
+            context.CancellationToken
+        );
+    }
+}

src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs

@@ -951,6 +951,7 @@ private Engine Map(WordAlignmentEngineConfigDto source)
             Type = source.Type.ToPascalCase(),
             Owner = Owner,
             ParallelCorpora = [],
+            IsInitialized = false
         };
     }
 }

src/Serval/src/Serval.WordAlignment/Models/Build.cs

@@ -1,6 +1,6 @@
 namespace Serval.WordAlignment.Models;
 
-public record Build : IEntity
+public record Build : IInitializableEntity
 {
     public string Id { get; set; } = "";
     public int Revision { get; set; } = 1;
@@ -15,4 +15,7 @@ public record Build : IEntity
     public JobState State { get; init; } = JobState.Pending;
     public DateTime? DateFinished { get; init; }
     public IReadOnlyDictionary<string, object>? Options { get; init; }
+    public string? DeploymentVersion { get; init; }
+    public bool? IsInitialized { get; set; }
+    public DateTime? DateCreated { get; set; }
 }

src/Serval/src/Serval.WordAlignment/Models/Engine.cs

@@ -1,6 +1,6 @@
 namespace Serval.WordAlignment.Models;
 
-public record Engine : IOwnedEntity
+public record Engine : IOwnedEntity, IInitializableEntity
 {
     public string Id { get; set; } = "";
     public int Revision { get; set; } = 1;
@@ -10,9 +10,10 @@ public record Engine : IOwnedEntity
     public required string Type { get; init; }
     public required string Owner { get; init; }
     public required IReadOnlyList<ParallelCorpus> ParallelCorpora { get; init; }
-    public bool? IsModelPersisted { get; init; }
     public bool IsBuilding { get; init; }
     public int ModelRevision { get; init; }
     public double Confidence { get; init; }
     public int CorpusSize { get; init; }
+    public bool? IsInitialized { get; set; }
+    public DateTime? DateCreated { get; set; }
 }

src/Serval/src/Serval.WordAlignment/Services/BuildCleanupService.cs

@@ -0,0 +1,7 @@
+namespace Serval.WordAlignment.Services;
+
+public class BuildCleanupService(
+    IServiceProvider services,
+    ILogger<BuildCleanupService> logger,
+    TimeSpan? timeout = null
+) : UninitializedCleanupService<Build>(services, logger, timeout) { }

src/Serval/src/Serval.WordAlignment/Services/EngineCleanupService.cs

@@ -0,0 +1,7 @@
+namespace Serval.WordAlignment.Services;
+
+public class EngineCleanupService(
+    IServiceProvider services,
+    ILogger<EngineCleanupService> logger,
+    TimeSpan? timeout = null
+) : UninitializedCleanupService<Engine>(services, logger, timeout) { }

src/Serval/src/Serval.WordAlignment/Services/EngineService.cs

@@ -21,6 +21,25 @@ IScriptureDataFileService scriptureDataFileService
     private readonly ILogger<EngineService> _logger = loggerFactory.CreateLogger<EngineService>();
     private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService;
 
+    public override async Task<Engine> GetAsync(string id, CancellationToken cancellationToken = default)
+    {
+        Engine engine = await base.GetAsync(id, cancellationToken);
+        if (!(engine.IsInitialized ?? true))
+            throw new EntityNotFoundException($"Could not find the {typeof(Engine).Name} '{id}'.");
+        return engine;
+    }
+
+    public override async Task<IEnumerable<Engine>> GetAllAsync(
+        string owner,
+        CancellationToken cancellationToken = default
+    )
+    {
+        return await Entities.GetAllAsync(
+            e => e.Owner == owner && (e.IsInitialized == null || e.IsInitialized.Value),
+            cancellationToken
+        );
+    }
+
     public async Task<Models.WordAlignmentResult> GetWordAlignmentAsync(
         string engineId,
         string sourceSegment,
@@ -47,9 +66,9 @@ IScriptureDataFileService scriptureDataFileService
 
     public override async Task<Engine> CreateAsync(Engine engine, CancellationToken cancellationToken = default)
     {
-        bool updateIsModelPersisted = engine.IsModelPersisted is null;
         try
         {
+            engine.DateCreated = DateTime.UtcNow;
             await Entities.InsertAsync(engine, cancellationToken);
             WordAlignmentEngineApi.WordAlignmentEngineApiClient client;
             try
@@ -91,14 +110,6 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
             await Entities.DeleteAsync(engine, CancellationToken.None);
             throw;
         }
-        if (updateIsModelPersisted)
-        {
-            await Entities.UpdateAsync(
-                engine,
-                u => u.Set(e => e.IsModelPersisted, engine.IsModelPersisted),
-                cancellationToken: cancellationToken
-            );
-        }
         return engine;
     }
 
@@ -143,6 +154,7 @@ private Dictionary<string, List<int>> GetChapters(string fileLocation, string sc
 
     public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default)
     {
+        build.DateCreated = DateTime.UtcNow;
         Engine engine = await GetAsync(build.EngineRef, cancellationToken);
         await _builds.InsertAsync(build, cancellationToken);
 
@@ -164,6 +176,7 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
                     || wordAlignOn.ContainsKey(pc.Id)
                 )
                 .ToList();
+
             request = new StartBuildRequest
             {
                 EngineType = engine.Type,
@@ -213,8 +226,12 @@ wordAlignOn is null
                 _logger.LogInformation("Error parsing build request summary.");
                 _logger.LogInformation("{request}", JsonSerializer.Serialize(request));
             }
-
             await client.StartBuildAsync(request, cancellationToken: cancellationToken);
+            await _builds.UpdateAsync(
+                b => b.Id == build.Id,
+                u => u.Set(e => e.IsInitialized, true),
+                cancellationToken: CancellationToken.None
+            );
         }
         catch
         {
@@ -254,7 +271,7 @@ public Task AddParallelCorpusAsync(
     )
     {
         return Entities.UpdateAsync(
-            engineId,
+            e => e.Id == engineId && (e.IsInitialized == null || e.IsInitialized.Value),
             u => u.Add(e => e.ParallelCorpora, corpus),
             cancellationToken: cancellationToken
         );
@@ -269,7 +286,10 @@ public Task AddParallelCorpusAsync(
     )
     {
         Engine? engine = await Entities.UpdateAsync(
-            e => e.Id == engineId && e.ParallelCorpora.Any(c => c.Id == parallelCorpusId),
+            e =>
+                e.Id == engineId
+                && (e.IsInitialized == null || e.IsInitialized.Value)
+                && e.ParallelCorpora.Any(c => c.Id == parallelCorpusId),
             u =>
             {
                 if (sourceCorpora is not null)
@@ -300,7 +320,7 @@ await _dataAccessContext.WithTransactionAsync(
             async (ct) =>
             {
                 originalEngine = await Entities.UpdateAsync(
-                    engineId,
+                    e => e.Id == engineId && (e.IsInitialized == null || e.IsInitialized.Value),
                     u => u.RemoveAll(e => e.ParallelCorpora, c => c.Id == parallelCorpusId),
                     returnOriginal: true,
                     cancellationToken: ct
@@ -326,15 +346,78 @@ public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cance
                     || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId))
                 ),
             u =>
+            {
                 u.RemoveAll(
-                        e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files,
-                        f => f.Id == dataFileId
-                    )
-                    .RemoveAll(
-                        e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files,
-                        f => f.Id == dataFileId
-                    ),
-            cancellationToken
+                    e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files,
+                    f => f.Id == dataFileId
+                );
+                u.RemoveAll(
+                    e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files,
+                    f => f.Id == dataFileId
+                );
+            },
+            cancellationToken: cancellationToken
+        );
+    }
+
+    public Task UpdateDataFileFilenameFilesAsync(
+        string dataFileId,
+        string filename,
+        CancellationToken cancellationToken = default
+    )
+    {
+        return Entities.UpdateAllAsync(
+            e =>
+                e.ParallelCorpora.Any(c =>
+                    c.SourceCorpora.Any(cs => cs.Files.Any(f => f.Id == dataFileId))
+                    || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId))
+                ),
+            u =>
+            {
+                u.SetAll(
+                    e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files,
+                    f => f.Filename,
+                    filename,
+                    f => f.Id == dataFileId
+                );
+                u.SetAll(
+                    e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files,
+                    f => f.Filename,
+                    filename,
+                    f => f.Id == dataFileId
+                );
+            },
+            cancellationToken: cancellationToken
+        );
+    }
+
+    public Task UpdateCorpusFilesAsync(
+        string corpusId,
+        IReadOnlyList<Shared.Models.CorpusFile> files,
+        CancellationToken cancellationToken = default
+    )
+    {
+        return Entities.UpdateAllAsync(
+            e =>
+                e.ParallelCorpora.Any(c =>
+                    c.SourceCorpora.Any(sc => sc.Id == corpusId) || c.TargetCorpora.Any(tc => tc.Id == corpusId)
+                ),
+            u =>
+            {
+                u.SetAll(
+                    e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora,
+                    mc => mc.Files,
+                    files,
+                    mc => mc.Id == corpusId
+                );
+                u.SetAll(
+                    e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora,
+                    mc => mc.Files,
+                    files,
+                    mc => mc.Id == corpusId
+                );
+            },
+            cancellationToken: cancellationToken
         );
     }