Sync corpora and data files in translation engine

Use mongoDB strings directly for Mongo array filtering (hacky solution - need to fix later #553). Broken implementation for MemoryRepository - #554
sillsdev · Dec 4, 2024 · de529ce · de529ce
1 parent 2fa330d
commit de529ce
Show file tree

Hide file tree

Showing 25 changed files with 426 additions and 114 deletions.
diff --git a/docker-compose.mongo.yml b/docker-compose.mongo.yml
@@ -15,5 +15,5 @@ services:
       [
         '/bin/sh',
         '-c',
-        'mongod --quiet --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity'
+        'mongod --profile=2 --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity'
       ]
diff --git a/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs b/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs
@@ -4,4 +4,5 @@ public static class ArrayPosition
 {
     public const int FirstMatching = int.MaxValue;
     public const int All = int.MaxValue - 1;
+    public const int ArrayFilter = int.MaxValue - 2;
 }
diff --git a/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs b/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs
@@ -17,6 +17,10 @@ LinqProvider linqProvider
             linqProvider
         );
         string fieldName = rendered.FieldName.Replace(ArrayPosition.All.ToString(CultureInfo.InvariantCulture), "$[]");
+        fieldName = fieldName.Replace(
+            ArrayPosition.ArrayFilter.ToString(CultureInfo.InvariantCulture),
+            "$[arrayFilter]"
+        );
         fieldName = fieldName.Replace(ArrayPosition.FirstMatching.ToString(CultureInfo.InvariantCulture), "$");
         if (fieldName != rendered.FieldName)
         {

diff --git a/src/DataAccess/src/SIL.DataAccess/IRepository.cs b/src/DataAccess/src/SIL.DataAccess/IRepository.cs
@@ -16,9 +16,17 @@ public interface IRepository<T>
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     );
+    Task<int> UpdateAllAsync<TFilter>(
+        Expression<Func<T, bool>> filter,
+        Action<IUpdateBuilder<T>> update,
+        string jsonArrayFilterDefinition,
+        CancellationToken cancellationToken = default
+    );
+
     Task<int> UpdateAllAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        UpdateOptions? updateOptions = null,
         CancellationToken cancellationToken = default
     );
     Task<T?> DeleteAsync(Expression<Func<T, bool>> filter, CancellationToken cancellationToken = default);

diff --git a/src/DataAccess/src/SIL.DataAccess/MemoryRepository.cs b/src/DataAccess/src/SIL.DataAccess/MemoryRepository.cs
@@ -233,9 +233,20 @@ public async Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationTo
         return returnOriginal ? original : entity;
     }
 
+    public async Task<int> UpdateAllAsync<TFilter>(
+        Expression<Func<T, bool>> filter,
+        Action<IUpdateBuilder<T>> update,
+        string jsonArrayFilterDefinition,
+        CancellationToken cancellationToken = default
+    )
+    {
+        return await UpdateAllAsync(filter, update, null, cancellationToken);
+    }
+
     public async Task<int> UpdateAllAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        UpdateOptions? updateOptions = null,
         CancellationToken cancellationToken = default
     )
     {

diff --git a/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs
@@ -224,6 +224,8 @@ Expression<Func<T, TField>> field
                                     }
                                     break;
                                 case ArrayPosition.All:
+                                // This doesn't filter as it should - but it's good enough for unit testing.
+                                case ArrayPosition.ArrayFilter:
                                     newOwners.AddRange(((IEnumerable)owner).Cast<object>());
                                     break;
                                 default:

diff --git a/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs b/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs
@@ -151,9 +151,24 @@ await _collection
         return entity;
     }
 
+    public async Task<int> UpdateAllAsync<TFilter>(
+        Expression<Func<T, bool>> filter,
+        Action<IUpdateBuilder<T>> update,
+        string jsonArrayFilterDefinition,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var updateOptions = new UpdateOptions
+        {
+            ArrayFilters = [new JsonArrayFilterDefinition<TFilter>(jsonArrayFilterDefinition)]
+        };
+        return await UpdateAllAsync(filter, update, updateOptions, cancellationToken).ConfigureAwait(false);
+    }
+
     public async Task<int> UpdateAllAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        UpdateOptions? updateOptions = null,
         CancellationToken cancellationToken = default
     )
     {
@@ -167,13 +182,19 @@ public async Task<int> UpdateAllAsync(
             if (_context.Session is not null)
             {
                 result = await _collection
-                    .UpdateManyAsync(_context.Session, filter, updateDef, cancellationToken: cancellationToken)
+                    .UpdateManyAsync(
+                        _context.Session,
+                        filter,
+                        updateDef,
+                        updateOptions,
+                        cancellationToken: cancellationToken
+                    )
                     .ConfigureAwait(false);
             }
             else
             {
                 result = await _collection
-                    .UpdateManyAsync(filter, updateDef, cancellationToken: cancellationToken)
+                    .UpdateManyAsync(filter, updateDef, updateOptions, cancellationToken: cancellationToken)
                     .ConfigureAwait(false);
             }
         }

diff --git a/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs
@@ -3,36 +3,38 @@ namespace SIL.DataAccess;
 public class MongoUpdateBuilder<T> : IUpdateBuilder<T>
     where T : IEntity
 {
-    private readonly UpdateDefinitionBuilder<T> _builder;
+    private readonly UpdateDefinitionBuilder<T> _updateBuilder;
+    private readonly FilterDefinitionBuilder<T> _filterBuilder;
     private readonly List<UpdateDefinition<T>> _defs;
 
     public MongoUpdateBuilder()
     {
-        _builder = Builders<T>.Update;
+        _updateBuilder = Builders<T>.Update;
+        _filterBuilder = Builders<T>.Filter;
         _defs = new List<UpdateDefinition<T>>();
     }
 
     public IUpdateBuilder<T> Set<TField>(Expression<Func<T, TField>> field, TField value)
     {
-        _defs.Add(_builder.Set(ToFieldDefinition(field), value));
+        _defs.Add(_updateBuilder.Set(ToFieldDefinition(field), value));
         return this;
     }
 
     public IUpdateBuilder<T> SetOnInsert<TField>(Expression<Func<T, TField>> field, TField value)
     {
-        _defs.Add(_builder.SetOnInsert(ToFieldDefinition(field), value));
+        _defs.Add(_updateBuilder.SetOnInsert(ToFieldDefinition(field), value));
         return this;
     }
 
     public IUpdateBuilder<T> Unset<TField>(Expression<Func<T, TField>> field)
     {
-        _defs.Add(_builder.Unset(ToFieldDefinition(field)));
+        _defs.Add(_updateBuilder.Unset(ToFieldDefinition(field)));
         return this;
     }
 
     public IUpdateBuilder<T> Inc(Expression<Func<T, int>> field, int value = 1)
     {
-        _defs.Add(_builder.Inc(ToFieldDefinition(field), value));
+        _defs.Add(_updateBuilder.Inc(ToFieldDefinition(field), value));
         return this;
     }
 
@@ -41,27 +43,27 @@ public IUpdateBuilder<T> RemoveAll<TItem>(
         Expression<Func<TItem, bool>> predicate
     )
     {
-        _defs.Add(_builder.PullFilter(ToFieldDefinition(field), Builders<TItem>.Filter.Where(predicate)));
+        _defs.Add(_updateBuilder.PullFilter(ToFieldDefinition(field), Builders<TItem>.Filter.Where(predicate)));
         return this;
     }
 
     public IUpdateBuilder<T> Remove<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value)
     {
-        _defs.Add(_builder.Pull(ToFieldDefinition(field), value));
+        _defs.Add(_updateBuilder.Pull(ToFieldDefinition(field), value));
         return this;
     }
 
     public IUpdateBuilder<T> Add<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value)
     {
-        _defs.Add(_builder.Push(ToFieldDefinition(field), value));
+        _defs.Add(_updateBuilder.Push(ToFieldDefinition(field), value));
         return this;
     }
 
     public UpdateDefinition<T> Build()
     {
         if (_defs.Count == 1)
             return _defs.Single();
-        return _builder.Combine(_defs);
+        return _updateBuilder.Combine(_defs);
     }
 
     private static FieldDefinition<T, TField> ToFieldDefinition<TField>(Expression<Func<T, TField>> field)

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs
@@ -55,7 +55,7 @@ private async Task ReleaseAllWriterLocksAsync(CancellationToken cancellationToke
         await _locks.UpdateAllAsync(
             rwl => rwl.WriterLock != null && rwl.WriterLock.HostId == _serviceOptions.ServiceId,
             u => u.Unset(rwl => rwl.WriterLock),
-            cancellationToken
+            cancellationToken: cancellationToken
         );
     }
 
@@ -64,7 +64,7 @@ private async Task ReleaseAllReaderLocksAsync(CancellationToken cancellationToke
         await _locks.UpdateAllAsync(
             rwl => rwl.ReaderLocks.Any(l => l.HostId == _serviceOptions.ServiceId),
             u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.HostId == _serviceOptions.ServiceId),
-            cancellationToken
+            cancellationToken: cancellationToken
         );
     }
 
@@ -73,7 +73,7 @@ private async Task RemoveAllWaitersAsync(CancellationToken cancellationToken)
         await _locks.UpdateAllAsync(
             rwl => rwl.WriterQueue.Any(l => l.HostId == _serviceOptions.ServiceId),
             u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.HostId == _serviceOptions.ServiceId),
-            cancellationToken
+            cancellationToken: cancellationToken
         );
     }
 }
diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs
@@ -7073,9 +7073,37 @@ public partial class Corpus
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class CorpusFile
     {
-        [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)]
+        [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public string FileId { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string? TextId { get; set; } = default!;
+
+    }
+
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class CorpusConfig
+    {
+        [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string? Name { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public string Language { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)]
         [System.ComponentModel.DataAnnotations.Required]
-        public DataFile File { get; set; } = new DataFile();
+        public System.Collections.Generic.IList<CorpusFileConfig> Files { get; set; } = new System.Collections.ObjectModel.Collection<CorpusFileConfig>();
+
+    }
+
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class CorpusFileConfig
+    {
+        [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public string FileId { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string? TextId { get; set; } = default!;
@@ -7118,34 +7146,6 @@ public enum FileFormat
 
     }
 
-    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
-    public partial class CorpusConfig
-    {
-        [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public string? Name { get; set; } = default!;
-
-        [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)]
-        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
-        public string Language { get; set; } = default!;
-
-        [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)]
-        [System.ComponentModel.DataAnnotations.Required]
-        public System.Collections.Generic.IList<CorpusFileConfig> Files { get; set; } = new System.Collections.ObjectModel.Collection<CorpusFileConfig>();
-
-    }
-
-    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
-    public partial class CorpusFileConfig
-    {
-        [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)]
-        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
-        public string FileId { get; set; } = default!;
-
-        [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public string? TextId { get; set; } = default!;
-
-    }
-
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class TranslationEngine
     {

diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs
@@ -1,8 +1,9 @@
 namespace Serval.DataFiles.Consumers;
 
-public class GetCorpusConsumer(ICorpusService corpusService) : IConsumer<GetCorpus>
+public class GetCorpusConsumer(ICorpusService corpusService, IDataFileService dataFileService) : IConsumer<GetCorpus>
 {
     private readonly ICorpusService _corpusService = corpusService;
+    private readonly IDataFileService _dataFileService = dataFileService;
 
     public async Task Consume(ConsumeContext<GetCorpus> context)
     {
@@ -19,19 +20,13 @@ await context.RespondAsync(
                     CorpusId = corpus.Id,
                     Name = corpus.Name,
                     Language = corpus.Language,
-                    Files = corpus
-                        .Files.Select(f => new CorpusFileResult
+                    Files = await Task.WhenAll(
+                        corpus.Files.Select(async f => new CorpusFileResult
                         {
                             TextId = f.TextId!,
-                            File = new DataFileResult
-                            {
-                                DataFileId = f.File.Id,
-                                Filename = f.File.Filename,
-                                Format = f.File.Format,
-                                Name = f.File.Name
-                            }
+                            File = Map(await _dataFileService.GetAsync(f.FileId))
                         })
-                        .ToList()
+                    )
                 }
             );
         }
@@ -42,4 +37,15 @@ await context.RespondAsync(
             );
         }
     }
+
+    private static DataFileResult Map(DataFile dataFile)
+    {
+        return new DataFileResult
+        {
+            DataFileId = dataFile.Id,
+            Name = dataFile.Name,
+            Filename = dataFile.Filename,
+            Format = dataFile.Format,
+        };
+    }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs
@@ -2,6 +2,6 @@ namespace Serval.DataFiles.Contracts;
 
 public record CorpusFileDto
 {
-    public required DataFileDto File { get; init; }
+    public required string FileId { get; init; }
     public string? TextId { get; init; }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs
@@ -177,7 +177,7 @@ CancellationToken cancellationToken
             DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken);
             if (dataFile == null)
                 throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist.");
-            dataFiles.Add(new CorpusFile { File = dataFile, TextId = file.TextId });
+            dataFiles.Add(new CorpusFile { FileId = file.FileId, TextId = file.TextId });
         }
         return dataFiles;
     }
@@ -197,18 +197,6 @@ private CorpusDto Map(Corpus source)
 
     private CorpusFileDto Map(CorpusFile source)
     {
-        return new CorpusFileDto { File = Map(source.File), TextId = source.TextId };
-    }
-
-    private DataFileDto Map(DataFile source)
-    {
-        return new DataFileDto
-        {
-            Id = source.Id,
-            Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }),
-            Name = source.Name,
-            Format = source.Format,
-            Revision = source.Revision
-        };
+        return new CorpusFileDto { FileId = source.FileId, TextId = source.TextId };
     }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs
@@ -2,6 +2,6 @@ namespace Serval.DataFiles.Models;
 
 public record CorpusFile
 {
-    public required DataFile File { get; init; }
+    public required string FileId { get; init; }
     public string? TextId { get; init; }
 }