diff --git a/docker-compose.mongo.yml b/docker-compose.mongo.yml index 1f1bbec9..b99403b9 100644 --- a/docker-compose.mongo.yml +++ b/docker-compose.mongo.yml @@ -15,5 +15,5 @@ services: [ '/bin/sh', '-c', - 'mongod --quiet --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity' + 'mongod --profile=2 --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity' ] diff --git a/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs b/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs index 674972cf..4a824ac5 100644 --- a/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs +++ b/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs @@ -4,4 +4,5 @@ public static class ArrayPosition { public const int FirstMatching = int.MaxValue; public const int All = int.MaxValue - 1; + internal const int ArrayFilter = int.MaxValue - 2; } diff --git a/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs b/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs index b7be7638..f31016f1 100644 --- a/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs +++ b/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs @@ -1,9 +1,12 @@ namespace SIL.DataAccess; -public class DataAccessFieldDefinition(Expression> expression) - : FieldDefinition +public class DataAccessFieldDefinition( + Expression> expression, + string arrayFilterId = "" +) : FieldDefinition { private readonly ExpressionFieldDefinition _internalDef = new(expression); + private readonly string _arrayFilterId = arrayFilterId; public override RenderedFieldDefinition Render( IBsonSerializer documentSerializer, @@ -18,6 +21,10 @@ LinqProvider linqProvider ); string fieldName = rendered.FieldName.Replace(ArrayPosition.All.ToString(CultureInfo.InvariantCulture), "$[]"); fieldName = fieldName.Replace(ArrayPosition.FirstMatching.ToString(CultureInfo.InvariantCulture), "$"); + fieldName = fieldName.Replace( + ArrayPosition.ArrayFilter.ToString(CultureInfo.InvariantCulture), + $"$[{_arrayFilterId}]" + ); if (fieldName != rendered.FieldName) { return new RenderedFieldDefinition( diff --git a/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs b/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs index b41948fb..b7a2eb0a 100644 --- a/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs +++ b/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs @@ -38,4 +38,14 @@ Expression expression finder.Visit(expression); return finder.Value; } + + public static Expression> Concatenate( + Expression> left, + Expression> right + ) + { + ParameterReplacer replacer = new(right.Parameters[0], left.Body); + Expression merged = replacer.Visit(right.Body); + return Expression.Lambda>(merged, left.Parameters[0]); + } } diff --git a/src/DataAccess/src/SIL.DataAccess/IRepository.cs b/src/DataAccess/src/SIL.DataAccess/IRepository.cs index 95539b96..d817859a 100644 --- a/src/DataAccess/src/SIL.DataAccess/IRepository.cs +++ b/src/DataAccess/src/SIL.DataAccess/IRepository.cs @@ -9,6 +9,7 @@ public interface IRepository Task InsertAsync(T entity, CancellationToken cancellationToken = default); Task InsertAllAsync(IReadOnlyCollection entities, CancellationToken cancellationToken = default); + Task UpdateAsync( Expression> filter, Action> update, @@ -21,6 +22,7 @@ Task UpdateAllAsync( Action> update, CancellationToken cancellationToken = default ); + Task DeleteAsync(Expression> filter, CancellationToken cancellationToken = default); Task DeleteAllAsync(Expression> filter, CancellationToken cancellationToken = default); Task> SubscribeAsync( diff --git a/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs index 90df3882..6eaf27e5 100644 --- a/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs +++ b/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs @@ -13,10 +13,17 @@ public interface IUpdateBuilder IUpdateBuilder RemoveAll( Expression?>> field, - Expression> predicate + Expression>? predicate = null ); IUpdateBuilder Remove(Expression?>> field, TItem value); IUpdateBuilder Add(Expression?>> field, TItem value); + + IUpdateBuilder SetAll( + Expression?>> collectionField, + Expression> itemField, + TField value, + Expression>? predicate = null + ); } diff --git a/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs index 78e00924..bbf6b5d9 100644 --- a/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs +++ b/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs @@ -9,23 +9,20 @@ public class MemoryUpdateBuilder(Expression> filter, T entity, public IUpdateBuilder Set(Expression> field, TField value) { - (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(field); - object[]? indices = index == null ? null : [index]; - foreach (object owner in owners) - prop.SetValue(owner, value, indices); + Set(_entity, _filter, field, value); return this; } public IUpdateBuilder SetOnInsert(Expression> field, TField value) { if (_isInsert) - Set(field, value); + Set(_entity, _filter, field, value); return this; } public IUpdateBuilder Unset(Expression> field) { - (IEnumerable owners, PropertyInfo prop, object? index) = GetFieldOwners(field); + (IEnumerable owners, PropertyInfo prop, object? index) = GetFieldOwners(_entity, _filter, field); if (index != null) { // remove value from a dictionary @@ -49,7 +46,7 @@ public IUpdateBuilder Unset(Expression> field) public IUpdateBuilder Inc(Expression> field, int value = 1) { - (IEnumerable owners, PropertyInfo prop, object? index) = GetFieldOwners(field); + (IEnumerable owners, PropertyInfo prop, object? index) = GetFieldOwners(_entity, _filter, field); object[]? indices = index == null ? null : [index]; foreach (object owner in owners) { @@ -62,12 +59,12 @@ public IUpdateBuilder Inc(Expression> field, int value = 1) public IUpdateBuilder RemoveAll( Expression?>> field, - Expression> predicate + Expression>? predicate = null ) { - (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(field); + (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field); object[]? indices = index == null ? null : [index]; - Func predicateFunc = predicate.Compile(); + Func? predicateFunc = predicate?.Compile(); foreach (object owner in owners) { var collection = (IEnumerable?)prop.GetValue(owner, indices); @@ -75,7 +72,7 @@ Expression> predicate if (collection is not null && removeMethod is not null) { // the collection is mutable, so use Remove method to remove item - TItem[] toRemove = collection.Where(predicateFunc).ToArray(); + TItem[] toRemove = collection.Where(i => predicateFunc?.Invoke(i) ?? true).ToArray(); foreach (TItem item in toRemove) removeMethod.Invoke(collection, [item]); } @@ -84,14 +81,17 @@ Expression> predicate if (prop.PropertyType.IsArray || prop.PropertyType.IsInterface) { // the collection type is an array or interface, so construct a new array and set property - TItem[] newValue = collection.Where(i => !predicateFunc(i)).ToArray(); + TItem[] newValue = collection.Where(i => !(predicateFunc?.Invoke(i) ?? false)).ToArray(); prop.SetValue(owner, newValue, indices); } else { // the collection type is a collection class, so construct a new collection and set property var newValue = (IEnumerable?) - Activator.CreateInstance(prop.PropertyType, collection.Where(i => !predicateFunc(i)).ToArray()); + Activator.CreateInstance( + prop.PropertyType, + collection.Where(i => !(predicateFunc?.Invoke(i) ?? false)).ToArray() + ); prop.SetValue(owner, newValue, indices); } } @@ -101,7 +101,7 @@ Expression> predicate public IUpdateBuilder Remove(Expression?>> field, TItem value) { - (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(field); + (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field); object[]? indices = index == null ? null : [index]; foreach (object owner in owners) { @@ -134,7 +134,7 @@ public IUpdateBuilder Remove(Expression?>> public IUpdateBuilder Add(Expression?>> field, TItem value) { - (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(field); + (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field); object[]? indices = index == null ? null : [index]; foreach (object owner in owners) { @@ -147,7 +147,7 @@ public IUpdateBuilder Add(Expression?>> fie } else { - collection ??= Array.Empty(); + collection ??= []; if (prop.PropertyType.IsArray || prop.PropertyType.IsInterface) { // the collection type is an array or interface, so construct a new array and set property @@ -166,6 +166,47 @@ public IUpdateBuilder Add(Expression?>> fie return this; } + public IUpdateBuilder SetAll( + Expression?>> collectionField, + Expression> itemField, + TField value, + Expression>? predicate = null + ) + { + (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners( + _entity, + _filter, + collectionField + ); + object[]? indices = index == null ? null : [index]; + Func? predicateFunc = predicate?.Compile(); + foreach (object owner in owners) + { + var collection = (IEnumerable?)prop.GetValue(owner, indices); + if (collection is null) + continue; + foreach (TItem item in collection) + { + if (predicateFunc == null || predicateFunc(item)) + Set(item, i => true, itemField, value); + } + } + return this; + } + + private static void Set( + TEntity entity, + Expression> filter, + Expression> field, + TField value + ) + { + (IEnumerable owners, PropertyInfo? prop, object? index) = GetFieldOwners(entity, filter, field); + object[]? indices = index == null ? null : [index]; + foreach (object owner in owners) + prop.SetValue(owner, value, indices); + } + private static bool IsAnyMethod(MethodInfo mi) { return mi.DeclaringType == typeof(Enumerable) && mi.Name == "Any"; @@ -180,8 +221,10 @@ private static MethodInfo GetFirstOrDefaultMethod(Type type) .MakeGenericMethod(type); } - private (IEnumerable Owners, PropertyInfo Property, object? Index) GetFieldOwners( - Expression> field + private static (IEnumerable Owners, PropertyInfo Property, object? Index) GetFieldOwners( + TEntity entity, + Expression> filter, + Expression> field ) { List? owners = null; @@ -192,8 +235,8 @@ Expression> field var newOwners = new List(); if (owners == null) { - if (_entity != null) - newOwners.Add(_entity); + if (entity != null) + newOwners.Add(entity); } else { @@ -206,17 +249,14 @@ Expression> field switch (index) { case ArrayPosition.FirstMatching: - foreach (Expression expression in ExpressionHelper.Flatten(_filter)) + foreach (Expression expression in ExpressionHelper.Flatten(filter)) { if (expression is MethodCallExpression callExpr && IsAnyMethod(callExpr.Method)) { var predicate = (LambdaExpression)callExpr.Arguments[1]; Type itemType = predicate.Parameters[0].Type; MethodInfo firstOrDefault = GetFirstOrDefaultMethod(itemType); - newOwner = firstOrDefault.Invoke( - null, - new object[] { owner, predicate.Compile() } - ); + newOwner = firstOrDefault.Invoke(null, [owner, predicate.Compile()]); if (newOwner != null) newOwners.Add(newOwner); break; @@ -245,7 +285,7 @@ Expression> field } else { - newOwner = method.Invoke(owner, new object[] { index }); + newOwner = method.Invoke(owner, [index]); if (newOwner != null) newOwners.Add(newOwner); } diff --git a/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs b/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs index 8ba08dde..d5b09a1b 100644 --- a/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs +++ b/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs @@ -122,12 +122,14 @@ await _collection var updateBuilder = new MongoUpdateBuilder(); update(updateBuilder); updateBuilder.Inc(e => e.Revision, 1); - UpdateDefinition updateDef = updateBuilder.Build(); + (UpdateDefinition updateDef, IReadOnlyList arrayFilters) = updateBuilder.Build(); var options = new FindOneAndUpdateOptions { IsUpsert = upsert, ReturnDocument = returnOriginal ? ReturnDocument.Before : ReturnDocument.After }; + if (arrayFilters.Count > 0) + options.ArrayFilters = arrayFilters; T? entity; try { @@ -160,20 +162,23 @@ public async Task UpdateAllAsync( var updateBuilder = new MongoUpdateBuilder(); update(updateBuilder); updateBuilder.Inc(e => e.Revision, 1); - UpdateDefinition updateDef = updateBuilder.Build(); + (UpdateDefinition updateDef, IReadOnlyList arrayFilters) = updateBuilder.Build(); + UpdateOptions? updateOptions = null; + if (arrayFilters.Count > 0) + updateOptions = new UpdateOptions { ArrayFilters = arrayFilters }; UpdateResult result; try { if (_context.Session is not null) { result = await _collection - .UpdateManyAsync(_context.Session, filter, updateDef, cancellationToken: cancellationToken) + .UpdateManyAsync(_context.Session, filter, updateDef, updateOptions, cancellationToken) .ConfigureAwait(false); } else { result = await _collection - .UpdateManyAsync(filter, updateDef, cancellationToken: cancellationToken) + .UpdateManyAsync(filter, updateDef, updateOptions, cancellationToken) .ConfigureAwait(false); } } diff --git a/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs index 91506f4d..662c3f92 100644 --- a/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs +++ b/src/DataAccess/src/SIL.DataAccess/MongoUpdateBuilder.cs @@ -5,11 +5,13 @@ public class MongoUpdateBuilder : IUpdateBuilder { private readonly UpdateDefinitionBuilder _builder; private readonly List> _defs; + private readonly Dictionary FilterDef)> _arrayFilters; public MongoUpdateBuilder() { _builder = Builders.Update; _defs = new List>(); + _arrayFilters = new Dictionary)>(); } public IUpdateBuilder Set(Expression> field, TField value) @@ -38,7 +40,7 @@ public IUpdateBuilder Inc(Expression> field, int value = 1) public IUpdateBuilder RemoveAll( Expression?>> field, - Expression> predicate + Expression>? predicate = null ) { _defs.Add(_builder.PullFilter(ToFieldDefinition(field), Builders.Filter.Where(predicate))); @@ -57,15 +59,67 @@ public IUpdateBuilder Add(Expression?>> fie return this; } - public UpdateDefinition Build() + public IUpdateBuilder SetAll( + Expression?>> collectionField, + Expression> itemField, + TField value, + Expression>? predicate = null + ) + { + Expression> itemExpr = ExpressionHelper.Concatenate( + collectionField, + (collection) => ((IReadOnlyList?)collection)![ArrayPosition.ArrayFilter] + ); + Expression> fieldExpr = ExpressionHelper.Concatenate(itemExpr, itemField); + if (predicate != null) + { + ExpressionFilterDefinition filter = new(predicate); + BsonDocument bsonDoc = filter.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry, + LinqProvider.V2 + ); + string filterId; + if (_arrayFilters.TryGetValue(bsonDoc, out var existingArrayFilter)) + { + filterId = existingArrayFilter.Id; + } + else + { + filterId = "f" + ObjectId.GenerateNewId().ToString(); + _arrayFilters.Add( + bsonDoc, + ( + filterId, + new BsonDocument( + $"{filterId}.{bsonDoc.Elements.Single().Name}", + bsonDoc.Elements.Single().Value + ) + ) + ); + } + _defs.Add(_builder.Set(ToFieldDefinition(fieldExpr, filterId), value)); + } + else + { + _defs.Add(_builder.Set(ToFieldDefinition(fieldExpr), value)); + } + return this; + } + + public (UpdateDefinition, IReadOnlyList) Build() { + ArrayFilterDefinition[] arrayFilters = _arrayFilters.Values.Select(f => f.FilterDef).ToArray(); if (_defs.Count == 1) - return _defs.Single(); - return _builder.Combine(_defs); + return (_defs.Single(), arrayFilters); + return (_builder.Combine(_defs), arrayFilters); } - private static FieldDefinition ToFieldDefinition(Expression> field) + private static FieldDefinition ToFieldDefinition( + Expression> field, + string arrayFilterId = "" + ) { - return new DataAccessFieldDefinition(field); + return new DataAccessFieldDefinition(field, arrayFilterId); } } diff --git a/src/DataAccess/src/SIL.DataAccess/ParameterReplacer.cs b/src/DataAccess/src/SIL.DataAccess/ParameterReplacer.cs new file mode 100644 index 00000000..0f5c36e0 --- /dev/null +++ b/src/DataAccess/src/SIL.DataAccess/ParameterReplacer.cs @@ -0,0 +1,16 @@ +namespace SIL.DataAccess; + +internal class ParameterReplacer(ParameterExpression oldExpression, Expression newExpression) + : System.Linq.Expressions.ExpressionVisitor +{ + private readonly ParameterExpression _oldExpression = oldExpression; + private readonly Expression _newExpression = newExpression; + + protected override Expression VisitParameter(ParameterExpression node) + { + if (node == _oldExpression) + return _newExpression; + + return base.VisitParameter(node); + } +} diff --git a/src/DataAccess/test/SIL.DataAccess.Tests/MemoryRepositoryTests.cs b/src/DataAccess/test/SIL.DataAccess.Tests/MemoryRepositoryTests.cs index 7859a93c..c56833c4 100644 --- a/src/DataAccess/test/SIL.DataAccess.Tests/MemoryRepositoryTests.cs +++ b/src/DataAccess/test/SIL.DataAccess.Tests/MemoryRepositoryTests.cs @@ -293,6 +293,30 @@ public async Task UpdateAsync_RemoveAll_ReadOnlyCollection() Assert.That(repo.Get("1").ReadOnlyCollection, Is.EqualTo(new int[] { 1 })); } + [Test] + public async Task UpdateAsync_SetAll() + { + MemoryRepository repo = new(); + repo.Add( + new TestEntity() + { + Id = "1", + Children = [new Child { Field = 1 }, new Child { Field = 2 }, new Child { Field = 3 }] + } + ); + + TestEntity? entity = await repo.UpdateAsync( + "1", + u => u.SetAll(e => e.Children, c => c.Field, 0, c => c.Field >= 2) + ); + + Assert.That(entity, Is.Not.Null); + Assert.That(entity.Children, Is.Not.Null); + Assert.That(entity.Children[0].Field, Is.EqualTo(1)); + Assert.That(entity.Children[1].Field, Is.EqualTo(0)); + Assert.That(entity.Children[2].Field, Is.EqualTo(0)); + } + [Test] public async Task DeleteAsync_DoesNotExist() { @@ -349,5 +373,11 @@ private record TestEntity : IEntity public List? List { get; init; } public int[]? Array { get; init; } public ReadOnlyCollection? ReadOnlyCollection { get; init; } + public IReadOnlyList? Children { get; init; } + } + + private record Child + { + public int Field { get; init; } } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs index e0d44795..484c1eda 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs @@ -55,7 +55,7 @@ private async Task ReleaseAllWriterLocksAsync(CancellationToken cancellationToke await _locks.UpdateAllAsync( rwl => rwl.WriterLock != null && rwl.WriterLock.HostId == _serviceOptions.ServiceId, u => u.Unset(rwl => rwl.WriterLock), - cancellationToken + cancellationToken: cancellationToken ); } @@ -64,7 +64,7 @@ private async Task ReleaseAllReaderLocksAsync(CancellationToken cancellationToke await _locks.UpdateAllAsync( rwl => rwl.ReaderLocks.Any(l => l.HostId == _serviceOptions.ServiceId), u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken + cancellationToken: cancellationToken ); } @@ -73,7 +73,7 @@ private async Task RemoveAllWaitersAsync(CancellationToken cancellationToken) await _locks.UpdateAllAsync( rwl => rwl.WriterQueue.Any(l => l.HostId == _serviceOptions.ServiceId), u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken + cancellationToken: cancellationToken ); } } diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index f7645763..b0edfda4 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -7073,9 +7073,37 @@ public partial class Corpus [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class CorpusFile { - [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CorpusConfig + { + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public DataFile File { get; set; } = new DataFile(); + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CorpusFileConfig + { + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? TextId { get; set; } = default!; @@ -7118,34 +7146,6 @@ public enum FileFormat } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class CorpusConfig - { - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Language { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class CorpusFileConfig - { - [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string FileId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationEngine { diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs index c369d528..02e7d3c3 100644 --- a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs +++ b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs @@ -1,8 +1,9 @@ namespace Serval.DataFiles.Consumers; -public class GetCorpusConsumer(ICorpusService corpusService) : IConsumer +public class GetCorpusConsumer(ICorpusService corpusService, IDataFileService dataFileService) : IConsumer { private readonly ICorpusService _corpusService = corpusService; + private readonly IDataFileService _dataFileService = dataFileService; public async Task Consume(ConsumeContext context) { @@ -19,19 +20,13 @@ await context.RespondAsync( CorpusId = corpus.Id, Name = corpus.Name, Language = corpus.Language, - Files = corpus - .Files.Select(f => new CorpusFileResult + Files = await Task.WhenAll( + corpus.Files.Select(async f => new CorpusFileResult { TextId = f.TextId!, - File = new DataFileResult - { - DataFileId = f.File.Id, - Filename = f.File.Filename, - Format = f.File.Format, - Name = f.File.Name - } + File = Map(await _dataFileService.GetAsync(f.FileId)) }) - .ToList() + ) } ); } @@ -42,4 +37,15 @@ await context.RespondAsync( ); } } + + private static DataFileResult Map(DataFile dataFile) + { + return new DataFileResult + { + DataFileId = dataFile.Id, + Name = dataFile.Name, + Filename = dataFile.Filename, + Format = dataFile.Format, + }; + } } diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs index d2d175be..0efeaac9 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs +++ b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs @@ -2,6 +2,6 @@ namespace Serval.DataFiles.Contracts; public record CorpusFileDto { - public required DataFileDto File { get; init; } + public required string FileId { get; init; } public string? TextId { get; init; } } diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs index 29bf041e..c30a5ed3 100644 --- a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs +++ b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs @@ -177,7 +177,7 @@ CancellationToken cancellationToken DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken); if (dataFile == null) throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist."); - dataFiles.Add(new CorpusFile { File = dataFile, TextId = file.TextId }); + dataFiles.Add(new CorpusFile { FileId = file.FileId, TextId = file.TextId }); } return dataFiles; } @@ -197,18 +197,6 @@ private CorpusDto Map(Corpus source) private CorpusFileDto Map(CorpusFile source) { - return new CorpusFileDto { File = Map(source.File), TextId = source.TextId }; - } - - private DataFileDto Map(DataFile source) - { - return new DataFileDto - { - Id = source.Id, - Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }), - Name = source.Name, - Format = source.Format, - Revision = source.Revision - }; + return new CorpusFileDto { FileId = source.FileId, TextId = source.TextId }; } } diff --git a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs index a4311e39..277c9ccd 100644 --- a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs @@ -2,6 +2,6 @@ namespace Serval.DataFiles.Models; public record CorpusFile { - public required DataFile File { get; init; } + public required string FileId { get; init; } public string? TextId { get; init; } } diff --git a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs index f5b8e4b6..61622a93 100644 --- a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs @@ -1,7 +1,17 @@ namespace Serval.DataFiles.Services; -public class CorpusService(IRepository corpora) : OwnedEntityServiceBase(corpora), ICorpusService +public class CorpusService( + IRepository corpora, + IDataAccessContext dataAccessContext, + IDataFileService dataFileService, + IScopedMediator mediator +) : OwnedEntityServiceBase(corpora), ICorpusService { + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IDataFileService _dataFileService = dataFileService; + + private readonly IScopedMediator _mediator = mediator; + public async Task GetAsync(string id, string owner, CancellationToken cancellationToken = default) { Corpus? corpus = await Entities.GetAsync(c => c.Id == id && c.Owner == owner, cancellationToken); @@ -16,13 +26,44 @@ public async Task UpdateAsync( CancellationToken cancellationToken = default ) { - Corpus? corpus = await Entities.UpdateAsync( - c => c.Id == id, - u => u.Set(c => c.Files, files), - cancellationToken: cancellationToken + return await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Corpus? corpus = await Entities.UpdateAsync( + c => c.Id == id, + u => u.Set(c => c.Files, files), + cancellationToken: cancellationToken + ); + if (corpus is null) + throw new EntityNotFoundException($"Could not find Corpus '{id}."); + await _mediator.Publish( + new CorpusUpdated + { + CorpusId = corpus.Id, + Files = await Task.WhenAll( + corpus.Files.Select(async f => new CorpusFileResult + { + TextId = f.TextId!, + File = Map(await _dataFileService.GetAsync(f.FileId)) + }) + ) + }, + ct + ); + return corpus; + }, + cancellationToken ); - if (corpus is null) - throw new EntityNotFoundException($"Could not find Corpus '{id}."); - return corpus; + } + + private static DataFileResult Map(DataFile dataFile) + { + return new DataFileResult + { + DataFileId = dataFile.Id, + Name = dataFile.Name, + Filename = dataFile.Filename, + Format = dataFile.Format, + }; } } diff --git a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs index 6df3a01f..b7810f1c 100644 --- a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs @@ -89,6 +89,7 @@ await _deletedFiles.InsertAsync( cancellationToken: ct ); } + await _mediator.Publish(new DataFileUpdated { DataFileId = id, Filename = filename }, ct); }, cancellationToken: cancellationToken ); diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs new file mode 100644 index 00000000..402658f3 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusUpdated.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Contracts; + +public record CorpusUpdated +{ + public required string CorpusId { get; init; } + public required IReadOnlyList Files { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs index 1d32e196..7968f28a 100644 --- a/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs +++ b/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs @@ -3,4 +3,5 @@ public record DataFileUpdated { public required string DataFileId { get; init; } + public required string Filename { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs index 1397ed4d..d9b91b0f 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -7,6 +7,8 @@ this IMediatorRegistrationConfigurator configurator ) { configurator.AddConsumer(); + configurator.AddConsumer(); + configurator.AddConsumer(); return configurator; } } diff --git a/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs new file mode 100644 index 00000000..366e8099 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs @@ -0,0 +1,26 @@ +namespace Serval.Translation.Consumers; + +public class CorpusUpdatedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public async Task Consume(ConsumeContext context) + { + await _engineService.UpdateCorpusFilesAsync( + context.Message.CorpusId, + context.Message.Files.Select(Map).ToList(), + context.CancellationToken + ); + } + + private static CorpusFile Map(CorpusFileResult corpusFile) + { + return new CorpusFile + { + Id = corpusFile.File.DataFileId, + TextId = corpusFile.TextId, + Filename = corpusFile.File.Filename, + Format = corpusFile.File.Format, + }; + } +} diff --git a/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs new file mode 100644 index 00000000..b75572f7 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs @@ -0,0 +1,15 @@ +namespace Serval.Translation.Consumers; + +public class DataFileUpdatedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public async Task Consume(ConsumeContext context) + { + await _engineService.UpdateDataFileFilenameFilesAsync( + context.Message.DataFileId, + context.Message.Filename, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 5e653059..b0cfb5be 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -561,11 +561,101 @@ public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cance e => e.Corpora.Any(c => c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId) + ) + || e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) + || c.TargetCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) ), u => - u.RemoveAll(e => e.Corpora[ArrayPosition.All].SourceFiles, f => f.Id == dataFileId) - .RemoveAll(e => e.Corpora[ArrayPosition.All].TargetFiles, f => f.Id == dataFileId), - cancellationToken + { + u.RemoveAll(e => e.Corpora[ArrayPosition.All].SourceFiles, f => f.Id == dataFileId); + u.RemoveAll(e => e.Corpora[ArrayPosition.All].TargetFiles, f => f.Id == dataFileId); + u.RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ); + u.RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ); + }, + cancellationToken: cancellationToken + ); + } + + public Task UpdateDataFileFilenameFilesAsync( + string dataFileId, + string filename, + CancellationToken cancellationToken = default + ) + { + return Entities.UpdateAllAsync( + e => + e.Corpora.Any(c => + c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId) + ) + || e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) + || c.TargetCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) + ), + u => + { + u.SetAll( + e => e.Corpora[ArrayPosition.All].SourceFiles, + f => f.Filename, + filename, + f => f.Id == dataFileId + ); + u.SetAll( + e => e.Corpora[ArrayPosition.All].TargetFiles, + f => f.Filename, + filename, + f => f.Id == dataFileId + ); + u.SetAll( + e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files, + f => f.Filename, + filename, + f => f.Id == dataFileId + ); + u.SetAll( + e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files, + f => f.Filename, + filename, + f => f.Id == dataFileId + ); + }, + cancellationToken: cancellationToken + ); + } + + public Task UpdateCorpusFilesAsync( + string corpusId, + IReadOnlyList files, + CancellationToken cancellationToken = default + ) + { + return Entities.UpdateAllAsync( + e => + e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(mc => mc.Id == corpusId) || c.TargetCorpora.Any(mc => mc.Id == corpusId) + ), + u => + { + u.SetAll( + e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora, + mc => mc.Files, + files, + mc => mc.Id == corpusId + ); + u.SetAll( + e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora, + mc => mc.Files, + files, + mc => mc.Id == corpusId + ); + }, + cancellationToken: cancellationToken ); } diff --git a/src/Serval/src/Serval.Translation/Services/IEngineService.cs b/src/Serval/src/Serval.Translation/Services/IEngineService.cs index 6497ac1a..c25d522c 100644 --- a/src/Serval/src/Serval.Translation/Services/IEngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/IEngineService.cs @@ -68,6 +68,18 @@ Task DeleteParallelCorpusAsync( Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); + Task UpdateDataFileFilenameFilesAsync( + string dataFileId, + string filename, + CancellationToken cancellationToken = default + ); + + Task UpdateCorpusFilesAsync( + string corpusId, + IReadOnlyList files, + CancellationToken cancellationToken = default + ); + Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); Task GetLanguageInfoAsync( diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index d66b3557..07ccb9f2 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -19,11 +19,11 @@ public class TranslationEngineTests TargetLanguage = "en", SourceFiles = { - new TranslationCorpusFileConfig { FileId = FILE1_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE1_SRC_ID, TextId = "all" } }, TargetFiles = { - new TranslationCorpusFileConfig { FileId = FILE2_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE2_TRG_ID, TextId = "all" } } }; private static readonly TranslationParallelCorpusConfig TestParallelCorpusConfig = @@ -49,11 +49,11 @@ public class TranslationEngineTests TargetLanguage = "es", SourceFiles = { - new TranslationCorpusFileConfig { FileId = FILE1_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE1_SRC_ID, TextId = "all" } }, TargetFiles = { - new TranslationCorpusFileConfig { FileId = FILE2_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE2_TRG_ID, TextId = "all" } } }; @@ -63,8 +63,8 @@ public class TranslationEngineTests Name = "TestCorpus", SourceLanguage = "en", TargetLanguage = "en", - SourceFiles = { new TranslationCorpusFileConfig { FileId = FILE3_ID } }, - TargetFiles = { new TranslationCorpusFileConfig { FileId = FILE4_ID } } + SourceFiles = { new TranslationCorpusFileConfig { FileId = FILE3_SRC_ZIP_ID } }, + TargetFiles = { new TranslationCorpusFileConfig { FileId = FILE4_TRG_ZIP_ID } } }; private const string ECHO_ENGINE1_ID = "e00000000000000000000001"; @@ -72,13 +72,13 @@ public class TranslationEngineTests private const string ECHO_ENGINE3_ID = "e00000000000000000000003"; private const string SMT_ENGINE1_ID = "be0000000000000000000001"; private const string NMT_ENGINE1_ID = "ce0000000000000000000001"; - private const string FILE1_ID = "f00000000000000000000001"; + private const string FILE1_SRC_ID = "f00000000000000000000001"; private const string FILE1_FILENAME = "file_a"; - private const string FILE2_ID = "f00000000000000000000002"; + private const string FILE2_TRG_ID = "f00000000000000000000002"; private const string FILE2_FILENAME = "file_b"; - private const string FILE3_ID = "f00000000000000000000003"; + private const string FILE3_SRC_ZIP_ID = "f00000000000000000000003"; private const string FILE3_FILENAME = "file_c"; - private const string FILE4_ID = "f00000000000000000000004"; + private const string FILE4_TRG_ZIP_ID = "f00000000000000000000004"; private const string FILE4_FILENAME = "file_d"; private const string SOURCE_CORPUS_ID_1 = "cc0000000000000000000001"; private const string SOURCE_CORPUS_ID_2 = "cc0000000000000000000002"; @@ -147,7 +147,7 @@ public async Task SetUp() var srcFile = new DataFiles.Models.DataFile { - Id = FILE1_ID, + Id = FILE1_SRC_ID, Owner = "client1", Name = "src.txt", Filename = FILE1_FILENAME, @@ -155,7 +155,7 @@ public async Task SetUp() }; var trgFile = new DataFiles.Models.DataFile { - Id = FILE2_ID, + Id = FILE2_TRG_ID, Owner = "client1", Name = "trg.txt", Filename = FILE2_FILENAME, @@ -163,7 +163,7 @@ public async Task SetUp() }; var srcParatextFile = new DataFiles.Models.DataFile { - Id = FILE3_ID, + Id = FILE3_SRC_ZIP_ID, Owner = "client1", Name = "src.zip", Filename = FILE3_FILENAME, @@ -171,7 +171,7 @@ public async Task SetUp() }; var trgParatextFile = new DataFiles.Models.DataFile { - Id = FILE4_ID, + Id = FILE4_TRG_ZIP_ID, Owner = "client1", Name = "trg.zip", Filename = FILE4_FILENAME, @@ -184,21 +184,21 @@ public async Task SetUp() Id = SOURCE_CORPUS_ID_1, Language = "en", Owner = "client1", - Files = [new() { File = srcFile, TextId = "all" }] + Files = [new() { FileId = srcFile.Id, TextId = "all" }] }; var srcCorpus2 = new DataFiles.Models.Corpus { Id = SOURCE_CORPUS_ID_2, Language = "en", Owner = "client1", - Files = [new() { File = srcFile, TextId = "all" }] + Files = [new() { FileId = srcFile.Id, TextId = "all" }] }; var trgCorpus = new DataFiles.Models.Corpus { Id = TARGET_CORPUS_ID, Language = "en", Owner = "client1", - Files = [new() { File = trgFile, TextId = "all" }] + Files = [new() { FileId = trgFile.Id, TextId = "all" }] }; await _env.Corpora.InsertAllAsync([srcCorpus, srcCorpus2, trgCorpus]); } @@ -594,8 +594,8 @@ public async Task AddCorpusToEngineByIdAsync(IEnumerable scope, int expe Assert.Multiple(() => { Assert.That(result.Name, Is.EqualTo("TestCorpus")); - Assert.That(result.SourceFiles.First().File.Id, Is.EqualTo(FILE1_ID)); - Assert.That(result.TargetFiles.First().File.Id, Is.EqualTo(FILE2_ID)); + Assert.That(result.SourceFiles.First().File.Id, Is.EqualTo(FILE1_SRC_ID)); + Assert.That(result.TargetFiles.First().File.Id, Is.EqualTo(FILE2_TRG_ID)); }); Engine? engine = await _env.Engines.GetAsync(engineId); Assert.That(engine, Is.Not.Null); @@ -651,11 +651,11 @@ string engineId TranslationCorpus result = await client.AddCorpusAsync(engineId, TestCorpusConfig); TranslationCorpusFileConfig[] src = new[] { - new TranslationCorpusFileConfig { FileId = FILE2_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE2_TRG_ID, TextId = "all" } }; TranslationCorpusFileConfig[] trg = new[] { - new TranslationCorpusFileConfig { FileId = FILE1_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE1_SRC_ID, TextId = "all" } }; var updateConfig = new TranslationCorpusUpdateConfig { SourceFiles = src, TargetFiles = trg }; await client.UpdateCorpusAsync(engineId, result.Id, updateConfig); @@ -675,11 +675,11 @@ string engineId { TranslationCorpusFileConfig[] src = new[] { - new TranslationCorpusFileConfig { FileId = FILE2_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE2_TRG_ID, TextId = "all" } }; TranslationCorpusFileConfig[] trg = new[] { - new TranslationCorpusFileConfig { FileId = FILE1_ID, TextId = "all" } + new TranslationCorpusFileConfig { FileId = FILE1_SRC_ID, TextId = "all" } }; var updateConfig = new TranslationCorpusUpdateConfig { SourceFiles = src, TargetFiles = trg }; await client.UpdateCorpusAsync(engineId, DOES_NOT_EXIST_CORPUS_ID, updateConfig); @@ -2031,6 +2031,53 @@ public void GetLanguageInfo_Error() Assert.That(ex.StatusCode, Is.EqualTo(403)); } + [Test] + public async Task DataFileUpdate_Propagated() + { + TranslationEnginesClient translationClient = _env.CreateTranslationEnginesClient(); + DataFilesClient dataFilesClient = _env.CreateDataFilesClient(); + CorporaClient corporaClient = _env.CreateCorporaClient(); + await translationClient.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfig); + await translationClient.AddParallelCorpusAsync(ECHO_ENGINE2_ID, TestParallelCorpusConfig); + + // Get the original files + DataFile orgFileFromClient = await dataFilesClient.GetAsync(FILE1_SRC_ID); + DataFiles.Models.DataFile orgFileFromRepo = (await _env.DataFiles.GetAsync(FILE1_SRC_ID))!; + DataFiles.Models.Corpus orgCorpusFromRepo = (await _env.Corpora.GetAsync(TARGET_CORPUS_ID))!; + Assert.That(orgFileFromClient.Name, Is.EqualTo(orgFileFromRepo.Name)); + Assert.That(orgCorpusFromRepo.Files[0].FileId, Is.EqualTo(FILE2_TRG_ID)); + + // Update the file + await dataFilesClient.UpdateAsync(FILE1_SRC_ID, new FileParameter(new MemoryStream([1, 2, 3]), "test.txt")); + await corporaClient.UpdateAsync( + TARGET_CORPUS_ID, + [new CorpusFileConfig { FileId = FILE4_TRG_ZIP_ID, TextId = "all" }] + ); + + // Confirm the change is propagated everywhere + DataFiles.Models.DataFile newFileFromRepo = (await _env.DataFiles.GetAsync(FILE1_SRC_ID))!; + Assert.That(newFileFromRepo.Filename, Is.Not.EqualTo(orgFileFromRepo.Filename)); + + Engine newEngine1 = (await _env.Engines.GetAsync(ECHO_ENGINE1_ID))!; + Engine newEngine2 = (await _env.Engines.GetAsync(ECHO_ENGINE2_ID))!; + + // Updated (legacy) Corpus file filename + Assert.That(newEngine1.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(newFileFromRepo.Filename)); + Assert.That(newEngine1.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); + + // Updated parallel corpus file filename + Assert.That( + newEngine2.ParallelCorpora[0].SourceCorpora[0].Files[0].Filename, + Is.EqualTo(newFileFromRepo.Filename) + ); + + // Updated set of new corpus files + Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Id, Is.EqualTo(TARGET_CORPUS_ID)); + Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Files[0].Id, Is.EqualTo(FILE4_TRG_ZIP_ID)); + Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Files[0].Filename, Is.EqualTo(FILE4_FILENAME)); + Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Files.Count, Is.EqualTo(1)); + } + [TearDown] public void TearDown() { @@ -2235,13 +2282,13 @@ public TestEnvironment() public TranslationEnginesClient CreateTranslationEnginesClient(IEnumerable? scope = null) { - scope ??= new[] - { + scope ??= + [ Scopes.CreateTranslationEngines, Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines, Scopes.DeleteTranslationEngines - }; + ]; HttpClient httpClient = Factory .WithWebHostBuilder(builder => { @@ -2265,13 +2312,13 @@ public TranslationEnginesClient CreateTranslationEnginesClient(IEnumerable? scope = null) { - scope ??= new[] - { + scope ??= + [ Scopes.CreateTranslationEngines, Scopes.ReadTranslationEngines, Scopes.UpdateTranslationEngines, Scopes.DeleteTranslationEngines - }; + ]; HttpClient httpClient = Factory .WithWebHostBuilder(builder => { @@ -2300,6 +2347,32 @@ public TranslationEngineTypesClient CreateTranslationEngineTypesClient(IEnumerab return new TranslationEngineTypesClient(httpClient); } + public DataFilesClient CreateDataFilesClient() + { + IEnumerable scope = [Scopes.DeleteFiles, Scopes.ReadFiles, Scopes.UpdateFiles, Scopes.CreateFiles]; + HttpClient httpClient = Factory + .WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + services.AddTransient(CreateFileSystem); + }); + }) + .CreateClient(); + if (scope is not null) + httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); + return new DataFilesClient(httpClient); + } + + public CorporaClient CreateCorporaClient() + { + IEnumerable scope = [Scopes.DeleteFiles, Scopes.ReadFiles, Scopes.UpdateFiles, Scopes.CreateFiles]; + HttpClient httpClient = Factory.WithWebHostBuilder(_ => { }).CreateClient(); + if (scope is not null) + httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); + return new CorporaClient(httpClient); + } + public void ResetDatabases() { _mongoClient.DropDatabase("serval_test"); @@ -2337,6 +2410,7 @@ private static IFileSystem CreateFileSystem(IServiceProvider sp) target.EntryExists("MATTRG.SFM").Returns(false); return target; }); + fileSystem.OpenWrite(Arg.Any()).Returns(ci => new MemoryStream()); return fileSystem; } diff --git a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs index 22cdd14e..a9e498b4 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs +++ b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs @@ -21,7 +21,7 @@ public class CorpusServiceTests Owner = "owner1", Name = "corpus1", Language = "en", - Files = new List() { new() { File = DefaultDataFile } } + Files = new List() { new() { FileId = DefaultDataFile.Id } } }; [Test] @@ -47,11 +47,25 @@ private class TestEnvironment public TestEnvironment() { Corpora = new MemoryRepository(); - Service = new CorpusService(Corpora); + DataAccessContext = Substitute.For(); + DataAccessContext + .WithTransactionAsync(Arg.Any>>(), Arg.Any()) + .Returns(x => + { + return ((Func>)x[0])((CancellationToken)x[1]); + }); + Service = new CorpusService( + Corpora, + DataAccessContext, + Substitute.For(), + Substitute.For() + ); } public MemoryRepository Corpora { get; } public CorpusService Service { get; } + + public IDataAccessContext DataAccessContext { get; } } }