Fix Filename sync issue (#555)

* Sync corpora and data files in translation engine Use mongoDB strings directly for Mongo array filtering (hacky solution - need to fix later #553). Broken implementation for MemoryRepository - #554 * Add SetAll operator to IUpdateBuilder * Cleanup MemoryUpdateBuilder * Set array filters in MongoRepository.UpdateAsync --------- Co-authored-by: Damien Daspit <[email protected]>
sillsdev · Dec 5, 2024 · f25fc43 · f25fc43
1 parent 6d22621
commit f25fc43
Show file tree

Hide file tree

Showing 28 changed files with 591 additions and 142 deletions.
diff --git a/docker-compose.mongo.yml b/docker-compose.mongo.yml
@@ -15,5 +15,5 @@ services:
       [
         '/bin/sh',
         '-c',
-        'mongod --quiet --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity'
+        'mongod --profile=2 --replSet myRS --bind_ip 0.0.0.0 & sleep 2s; mongosh --host localhost:27017 --eval '' config = { "_id" : "myRS", "members" : [{"_id" : 0,"host" : "mongo:27017"}] }; rs.initiate(config, { force: true }); '' ; sleep infinity'
       ]
diff --git a/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs b/src/DataAccess/src/SIL.DataAccess/ArrayPosition.cs
@@ -4,4 +4,5 @@ public static class ArrayPosition
 {
     public const int FirstMatching = int.MaxValue;
     public const int All = int.MaxValue - 1;
+    internal const int ArrayFilter = int.MaxValue - 2;
 }
diff --git a/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs b/src/DataAccess/src/SIL.DataAccess/DataAccessFieldDefinition.cs
@@ -1,9 +1,12 @@
 namespace SIL.DataAccess;
 
-public class DataAccessFieldDefinition<TDocument, TField>(Expression<Func<TDocument, TField>> expression)
-    : FieldDefinition<TDocument, TField>
+public class DataAccessFieldDefinition<TDocument, TField>(
+    Expression<Func<TDocument, TField>> expression,
+    string arrayFilterId = ""
+) : FieldDefinition<TDocument, TField>
 {
     private readonly ExpressionFieldDefinition<TDocument, TField> _internalDef = new(expression);
+    private readonly string _arrayFilterId = arrayFilterId;
 
     public override RenderedFieldDefinition<TField> Render(
         IBsonSerializer<TDocument> documentSerializer,
@@ -18,6 +21,10 @@ LinqProvider linqProvider
         );
         string fieldName = rendered.FieldName.Replace(ArrayPosition.All.ToString(CultureInfo.InvariantCulture), "$[]");
         fieldName = fieldName.Replace(ArrayPosition.FirstMatching.ToString(CultureInfo.InvariantCulture), "$");
+        fieldName = fieldName.Replace(
+            ArrayPosition.ArrayFilter.ToString(CultureInfo.InvariantCulture),
+            $"$[{_arrayFilterId}]"
+        );
         if (fieldName != rendered.FieldName)
         {
             return new RenderedFieldDefinition<TField>(

diff --git a/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs b/src/DataAccess/src/SIL.DataAccess/ExpressionHelper.cs
@@ -38,4 +38,14 @@ Expression expression
         finder.Visit(expression);
         return finder.Value;
     }
+
+    public static Expression<Func<TIn, TOut>> Concatenate<TIn, TInter, TOut>(
+        Expression<Func<TIn, TInter>> left,
+        Expression<Func<TInter, TOut>> right
+    )
+    {
+        ParameterReplacer replacer = new(right.Parameters[0], left.Body);
+        Expression merged = replacer.Visit(right.Body);
+        return Expression.Lambda<Func<TIn, TOut>>(merged, left.Parameters[0]);
+    }
 }
diff --git a/src/DataAccess/src/SIL.DataAccess/IRepository.cs b/src/DataAccess/src/SIL.DataAccess/IRepository.cs
@@ -9,6 +9,7 @@ public interface IRepository<T>
 
     Task InsertAsync(T entity, CancellationToken cancellationToken = default);
     Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationToken cancellationToken = default);
+
     Task<T?> UpdateAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
@@ -21,6 +22,7 @@ Task<int> UpdateAllAsync(
         Action<IUpdateBuilder<T>> update,
         CancellationToken cancellationToken = default
     );
+
     Task<T?> DeleteAsync(Expression<Func<T, bool>> filter, CancellationToken cancellationToken = default);
     Task<int> DeleteAllAsync(Expression<Func<T, bool>> filter, CancellationToken cancellationToken = default);
     Task<ISubscription<T>> SubscribeAsync(

diff --git a/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/IUpdateBuilder.cs
@@ -13,10 +13,17 @@ public interface IUpdateBuilder<T>
 
     IUpdateBuilder<T> RemoveAll<TItem>(
         Expression<Func<T, IEnumerable<TItem>?>> field,
-        Expression<Func<TItem, bool>> predicate
+        Expression<Func<TItem, bool>>? predicate = null
     );
 
     IUpdateBuilder<T> Remove<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value);
 
     IUpdateBuilder<T> Add<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value);
+
+    IUpdateBuilder<T> SetAll<TItem, TField>(
+        Expression<Func<T, IEnumerable<TItem>?>> collectionField,
+        Expression<Func<TItem, TField>> itemField,
+        TField value,
+        Expression<Func<TItem, bool>>? predicate = null
+    );
 }
diff --git a/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs b/src/DataAccess/src/SIL.DataAccess/MemoryUpdateBuilder.cs
@@ -9,23 +9,20 @@ public class MemoryUpdateBuilder<T>(Expression<Func<T, bool>> filter, T entity,
 
     public IUpdateBuilder<T> Set<TField>(Expression<Func<T, TField>> field, TField value)
     {
-        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(field);
-        object[]? indices = index == null ? null : [index];
-        foreach (object owner in owners)
-            prop.SetValue(owner, value, indices);
+        Set(_entity, _filter, field, value);
         return this;
     }
 
     public IUpdateBuilder<T> SetOnInsert<TField>(Expression<Func<T, TField>> field, TField value)
     {
         if (_isInsert)
-            Set(field, value);
+            Set(_entity, _filter, field, value);
         return this;
     }
 
     public IUpdateBuilder<T> Unset<TField>(Expression<Func<T, TField>> field)
     {
-        (IEnumerable<object> owners, PropertyInfo prop, object? index) = GetFieldOwners(field);
+        (IEnumerable<object> owners, PropertyInfo prop, object? index) = GetFieldOwners(_entity, _filter, field);
         if (index != null)
         {
             // remove value from a dictionary
@@ -49,7 +46,7 @@ public IUpdateBuilder<T> Unset<TField>(Expression<Func<T, TField>> field)
 
     public IUpdateBuilder<T> Inc(Expression<Func<T, int>> field, int value = 1)
     {
-        (IEnumerable<object> owners, PropertyInfo prop, object? index) = GetFieldOwners(field);
+        (IEnumerable<object> owners, PropertyInfo prop, object? index) = GetFieldOwners(_entity, _filter, field);
         object[]? indices = index == null ? null : [index];
         foreach (object owner in owners)
         {
@@ -62,20 +59,20 @@ public IUpdateBuilder<T> Inc(Expression<Func<T, int>> field, int value = 1)
 
     public IUpdateBuilder<T> RemoveAll<TItem>(
         Expression<Func<T, IEnumerable<TItem>?>> field,
-        Expression<Func<TItem, bool>> predicate
+        Expression<Func<TItem, bool>>? predicate = null
     )
     {
-        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(field);
+        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field);
         object[]? indices = index == null ? null : [index];
-        Func<TItem, bool> predicateFunc = predicate.Compile();
+        Func<TItem, bool>? predicateFunc = predicate?.Compile();
         foreach (object owner in owners)
         {
             var collection = (IEnumerable<TItem>?)prop.GetValue(owner, indices);
             MethodInfo? removeMethod = collection?.GetType().GetMethod("Remove");
             if (collection is not null && removeMethod is not null)
             {
                 // the collection is mutable, so use Remove method to remove item
-                TItem[] toRemove = collection.Where(predicateFunc).ToArray();
+                TItem[] toRemove = collection.Where(i => predicateFunc?.Invoke(i) ?? true).ToArray();
                 foreach (TItem item in toRemove)
                     removeMethod.Invoke(collection, [item]);
             }
@@ -84,14 +81,17 @@ Expression<Func<TItem, bool>> predicate
                 if (prop.PropertyType.IsArray || prop.PropertyType.IsInterface)
                 {
                     // the collection type is an array or interface, so construct a new array and set property
-                    TItem[] newValue = collection.Where(i => !predicateFunc(i)).ToArray();
+                    TItem[] newValue = collection.Where(i => !(predicateFunc?.Invoke(i) ?? false)).ToArray();
                     prop.SetValue(owner, newValue, indices);
                 }
                 else
                 {
                     // the collection type is a collection class, so construct a new collection and set property
                     var newValue = (IEnumerable<TItem>?)
-                        Activator.CreateInstance(prop.PropertyType, collection.Where(i => !predicateFunc(i)).ToArray());
+                        Activator.CreateInstance(
+                            prop.PropertyType,
+                            collection.Where(i => !(predicateFunc?.Invoke(i) ?? false)).ToArray()
+                        );
                     prop.SetValue(owner, newValue, indices);
                 }
             }
@@ -101,7 +101,7 @@ Expression<Func<TItem, bool>> predicate
 
     public IUpdateBuilder<T> Remove<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value)
     {
-        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(field);
+        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field);
         object[]? indices = index == null ? null : [index];
         foreach (object owner in owners)
         {
@@ -134,7 +134,7 @@ public IUpdateBuilder<T> Remove<TItem>(Expression<Func<T, IEnumerable<TItem>?>>
 
     public IUpdateBuilder<T> Add<TItem>(Expression<Func<T, IEnumerable<TItem>?>> field, TItem value)
     {
-        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(field);
+        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(_entity, _filter, field);
         object[]? indices = index == null ? null : [index];
         foreach (object owner in owners)
         {
@@ -147,7 +147,7 @@ public IUpdateBuilder<T> Add<TItem>(Expression<Func<T, IEnumerable<TItem>?>> fie
             }
             else
             {
-                collection ??= Array.Empty<TItem>();
+                collection ??= [];
                 if (prop.PropertyType.IsArray || prop.PropertyType.IsInterface)
                 {
                     // the collection type is an array or interface, so construct a new array and set property
@@ -166,6 +166,47 @@ public IUpdateBuilder<T> Add<TItem>(Expression<Func<T, IEnumerable<TItem>?>> fie
         return this;
     }
 
+    public IUpdateBuilder<T> SetAll<TItem, TField>(
+        Expression<Func<T, IEnumerable<TItem>?>> collectionField,
+        Expression<Func<TItem, TField>> itemField,
+        TField value,
+        Expression<Func<TItem, bool>>? predicate = null
+    )
+    {
+        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(
+            _entity,
+            _filter,
+            collectionField
+        );
+        object[]? indices = index == null ? null : [index];
+        Func<TItem, bool>? predicateFunc = predicate?.Compile();
+        foreach (object owner in owners)
+        {
+            var collection = (IEnumerable<TItem>?)prop.GetValue(owner, indices);
+            if (collection is null)
+                continue;
+            foreach (TItem item in collection)
+            {
+                if (predicateFunc == null || predicateFunc(item))
+                    Set(item, i => true, itemField, value);
+            }
+        }
+        return this;
+    }
+
+    private static void Set<TEntity, TField>(
+        TEntity entity,
+        Expression<Func<TEntity, bool>> filter,
+        Expression<Func<TEntity, TField>> field,
+        TField value
+    )
+    {
+        (IEnumerable<object> owners, PropertyInfo? prop, object? index) = GetFieldOwners(entity, filter, field);
+        object[]? indices = index == null ? null : [index];
+        foreach (object owner in owners)
+            prop.SetValue(owner, value, indices);
+    }
+
     private static bool IsAnyMethod(MethodInfo mi)
     {
         return mi.DeclaringType == typeof(Enumerable) && mi.Name == "Any";
@@ -180,8 +221,10 @@ private static MethodInfo GetFirstOrDefaultMethod(Type type)
             .MakeGenericMethod(type);
     }
 
-    private (IEnumerable<object> Owners, PropertyInfo Property, object? Index) GetFieldOwners<TField>(
-        Expression<Func<T, TField>> field
+    private static (IEnumerable<object> Owners, PropertyInfo Property, object? Index) GetFieldOwners<TEntity, TField>(
+        TEntity entity,
+        Expression<Func<TEntity, bool>> filter,
+        Expression<Func<TEntity, TField>> field
     )
     {
         List<object>? owners = null;
@@ -192,8 +235,8 @@ Expression<Func<T, TField>> field
             var newOwners = new List<object>();
             if (owners == null)
             {
-                if (_entity != null)
-                    newOwners.Add(_entity);
+                if (entity != null)
+                    newOwners.Add(entity);
             }
             else
             {
@@ -206,17 +249,14 @@ Expression<Func<T, TField>> field
                             switch (index)
                             {
                                 case ArrayPosition.FirstMatching:
-                                    foreach (Expression expression in ExpressionHelper.Flatten(_filter))
+                                    foreach (Expression expression in ExpressionHelper.Flatten(filter))
                                     {
                                         if (expression is MethodCallExpression callExpr && IsAnyMethod(callExpr.Method))
                                         {
                                             var predicate = (LambdaExpression)callExpr.Arguments[1];
                                             Type itemType = predicate.Parameters[0].Type;
                                             MethodInfo firstOrDefault = GetFirstOrDefaultMethod(itemType);
-                                            newOwner = firstOrDefault.Invoke(
-                                                null,
-                                                new object[] { owner, predicate.Compile() }
-                                            );
+                                            newOwner = firstOrDefault.Invoke(null, [owner, predicate.Compile()]);
                                             if (newOwner != null)
                                                 newOwners.Add(newOwner);
                                             break;
@@ -245,7 +285,7 @@ Expression<Func<T, TField>> field
                                     }
                                     else
                                     {
-                                        newOwner = method.Invoke(owner, new object[] { index });
+                                        newOwner = method.Invoke(owner, [index]);
                                         if (newOwner != null)
                                             newOwners.Add(newOwner);
                                     }

diff --git a/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs b/src/DataAccess/src/SIL.DataAccess/MongoRepository.cs
@@ -122,12 +122,14 @@ await _collection
         var updateBuilder = new MongoUpdateBuilder<T>();
         update(updateBuilder);
         updateBuilder.Inc(e => e.Revision, 1);
-        UpdateDefinition<T> updateDef = updateBuilder.Build();
+        (UpdateDefinition<T> updateDef, IReadOnlyList<ArrayFilterDefinition> arrayFilters) = updateBuilder.Build();
         var options = new FindOneAndUpdateOptions<T>
         {
             IsUpsert = upsert,
             ReturnDocument = returnOriginal ? ReturnDocument.Before : ReturnDocument.After
         };
+        if (arrayFilters.Count > 0)
+            options.ArrayFilters = arrayFilters;
         T? entity;
         try
         {
@@ -160,20 +162,23 @@ public async Task<int> UpdateAllAsync(
         var updateBuilder = new MongoUpdateBuilder<T>();
         update(updateBuilder);
         updateBuilder.Inc(e => e.Revision, 1);
-        UpdateDefinition<T> updateDef = updateBuilder.Build();
+        (UpdateDefinition<T> updateDef, IReadOnlyList<ArrayFilterDefinition> arrayFilters) = updateBuilder.Build();
+        UpdateOptions? updateOptions = null;
+        if (arrayFilters.Count > 0)
+            updateOptions = new UpdateOptions { ArrayFilters = arrayFilters };
         UpdateResult result;
         try
         {
             if (_context.Session is not null)
             {
                 result = await _collection
-                    .UpdateManyAsync(_context.Session, filter, updateDef, cancellationToken: cancellationToken)
+                    .UpdateManyAsync(_context.Session, filter, updateDef, updateOptions, cancellationToken)
                     .ConfigureAwait(false);
             }
             else
             {
                 result = await _collection
-                    .UpdateManyAsync(filter, updateDef, cancellationToken: cancellationToken)
+                    .UpdateManyAsync(filter, updateDef, updateOptions, cancellationToken)
                     .ConfigureAwait(false);
             }
         }