Cleanup data access

- remove unnecessary Mongo attributes - fix various coding convention inconsistencies - reimplement upsert support - cleanup integration tests
sillsdev · Aug 31, 2023 · a935d67 · a935d67
1 parent e688077
commit a935d67
Show file tree

Hide file tree

Showing 29 changed files with 363 additions and 571 deletions.
diff --git a/Serval.sln b/Serval.sln
@@ -50,7 +50,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.ApiServer.Integratio
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.E2ETests", "tests\Serval.E2ETests\Serval.E2ETests.csproj", "{1F020042-D7B8-4541-9691-26ECFD1FFC73}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.DataFiles.Tests", "tests\Serval.DataFiles.Tests\Serval.DataFiles.Tests.csproj", "{63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.DataFiles.Tests", "tests\Serval.DataFiles.Tests\Serval.DataFiles.Tests.csproj", "{63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

diff --git a/src/SIL.DataAccess/DataAccessExtensions.cs b/src/SIL.DataAccess/DataAccessExtensions.cs
@@ -49,24 +49,26 @@ public static async Task<bool> ExistsAsync<T>(
         this IRepository<T> repo,
         string id,
         Action<IUpdateBuilder<T>> update,
+        bool upsert = false,
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     )
         where T : IEntity
     {
-        return repo.UpdateAsync(e => e.Id == id, update, returnOriginal, cancellationToken);
+        return repo.UpdateAsync(e => e.Id == id, update, upsert, returnOriginal, cancellationToken);
     }
 
     public static Task<T?> UpdateAsync<T>(
         this IRepository<T> repo,
         T entity,
         Action<IUpdateBuilder<T>> update,
+        bool upsert = false,
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     )
         where T : IEntity
     {
-        return repo.UpdateAsync(e => e.Id == entity.Id, update, returnOriginal, cancellationToken);
+        return repo.UpdateAsync(e => e.Id == entity.Id, update, upsert, returnOriginal, cancellationToken);
     }
 
     public static Task<T?> DeleteAsync<T>(

diff --git a/src/SIL.DataAccess/IEntity.cs b/src/SIL.DataAccess/IEntity.cs
@@ -2,8 +2,6 @@ namespace SIL.DataAccess;
 
 public interface IEntity
 {
-    [BsonId]
-    [BsonRepresentation(BsonType.ObjectId)]
     string Id { get; set; }
     int Revision { get; set; }
 }
diff --git a/src/SIL.DataAccess/IRepository.cs b/src/SIL.DataAccess/IRepository.cs
@@ -12,6 +12,7 @@ public interface IRepository<T>
     Task<T?> UpdateAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        bool upsert = false,
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     );

diff --git a/src/SIL.DataAccess/MemoryRepository.cs b/src/SIL.DataAccess/MemoryRepository.cs
@@ -116,6 +116,8 @@ public async Task<bool> ExistsAsync(Expression<Func<T, bool>> filter, Cancellati
     public async Task InsertAsync(T entity, CancellationToken cancellationToken = default)
     {
         entity.Revision = 1;
+        if (string.IsNullOrEmpty(entity.Id))
+            entity.Id = ObjectId.GenerateNewId().ToString();
         var allSubscriptions = new List<MemorySubscription<T>>();
         string serializedEntity;
         using (await _lock.LockAsync(cancellationToken))
@@ -134,6 +136,8 @@ public async Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationTo
         foreach (T entity in entities)
         {
             entity.Revision = 1;
+            if (string.IsNullOrEmpty(entity.Id))
+                entity.Id = ObjectId.GenerateNewId().ToString();
         }
         var serializedEntities = new List<(string, string, List<MemorySubscription<T>>)>();
         using (await _lock.LockAsync(cancellationToken))
@@ -156,6 +160,7 @@ public async Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationTo
     public async Task<T?> UpdateAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        bool upsert = false,
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     )
@@ -178,17 +183,34 @@ public async Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationTo
                     return false;
                 }
             });
-            if (entity == null)
-                return entity; // file not found
+            if (entity != null || upsert)
+            {
+                bool isInsert = entity == null;
+                if (isInsert)
+                {
+                    entity = (T)Activator.CreateInstance(typeof(T))!;
+                    string? id = ExpressionHelper.FindEqualsConstantValue<T, string>(e => e.Id, filter.Body);
+                    entity.Id = id ?? ObjectId.GenerateNewId().ToString();
+                    entity.Revision = 0;
+                }
+                else
+                {
+                    original = Entities.AsQueryable().FirstOrDefault(filter);
+                }
+                Debug.Assert(entity != null);
+                var builder = new MemoryUpdateBuilder<T>(filter, entity, isInsert);
+                update(builder);
+                entity.Revision++;
 
-            original = Entities.AsQueryable().FirstOrDefault(filter);
-            Debug.Assert(entity != null);
-            var builder = new MemoryUpdateBuilder<T>(filter, entity, isInsert: false);
-            update(builder);
-            entity.Revision++;
+                if (isInsert && Contains(entity.Id))
+                    throw new DuplicateKeyException();
 
-            serializedEntity = Replace(entity);
-            GetSubscriptions(entity, allSubscriptions);
+                if (CheckDuplicateKeys(entity, original))
+                    throw new DuplicateKeyException();
+
+                serializedEntity = Replace(entity);
+                GetSubscriptions(entity, allSubscriptions);
+            }
         }
         if (entity != null && serializedEntity != null)
             SendToSubscribers(allSubscriptions, EntityChangeType.Update, entity.Id, serializedEntity);
@@ -212,6 +234,9 @@ public async Task<int> UpdateAllAsync(
                 update(builder);
                 entity.Revision++;
 
+                if (CheckDuplicateKeys(entity, original))
+                    throw new DuplicateKeyException();
+
                 Replace(entity);
             }
 

diff --git a/src/SIL.DataAccess/MongoRepository.cs b/src/SIL.DataAccess/MongoRepository.cs
@@ -55,7 +55,7 @@ public async Task<bool> ExistsAsync(Expression<Func<T, bool>> filter, Cancellati
     public async Task InsertAsync(T entity, CancellationToken cancellationToken = default)
     {
         entity.Revision = 1;
-        await TryCatchDuplicate<object?>(async () =>
+        await TryCatchDuplicate(async () =>
         {
             if (_context.Session is not null)
             {
@@ -67,7 +67,6 @@ await _collection
             {
                 await _collection.InsertOneAsync(entity, cancellationToken: cancellationToken).ConfigureAwait(false);
             }
-            return null;
         });
     }
 
@@ -76,7 +75,7 @@ public async Task InsertAllAsync(IReadOnlyCollection<T> entities, CancellationTo
         foreach (T entity in entities)
             entity.Revision = 1;
 
-        await TryCatchDuplicate<object?>(async () =>
+        await TryCatchDuplicate(async () =>
         {
             if (_context.Session is not null)
             {
@@ -88,13 +87,13 @@ await _collection
             {
                 await _collection.InsertManyAsync(entities, cancellationToken: cancellationToken).ConfigureAwait(false);
             }
-            return null;
         });
     }
 
     public async Task<T?> UpdateAsync(
         Expression<Func<T, bool>> filter,
         Action<IUpdateBuilder<T>> update,
+        bool upsert = false,
         bool returnOriginal = false,
         CancellationToken cancellationToken = default
     )
@@ -105,6 +104,7 @@ await _collection
         UpdateDefinition<T> updateDef = updateBuilder.Build();
         var options = new FindOneAndUpdateOptions<T>
         {
+            IsUpsert = upsert,
             ReturnDocument = returnOriginal ? ReturnDocument.Before : ReturnDocument.After
         };
         T? entity;
@@ -221,11 +221,11 @@ public async Task<ISubscription<T>> SubscribeAsync(
         return subscription;
     }
 
-    private async Task<TR> TryCatchDuplicate<TR>(Func<Task<TR>> func)
+    private async Task TryCatchDuplicate(Func<Task> action)
     {
         try
         {
-            return await func();
+            await action();
         }
         catch (MongoCommandException e)
         {

diff --git a/src/SIL.DataAccess/Usings.cs b/src/SIL.DataAccess/Usings.cs
@@ -11,7 +11,6 @@
 global using Microsoft.Extensions.Options;
 global using MongoDB.Bson;
 global using MongoDB.Bson.Serialization;
-global using MongoDB.Bson.Serialization.Attributes;
 global using MongoDB.Bson.Serialization.Conventions;
 global using MongoDB.Bson.Serialization.Serializers;
 global using MongoDB.Driver;

diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs
@@ -2998,7 +2998,7 @@ public string BaseUrl
                         if (status_ == 400)
                         {
                             string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
-                            throw new ServalApiException("Bad request", status_, responseText_, headers_, null);
+                            throw new ServalApiException("A corpus id is invalid", status_, responseText_, headers_, null);
                         }
                         else
                         if (status_ == 401)
@@ -3016,7 +3016,7 @@ public string BaseUrl
                         if (status_ == 404)
                         {
                             string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
-                            throw new ServalApiException("The engine or corpora specified in the config do not exist", status_, responseText_, headers_, null);
+                            throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null);
                         }
                         else
                         if (status_ == 409)
@@ -3028,7 +3028,7 @@ public string BaseUrl
                         if (status_ == 503)
                         {
                             string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
-                            throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null);
+                            throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null);
                         }
                         else
                         {

diff --git a/src/Serval.DataFiles/Models/DataFile.cs b/src/Serval.DataFiles/Models/DataFile.cs
@@ -2,8 +2,6 @@
 
 public class DataFile : IOwnedEntity
 {
-    [BsonId]
-    [BsonRepresentation(BsonType.ObjectId)]
     public string Id { get; set; } = default!;
     public int Revision { get; set; } = 1;
     public string Owner { get; set; } = default!;

diff --git a/src/Serval.DataFiles/Models/DeletedFile.cs b/src/Serval.DataFiles/Models/DeletedFile.cs
@@ -2,8 +2,6 @@
 
 public class DeletedFile : IEntity
 {
-    [BsonId]
-    [BsonRepresentation(BsonType.ObjectId)]
     public string Id { get; set; } = default!;
     public int Revision { get; set; } = 1;
     public string Filename { get; set; } = default!;

diff --git a/src/Serval.DataFiles/Usings.cs b/src/Serval.DataFiles/Usings.cs
@@ -13,8 +13,6 @@
 global using Microsoft.Extensions.Hosting;
 global using Microsoft.Extensions.Logging;
 global using Microsoft.Extensions.Options;
-global using MongoDB.Bson;
-global using MongoDB.Bson.Serialization.Attributes;
 global using NSwag.Annotations;
 global using Serval.DataFiles.Consumers;
 global using Serval.DataFiles.Contracts;

diff --git a/src/Serval.Shared/Utils/AutoToString.cs b/src/Serval.Shared/Utils/AutoToString.cs