[ODS-6551] Support feature for optimized use of serialized aggregate …

…data (#1184)
Ed-Fi-Alliance-OSS · Nov 22, 2024 · aa35e6e · aa35e6e
1 parent 74ebd05
commit aa35e6e
Show file tree

Hide file tree

Showing 87 changed files with 88,639 additions and 30,523 deletions.
diff --git a/.github/workflows/Pkg EdFi.Ods.CodeGen.yml b/.github/workflows/Pkg EdFi.Ods.CodeGen.yml
@@ -129,3 +129,9 @@ jobs:
       with:
         name: NugetPackage.Artifacts
         path: ${{ github.workspace }}/Ed-Fi-ODS/NugetPackages/*.nupkg
+    - name: Upload test reports
+      if: always()
+      uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+      with:
+        name: TestReports
+        path: ${{ github.workspace }}/Ed-Fi-ODS/reports/*.*
diff --git a/Application/EdFi.Ods.Api/Container/Modules/ApplicationModule.cs b/Application/EdFi.Ods.Api/Container/Modules/ApplicationModule.cs
@@ -292,11 +292,6 @@ protected override void Load(ContainerBuilder builder)
                 .As<IResourceValidator>()
                 .SingleInstance();
 
-            builder.RegisterType<NoEntityExtensionsFactory>()
-                .As<IEntityExtensionsFactory>()
-                .PreserveExistingDefaults()
-                .SingleInstance();
-
             builder.RegisterType<MappingContractProvider>()
                 .As<IMappingContractProvider>()
                 .SingleInstance();

diff --git a/Application/EdFi.Ods.Api/Container/Modules/ExtensionsModule.cs b/Application/EdFi.Ods.Api/Container/Modules/ExtensionsModule.cs
@@ -17,6 +17,7 @@
 using EdFi.Ods.Common.Infrastructure.Extensibility;
 using EdFi.Ods.Common.Models;
 using EdFi.Ods.Api.Security.AuthorizationStrategies.Relationships;
+using EdFi.Ods.Common.Dependencies;
 
 namespace EdFi.Ods.Api.Container.Modules
 {
@@ -47,6 +48,13 @@ public override void ApplyConfigurationSpecificRegistrations(ContainerBuilder bu
                 .As<IEntityExtensionRegistrar>()
                 .SingleInstance();
 
+            // Set feature-specific static resolvers
+            builder.RegisterBuildCallback(container =>
+            {
+                GeneratedArtifactStaticDependencies.Resolvers.Set(container.Resolve<IEntityExtensionsFactory>);
+                GeneratedArtifactStaticDependencies.Resolvers.Set(container.Resolve<IEntityExtensionRegistrar>);
+            });
+
             installedExtensionAssemblies.ForEach(
                 assembly =>
                 {

diff --git a/Application/EdFi.Ods.Api/Container/Modules/PersistenceModule.cs b/Application/EdFi.Ods.Api/Container/Modules/PersistenceModule.cs
@@ -32,6 +32,7 @@
 using EdFi.Security.DataAccess.Providers;
 using Microsoft.Extensions.Caching.Memory;
 using NHibernate;
+using NHibernate.Engine;
 using IInterceptor = Castle.DynamicProxy.IInterceptor;
 using Module = Autofac.Module;
 
@@ -175,6 +176,19 @@ protected override void Load(ContainerBuilder builder)
 
             RegisterPersonIdentifierCaching(builder);
 
+            // Register surrogate key mutators (for deserialization support)
+            builder.RegisterType<EntityDeserializer>()
+                .As<IEntityDeserializer>()
+                .SingleInstance();
+
+            builder.RegisterType<PersonSurrogateIdMutator>()
+                .As<ISurrogateIdMutator>()
+                .SingleInstance();
+
+            builder.RegisterType<DescriptorSurrogateIdMutator>()
+                .As<ISurrogateIdMutator>()
+                .SingleInstance();
+
             builder.RegisterType<OrmMappingFileDataProvider>()
                 .WithParameter(new NamedParameter("assemblyName", OrmMappingFileConventions.OrmMappingAssembly))
                 .As<IOrmMappingFileDataProvider>()
@@ -197,6 +211,11 @@ protected override void Load(ContainerBuilder builder)
                 .As<ISessionFactory>()
                 .SingleInstance();
 
+            builder.Register(
+                    c => (ISessionFactoryImplementor) c.Resolve<ISessionFactory>())
+                .As<ISessionFactoryImplementor>()
+                .SingleInstance();
+
             // ----------------------------------------------------------------------------------------------------
             // NOTE: Sometimes ISessionFactory cannot be injected, so we're injecting a Func<IStatelessSession> rather
             // than the ISessionFactory or IStatelessSession (the latter of which can result in a memory leak).

diff --git a/Application/EdFi.Ods.Api/Controllers/DataManagementControllerBase.cs b/Application/EdFi.Ods.Api/Controllers/DataManagementControllerBase.cs
@@ -118,7 +118,7 @@ protected ILog Logger
 
         private IActionResult CreateActionResultFromException(Exception exception)
         {
-            HttpContext.Items.Add("Exception", exception);
+            HttpContext.Items.TryAdd("Exception", exception);
 
             // Process translations to Problem Details
             var problemDetails = _problemDetailsProvider.GetProblemDetails(exception);

diff --git a/Application/EdFi.Ods.Api/Infrastructure/Pipelines/GetMany/GetManyContext.cs b/Application/EdFi.Ods.Api/Infrastructure/Pipelines/GetMany/GetManyContext.cs
@@ -5,6 +5,7 @@
 
 using System.Collections.Generic;
 using EdFi.Ods.Api.Infrastructure.Pipelines;
+using EdFi.Ods.Common.Repositories;
 
 namespace EdFi.Ods.Common.Infrastructure.Pipelines.GetMany
 {

diff --git a/Application/EdFi.Ods.Api/Infrastructure/Pipelines/Steps/GetModelsByIds.cs b/Application/EdFi.Ods.Api/Infrastructure/Pipelines/Steps/GetModelsByIds.cs
@@ -4,6 +4,7 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using EdFi.Ods.Common;

diff --git a/Application/EdFi.Ods.Api/Startup/OdsStartupBase.cs b/Application/EdFi.Ods.Api/Startup/OdsStartupBase.cs
@@ -57,6 +57,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
+using NHibernate.Engine;
 
 namespace EdFi.Ods.Api.Startup
 {
@@ -439,6 +440,7 @@ void SetStaticResolvers()
                 GeneratedArtifactStaticDependencies.Resolvers.Set(() => (StringComparer) Container.Resolve<IDatabaseEngineSpecificEqualityComparerProvider<string>>().GetEqualityComparer());
                 GeneratedArtifactStaticDependencies.Resolvers.Set(() => Container.Resolve<IDescriptorResolver>());
                 GeneratedArtifactStaticDependencies.Resolvers.Set(() => Container.Resolve<IContextProvider<DataPolicyException>>());
+                GeneratedArtifactStaticDependencies.Resolvers.Set(() => Container.Resolve<ISessionFactoryImplementor>());
 
                 // netcore has removed the claims principal from the thread, to be on the controller.
                 // as a workaround for our current application we can resolve the IHttpContextAccessor.
@@ -451,8 +453,6 @@ void SetStaticResolvers()
                         () => Container.Resolve<IResourceModelProvider>()
                             .GetResourceModel());
 
-                EntityExtensionsFactory.Instance = Container.Resolve<IEntityExtensionsFactory>();
-
                 // Set NHibernate to use Autofac to resolve its dependencies
                 NHibernate.Cfg.Environment.ObjectsFactory = new NHibernateAutofacObjectsFactory(Container);
             }

diff --git a/Application/EdFi.Ods.Common/Constants/ApiFeature.cs b/Application/EdFi.Ods.Common/Constants/ApiFeature.cs
@@ -22,6 +22,7 @@ public class ApiFeature : Enumeration<ApiFeature, string>
         public static readonly ApiFeature XsdMetadata = new ApiFeature("xsdMetadata", "Xsd Metadata");
         public static readonly ApiFeature MultiTenancy = new ApiFeature("multiTenancy", "Multi-Tenancy");
         public static readonly ApiFeature Notifications = new ApiFeature("notifications", "Notifications");
+        public static readonly ApiFeature SerializedData = new ApiFeature("serializedData", "SerializedData");
 
         public ApiFeature(string value, string displayName)
             : base(value, displayName) { }

diff --git a/Application/EdFi.Ods.Common/Container/ConditionalModule.cs b/Application/EdFi.Ods.Common/Container/ConditionalModule.cs
@@ -42,12 +42,12 @@ protected override void Load(ContainerBuilder builder)
         {
             if (IsSelected())
             {
+                _logger.Info($"Loading registrations from conditional module '{_moduleName}'...");
                 ApplyConfigurationSpecificRegistrations(builder);
             }
             else
             {
                 ApplyFeatureDisabledRegistrations(builder);
-
                 _logger.Debug($"{_moduleName} Module is disabled and will not be installed.");
             }
         }

diff --git a/Application/EdFi.Ods.Common/Database/ColumnNames.cs b/Application/EdFi.Ods.Common/Database/ColumnNames.cs
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+
+namespace EdFi.Ods.Common.Database;
+
+public static class ColumnNames
+{
+    public const string AggregateId = "AggregateId";
+    public const string AggregateData = "AggregateData";
+    public const string CreateDate = "CreateDate";
+    public const string LastModifiedDate = "LastModifiedDate";
+    public const string ChangeVersion = "ChangeVersion";
+}
diff --git a/Application/EdFi.Ods.Common/Dependencies/GeneratedArtifactStaticDependencies.cs b/Application/EdFi.Ods.Common/Dependencies/GeneratedArtifactStaticDependencies.cs
@@ -8,9 +8,11 @@
 using EdFi.Ods.Common.Context;
 using EdFi.Ods.Common.Descriptors;
 using EdFi.Ods.Common.Exceptions;
+using EdFi.Ods.Common.Infrastructure.Extensibility;
 using EdFi.Ods.Common.Models;
 using EdFi.Ods.Common.Profiles;
 using EdFi.Ods.Common.Security.Claims;
+using NHibernate.Engine;
 
 namespace EdFi.Ods.Common.Dependencies
 {
@@ -31,6 +33,9 @@ public static class GeneratedArtifactStaticDependencies
         private static Lazy<IContextProvider<DataPolicyException>> _dataPolicyExceptionContextProvider;
         private static Lazy<StringComparer> _databaseEngineSpecificStringComparer;
         private static Lazy<IDescriptorResolver> _descriptorResolver;
+        private static Lazy<IEntityExtensionRegistrar> _entityExtensionRegistrar;
+        private static Lazy<IEntityExtensionsFactory> _entityExtensionsFactory;
+        private static Lazy<ISessionFactoryImplementor> _sessionFactory;
 
         public static IAuthorizationContextProvider AuthorizationContextProvider => _authorizationContextProvider?.Value;
         public static IResourceModelProvider ResourceModelProvider => _resourceModelProvider?.Value;
@@ -44,6 +49,9 @@ public static class GeneratedArtifactStaticDependencies
         public static IContextProvider<DataPolicyException> DataPolicyExceptionContextProvider => _dataPolicyExceptionContextProvider?.Value;
         public static StringComparer DatabaseEngineSpecificStringComparer => _databaseEngineSpecificStringComparer?.Value;
         public static IDescriptorResolver DescriptorResolver => _descriptorResolver?.Value;
+        public static IEntityExtensionRegistrar EntityExtensionRegistrar => _entityExtensionRegistrar?.Value;
+        public static IEntityExtensionsFactory EntityExtensionsFactory => _entityExtensionsFactory?.Value;
+        public static ISessionFactoryImplementor SessionFactory => _sessionFactory?.Value;
 
         /// <summary>
         /// Provides a mechanism for providing resolution of container managed components (intended for use only
@@ -110,6 +118,21 @@ public static void Set(Func<IDescriptorResolver> resolver)
             {
                 _descriptorResolver = new Lazy<IDescriptorResolver>(resolver);
             }
+
+            public static void Set(Func<IEntityExtensionRegistrar> resolver)
+            {
+                _entityExtensionRegistrar = new Lazy<IEntityExtensionRegistrar>(resolver);
+            }
+
+            public static void Set(Func<IEntityExtensionsFactory> resolver)
+            {
+                _entityExtensionsFactory = new Lazy<IEntityExtensionsFactory>(resolver);
+            }
+
+            public static void Set(Func<ISessionFactoryImplementor> resolver)
+            {
+                _sessionFactory = new Lazy<ISessionFactoryImplementor>(resolver);
+            }
         }
     }
 }
diff --git a/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj b/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj
@@ -33,10 +33,12 @@
     <PackageReference Include="FluentValidation" Version="11.9.2" />
     <PackageReference Include="log4net" Version="2.0.17" />
     <PackageReference Include="MediatR" Version="12.3.0" />
+    <PackageReference Include="MessagePack" Version="2.5.187" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
     <PackageReference Include="NHibernate" Version="5.5.2" />
     <PackageReference Include="Npgsql" Version="8.0.3" />
     <PackageReference Include="Polly" Version="8.4.1" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="Sandwych.QuickGraph.Core" Version="1.0.0" />
     <PackageReference Include="Standart.Hash.xxHash" Version="4.0.5" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />

diff --git a/Application/EdFi.Ods.Common/Extensions/CollectionExtensions.cs b/Application/EdFi.Ods.Common/Extensions/CollectionExtensions.cs
@@ -9,6 +9,7 @@
 using System.Linq;
 using EdFi.Ods.Common.Dependencies;
 using EdFi.Ods.Common.Exceptions;
+using EdFi.Ods.Common.Repositories;
 
 namespace EdFi.Ods.Common.Extensions
 {
@@ -143,20 +144,20 @@ Type GetItemType()
                 {
                     return type;
                 }
-
+        
                 var listTypes = targetListType.GetGenericArguments();
-
+        
                 if (listTypes.Length == 0)
                 {
                     throw new ArgumentException(
                         $"Target list type of '{targetListType.FullName}' does not have any generic arguments.");
                 }
-
+        
                 // Assumption: ItemType is last generic argument (most of the time this will be a List<T>,
                 // but it could be a CovariantIListAdapter<TBase, TDerived>.  We want the last generic argument type.
                  type = listTypes[^1];
                 _itemTypeByUnderlyingListType[targetListType] = type;
-
+        
                 return type;
             }
         }

diff --git a/Application/EdFi.Ods.Common/IHasAlternateKeyValues.cs b/Application/EdFi.Ods.Common/IHasAlternateKeyValues.cs
@@ -15,7 +15,7 @@ public interface IHasAlternateKeyValues
         /// <summary>
         /// Gets the alternate key values (unique key values which are not the primary key), in the form of an <see cref="OrderedDictionary"/>.
         /// </summary>
-        /// <returns>The dictionary containing the name/value pairs of the alternate key fields.</returns>
-        OrderedDictionary GetAlternateKeyValues();
+        /// <returns>A tuple containing the dictionary of name/value pairs of the alternate key fields, and an indicator as to whether the alternate key is defined on a base type in the model.</returns>
+        (OrderedDictionary keyValues, bool isDefinedOnBaseType) GetAlternateKeyValues();
     }
 }
diff --git a/Application/EdFi.Ods.Common/IPersonUsiMutator.cs b/Application/EdFi.Ods.Common/IPersonUsiMutator.cs
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+namespace EdFi.Ods.Common;
+
+/// <summary>
+/// Defines methods for reading and writing USI values on Person entities.
+/// </summary>
+public interface IPersonUsiMutator : IIdentifiablePerson
+{
+    void SetUsi(int newValue);
+
+    int GetUsi();
+}
diff --git a/...ication/EdFi.Ods.Common/Infrastructure/Configuration/NHibernateConfigurationExtensions.cs b/...ication/EdFi.Ods.Common/Infrastructure/Configuration/NHibernateConfigurationExtensions.cs
@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using EdFi.Ods.Common.Configuration;
 using EdFi.Ods.Common.Infrastructure.Interceptors;
 using EdFi.Ods.Common.Infrastructure.Listeners;
 using EdFi.Ods.Common.Security.Authorization;
@@ -15,18 +16,20 @@ namespace EdFi.Ods.Common.Infrastructure.Configuration
 {
     public static class NHibernateConfigurationExtensions
     {
-        public static void AddCreateDateHooks(
+        public static void SetInterceptorAndEventListeners(
             this NHibernate.Cfg.Configuration configuration,
             Func<IEntityAuthorizer> entityAuthorizerResolver,
-            IAuthorizationContextProvider authorizationContextProvider)
+            IAuthorizationContextProvider authorizationContextProvider,
+            ApiSettings apiSettings)
         {
             configuration.Interceptor = new EdFiOdsInterceptor();
-            configuration.SetListener(ListenerType.PreInsert, new EdFiOdsPreInsertListener());
+            configuration.SetListener(ListenerType.PreInsert, new EdFiOdsPreInsertListener(apiSettings));
             configuration.SetListener(ListenerType.PostInsert, new EdFiOdsPostInsertListener());
+            configuration.SetListener(ListenerType.PreUpdate, new EdFiOdsPreUpdateListener(apiSettings));
 
             configuration.SetListener(
                 ListenerType.PostUpdate,
-                new EdFiOdsPostUpdateEventListener(entityAuthorizerResolver, authorizationContextProvider));
+                new EdFiOdsPostUpdateEventListener(entityAuthorizerResolver, authorizationContextProvider, apiSettings));
         }
     }
 }
diff --git a/Application/EdFi.Ods.Common/Infrastructure/Configuration/NHibernateConfigurator.cs b/Application/EdFi.Ods.Common/Infrastructure/Configuration/NHibernateConfigurator.cs
@@ -11,8 +11,8 @@
 using EdFi.Common;
 using EdFi.Common.Extensions;
 using EdFi.Common.Utils.Extensions;
+using EdFi.Ods.Common.Configuration;
 using EdFi.Ods.Common.Infrastructure.Extensibility;
-using EdFi.Ods.Common.Infrastructure.Filtering;
 using EdFi.Ods.Common.Providers;
 using EdFi.Ods.Common.Security.Authorization;
 using EdFi.Ods.Common.Security.Claims;
@@ -36,16 +36,19 @@ public class NHibernateConfigurator : INHibernateConfigurator
         private readonly IOrmMappingFileDataProvider _ormMappingFileDataProvider;
         private readonly Func<IEntityAuthorizer> _entityAuthorizerResolver;
         private readonly IAuthorizationContextProvider _authorizationContextProvider;
+        private readonly ApiSettings _apiSettings;
 
         public NHibernateConfigurator(IEnumerable<IExtensionNHibernateConfigurationProvider> extensionConfigurationProviders,
             IEnumerable<INHibernateBeforeBindMappingActivity> beforeBindMappingActivities,
             IEnumerable<INHibernateConfigurationActivity> configurationActivities,
             IOrmMappingFileDataProvider ormMappingFileDataProvider,
             Func<IEntityAuthorizer> entityAuthorizerResolver,
-            IAuthorizationContextProvider authorizationContextProvider)
+            IAuthorizationContextProvider authorizationContextProvider,
+            ApiSettings apiSettings)
         {
             _entityAuthorizerResolver = entityAuthorizerResolver;
             _authorizationContextProvider = authorizationContextProvider;
+            _apiSettings = apiSettings;
 
             _ormMappingFileDataProvider = Preconditions.ThrowIfNull(
                 ormMappingFileDataProvider, nameof(ormMappingFileDataProvider));
@@ -110,7 +113,7 @@ public NHibernate.Cfg.Configuration Configure()
                 configurationActivity.Execute(configuration);
             }
 
-            configuration.AddCreateDateHooks(_entityAuthorizerResolver, _authorizationContextProvider);
+            configuration.SetInterceptorAndEventListeners(_entityAuthorizerResolver, _authorizationContextProvider, _apiSettings);
 
             return configuration;