entity metadata aggregate removal. (#789)

* entity metadata aggregate removal.

CHANGELOG.md

@@ -1,12 +1,21 @@
 ## 1.8.0
 Release built: _not released yet_
 
+> [!CAUTION]
+> **Breaking Changes:**
+> - Changed ordering of entity metadata returned from `/state/entity/metadata`, `/state/entity/page/metadata` endpoints. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending.
+
+
 ### Bug fixes
 
 ### API Changes
 
 ### Database changes
-
+- Refactored entity metadata aggregate. Queries for metadata follow a similar strategy as key value stores and utilize `_entry_definition`, `_entry_history`, and `_totals_history` tables to return data
+  - Removed `entity_metadata_aggregate_history` table.
+  - New `entity_metadata_totals_history` table which holds total counts of metadata per entity.
+  - New `entity_metadata_entry_definition` table which holds information about all the metadata keys ever created for a given entity.
+  - Renamed `entity_metadata_history` to `entity_metadata_entry_history`, replaced `entity_id` and `key` columns with `entity_metadata_entry_definition_id`.
 ## 1.7.0
 Release built: _not released yet_
 

docs/configuration.md

@@ -30,6 +30,7 @@ The Network Gateway services can be configured in line with the [configuration i
 - `GatewayApi__Endpoint__ValidatorsUptimePageSize` (type: `int`, default value: `200`) -  fixed page size for `/statistics/validators/uptime` endpoint.
 
 - `GatewayApi__Endpoint__RequestTimeout` - (type: `timespan in string format [d.]hh:mm:ss[.fffffff]`, default value: `10s`) - the amount of time after which request gets canceled and timeout is returned from API.
+- `GatewayApi__Endpoint__MaxDefinitionsLookupLimit` - (type: `int`, default value: `50 000`) - Max number of definitions to scan when searching for non-deleted entries.
 
 #### Ledger Lag
 - `GatewayApi__AcceptableLedgerLag__PreventReadRequestsIfDbLedgerIsBehind` (type: `bool`, default value: `true`) - controls if API will return a response if observed ledger state by the gateway is behind.

src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs

@@ -110,6 +110,9 @@ public sealed class EndpointOptions
     [ConfigurationKeyName("ResourceHoldersMaxPageSize")]
     public int ResourceHoldersMaxPageSize { get; set; } = 1000;
 
+    [ConfigurationKeyName("MaxDefinitionsLookupLimit")]
+    public int MaxDefinitionsLookupLimit { get; set; } = 50_000;
+
     [ConfigurationKeyName("TransactionStreamMaxFilterCount")]
     public int TransactionStreamMaxFilterCount { get; set; } = 10;
 

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs

@@ -103,9 +103,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        var pageRequest = new IEntityStateQuerier.PageRequest(
+        var pageRequest = new IEntityStateQuerier.MetadataPageRequest(
             Address: (EntityAddress)request.Address,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: !string.IsNullOrEmpty(request.Cursor) ? GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor) : null,
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 

src/RadixDlt.NetworkGateway.GatewayApi/Services/IEntityStateQuerier.cs

@@ -74,6 +74,8 @@ public interface IEntityStateQuerier
 {
     public sealed record PageRequest(EntityAddress Address, int Offset, int Limit);
 
+    public sealed record MetadataPageRequest(EntityAddress Address, GatewayModel.IdBoundaryCoursor? Cursor, int Limit);
+
     public sealed record AccountLockerPageRequest(AccountLockerAddress AccountLockerAddress, GatewayModel.StateAccountLockerAccountResourcesCursor? Cursor, int Limit);
 
     public sealed record ResourceVaultsPageRequest(EntityAddress Address, EntityAddress ResourceAddress, int Offset, int Limit);
@@ -86,7 +88,7 @@ public sealed record ResourceVaultsPageRequest(EntityAddress Address, EntityAddr
         CancellationToken token = default);
 
     Task<GatewayModel.StateEntityMetadataPageResponse> EntityMetadata(
-        PageRequest request,
+        MetadataPageRequest request,
         GatewayModel.LedgerState ledgerState,
         CancellationToken token = default);
 

src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs

@@ -96,9 +96,11 @@ internal abstract class CommonDbContext : DbContext
 
     public DbSet<Entity> Entities => Set<Entity>();
 
-    public DbSet<EntityMetadataHistory> EntityMetadataHistory => Set<EntityMetadataHistory>();
+    public DbSet<EntityMetadataEntryDefinition> EntityMetadataEntryDefinition => Set<EntityMetadataEntryDefinition>();
 
-    public DbSet<EntityMetadataAggregateHistory> EntityMetadataAggregateHistory => Set<EntityMetadataAggregateHistory>();
+    public DbSet<EntityMetadataEntryHistory> EntityMetadataEntryHistory => Set<EntityMetadataEntryHistory>();
+
+    public DbSet<EntityMetadataTotalsHistory> EntityMetadataTotalHistory => Set<EntityMetadataTotalsHistory>();
 
     public DbSet<PackageBlueprintAggregateHistory> PackageBlueprintAggregateHistory => Set<PackageBlueprintAggregateHistory>();
 
@@ -430,11 +432,19 @@ private static void HookupHistory(ModelBuilder modelBuilder)
             .HasIndex(e => new { e.AccountEntityId, e.FromStateVersion });
 
         modelBuilder
-            .Entity<EntityMetadataHistory>()
-            .HasIndex(e => new { e.EntityId, e.Key, e.FromStateVersion });
+            .Entity<EntityMetadataEntryHistory>()
+            .HasIndex(e => new { e.EntityMetadataEntryDefinitionId, e.FromStateVersion });
+
+        modelBuilder
+            .Entity<EntityMetadataEntryDefinition>()
+            .HasIndex(e => new { e.EntityId, e.FromStateVersion });
+
+        modelBuilder
+            .Entity<EntityMetadataEntryDefinition>()
+            .HasIndex(e => new { e.EntityId, e.Key });
 
         modelBuilder
-            .Entity<EntityMetadataAggregateHistory>()
+            .Entity<EntityMetadataTotalsHistory>()
             .HasIndex(e => new { e.EntityId, e.FromStateVersion });
 
         modelBuilder

src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypes.cs

@@ -88,6 +88,7 @@ public static void EnsureConfigured()
 
         // needed to read int[], bigint[] and text[] columns using Dapper
         SqlMapper.AddTypeHandler(new EntityAddressHandler());
+        SqlMapper.AddTypeHandler(new IdBoundaryCursorHandler());
         SqlMapper.AddTypeHandler(new GenericArrayHandler<int>());
         SqlMapper.AddTypeHandler(new GenericArrayHandler<long>());
         SqlMapper.AddTypeHandler(new GenericArrayHandler<string>());