diff --git a/CHANGELOG.md b/CHANGELOG.md index 68957e20c..02d480f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,21 @@ ## 1.8.0 Release built: _not released yet_ +> [!CAUTION] +> **Breaking Changes:** +> - Changed ordering of entity metadata returned from `/state/entity/metadata`, `/state/entity/page/metadata` endpoints. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending. + + ### Bug fixes ### API Changes ### Database changes - +- Refactored entity metadata aggregate. Queries for metadata follow a similar strategy as key value stores and utilize `_entry_definition`, `_entry_history`, and `_totals_history` tables to return data + - Removed `entity_metadata_aggregate_history` table. + - New `entity_metadata_totals_history` table which holds total counts of metadata per entity. + - New `entity_metadata_entry_definition` table which holds information about all the metadata keys ever created for a given entity. + - Renamed `entity_metadata_history` to `entity_metadata_entry_history`, replaced `entity_id` and `key` columns with `entity_metadata_entry_definition_id`. ## 1.7.0 Release built: _not released yet_ diff --git a/docs/configuration.md b/docs/configuration.md index 3f17d436d..1d6a12dae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,6 +30,7 @@ The Network Gateway services can be configured in line with the [configuration i - `GatewayApi__Endpoint__ValidatorsUptimePageSize` (type: `int`, default value: `200`) - fixed page size for `/statistics/validators/uptime` endpoint. - `GatewayApi__Endpoint__RequestTimeout` - (type: `timespan in string format [d.]hh:mm:ss[.fffffff]`, default value: `10s`) - the amount of time after which request gets canceled and timeout is returned from API. +- `GatewayApi__Endpoint__MaxDefinitionsLookupLimit` - (type: `int`, default value: `50 000`) - Max number of definitions to scan when searching for non-deleted entries. #### Ledger Lag - `GatewayApi__AcceptableLedgerLag__PreventReadRequestsIfDbLedgerIsBehind` (type: `bool`, default value: `true`) - controls if API will return a response if observed ledger state by the gateway is behind. diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs index e39e651b6..39fa352d5 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs @@ -110,6 +110,9 @@ public sealed class EndpointOptions [ConfigurationKeyName("ResourceHoldersMaxPageSize")] public int ResourceHoldersMaxPageSize { get; set; } = 1000; + [ConfigurationKeyName("MaxDefinitionsLookupLimit")] + public int MaxDefinitionsLookupLimit { get; set; } = 50_000; + [ConfigurationKeyName("TransactionStreamMaxFilterCount")] public int TransactionStreamMaxFilterCount { get; set; } = 10; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs index 0f70e4428..508166ff1 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs @@ -103,9 +103,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState { var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token); - var pageRequest = new IEntityStateQuerier.PageRequest( + var pageRequest = new IEntityStateQuerier.MetadataPageRequest( Address: (EntityAddress)request.Address, - Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0, + Cursor: !string.IsNullOrEmpty(request.Cursor) ? GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor) : null, Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage) ); diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Services/IEntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Services/IEntityStateQuerier.cs index ec5eef604..a017f9e76 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Services/IEntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Services/IEntityStateQuerier.cs @@ -74,6 +74,8 @@ public interface IEntityStateQuerier { public sealed record PageRequest(EntityAddress Address, int Offset, int Limit); + public sealed record MetadataPageRequest(EntityAddress Address, GatewayModel.IdBoundaryCoursor? Cursor, int Limit); + public sealed record AccountLockerPageRequest(AccountLockerAddress AccountLockerAddress, GatewayModel.StateAccountLockerAccountResourcesCursor? Cursor, int Limit); public sealed record ResourceVaultsPageRequest(EntityAddress Address, EntityAddress ResourceAddress, int Offset, int Limit); @@ -86,7 +88,7 @@ public sealed record ResourceVaultsPageRequest(EntityAddress Address, EntityAddr CancellationToken token = default); Task EntityMetadata( - PageRequest request, + MetadataPageRequest request, GatewayModel.LedgerState ledgerState, CancellationToken token = default); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs index e87d39462..de7e9c519 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs @@ -96,9 +96,11 @@ internal abstract class CommonDbContext : DbContext public DbSet Entities => Set(); - public DbSet EntityMetadataHistory => Set(); + public DbSet EntityMetadataEntryDefinition => Set(); - public DbSet EntityMetadataAggregateHistory => Set(); + public DbSet EntityMetadataEntryHistory => Set(); + + public DbSet EntityMetadataTotalHistory => Set(); public DbSet PackageBlueprintAggregateHistory => Set(); @@ -430,11 +432,19 @@ private static void HookupHistory(ModelBuilder modelBuilder) .HasIndex(e => new { e.AccountEntityId, e.FromStateVersion }); modelBuilder - .Entity() - .HasIndex(e => new { e.EntityId, e.Key, e.FromStateVersion }); + .Entity() + .HasIndex(e => new { e.EntityMetadataEntryDefinitionId, e.FromStateVersion }); + + modelBuilder + .Entity() + .HasIndex(e => new { e.EntityId, e.FromStateVersion }); + + modelBuilder + .Entity() + .HasIndex(e => new { e.EntityId, e.Key }); modelBuilder - .Entity() + .Entity() .HasIndex(e => new { e.EntityId, e.FromStateVersion }); modelBuilder diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypes.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypes.cs index 3ff8a5257..3e9183aeb 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypes.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypes.cs @@ -88,6 +88,7 @@ public static void EnsureConfigured() // needed to read int[], bigint[] and text[] columns using Dapper SqlMapper.AddTypeHandler(new EntityAddressHandler()); + SqlMapper.AddTypeHandler(new IdBoundaryCursorHandler()); SqlMapper.AddTypeHandler(new GenericArrayHandler()); SqlMapper.AddTypeHandler(new GenericArrayHandler()); SqlMapper.AddTypeHandler(new GenericArrayHandler()); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs index 9139a00bb..61660eefc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs @@ -77,201 +77,220 @@ internal record struct MetadataEntryDbLookup(long EntityId, string Key); internal record struct MetadataChangePointerLookup(long EntityId, long StateVersion); +internal record struct MetadataEntry(CoreModel.MetadataModuleEntrySubstate NewValue, CoreModel.MetadataModuleEntrySubstate? PreviousValue); + internal record MetadataChangePointer { - public List Entries { get; } = new(); + public List Entries { get; } = new(); } internal class EntityMetadataProcessor { private readonly ProcessorContext _context; - private ChangeTracker _changes = new(); + private readonly Dictionary _observedEntryDefinitions = new(); + private readonly Dictionary _existingEntryDefinitions = new(); + private readonly Dictionary _existingTotalsHistory = new(); - private Dictionary _mostRecentAggregates = new(); - private Dictionary _mostRecentEntries = new(); + private readonly Dictionary _entryDefinitionsToAdd = new(); + private readonly List _entryHistoryToAdd = new(); + private readonly List _totalsHistoryToAdd = new(); + private readonly Dictionary _mostRecentHistoryEntry = new(); - private List _aggregatesToAdd = new(); - private List _entriesToAdd = new(); + private readonly ChangeTracker _changes = new(); public EntityMetadataProcessor(ProcessorContext context) { _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.MetadataModuleEntrySubstate metadataEntry) { - _changes - .GetOrAdd(new MetadataChangePointerLookup(referencedEntity.DatabaseId, stateVersion), _ => new MetadataChangePointer()) - .Entries.Add(metadataEntry); + var lookup = new MetadataChangePointerLookup(referencedEntity.DatabaseId, stateVersion); + + _changes.GetOrAdd(lookup, _ => new MetadataChangePointer()) + .Entries + .Add(new MetadataEntry(metadataEntry, substate.PreviousValue?.SubstateData as CoreModel.MetadataModuleEntrySubstate)); + + _observedEntryDefinitions.TryAdd(new MetadataEntryDbLookup(referencedEntity.DatabaseId, metadataEntry.Key.Name), stateVersion); } } public async Task LoadDependencies() { - _mostRecentEntries.AddRange(await MostRecentEntityMetadataHistory()); - _mostRecentAggregates.AddRange(await MostRecentEntityAggregateMetadataHistory()); + _existingEntryDefinitions.AddRange(await ExistingMetadataEntryDefinitions()); + _existingTotalsHistory.AddRange(await ExistingMetadataTotalsHistory()); } public void ProcessChanges() { - foreach (var (lookup, change) in _changes.AsEnumerable()) + foreach (var lookup in _observedEntryDefinitions.Keys.Except(_existingEntryDefinitions.Keys)) { - EntityMetadataAggregateHistory aggregate; - - if (!_mostRecentAggregates.TryGetValue(lookup.EntityId, out var previousAggregate) || previousAggregate.FromStateVersion != lookup.StateVersion) + var entryDefinition = new EntityMetadataEntryDefinition { - aggregate = new EntityMetadataAggregateHistory - { - Id = _context.Sequences.EntityMetadataAggregateHistorySequence++, - FromStateVersion = lookup.StateVersion, - EntityId = lookup.EntityId, - MetadataIds = new List(), - }; + Id = _context.Sequences.KeyValueStoreEntryDefinitionSequence++, + FromStateVersion = _observedEntryDefinitions[lookup], + EntityId = lookup.EntityId, + Key = lookup.Key, + }; + + _entryDefinitionsToAdd[lookup] = entryDefinition; + _existingEntryDefinitions[lookup] = entryDefinition; + } - if (previousAggregate != null) - { - aggregate.MetadataIds.AddRange(previousAggregate.MetadataIds); - } + foreach (var change in _changes.AsEnumerable()) + { + var totalsExists = _existingTotalsHistory.TryGetValue(change.Key.EntityId, out var previousTotals); - _aggregatesToAdd.Add(aggregate); - _mostRecentAggregates[lookup.EntityId] = aggregate; - } - else + var newTotals = new EntityMetadataTotalsHistory { - aggregate = previousAggregate; - } - - foreach (var entry in change.Entries) + Id = _context.Sequences.EntityMetadataTotalsHistorySequence++, + FromStateVersion = change.Key.StateVersion, + EntityId = change.Key.EntityId, + TotalEntriesExcludingDeleted = totalsExists ? previousTotals!.TotalEntriesExcludingDeleted : 0, + TotalEntriesIncludingDeleted = totalsExists ? previousTotals!.TotalEntriesIncludingDeleted : 0, + }; + + foreach (var entry in change.Value.Entries) { - var entryLookup = new MetadataEntryDbLookup(lookup.EntityId, entry.Key.Name); - var entryHistory = new EntityMetadataHistory - { - Id = _context.Sequences.EntityMetadataHistorySequence++, - FromStateVersion = lookup.StateVersion, - EntityId = lookup.EntityId, - Key = entry.Key.Name, - Value = entry.Value?.DataStruct.StructData.Hex.ConvertFromHex(), - IsDeleted = entry.Value == null, - IsLocked = entry.IsLocked, - }; - - _entriesToAdd.Add(entryHistory); - - if (_mostRecentEntries.TryGetValue(entryLookup, out var previousEntry)) - { - var currentPosition = aggregate.MetadataIds.IndexOf(previousEntry.Id); + var isDeleted = entry.NewValue.Value == null; + var newEntry = entry.NewValue; - if (currentPosition != -1) + _entryHistoryToAdd.Add( + new EntityMetadataEntryHistory { - aggregate.MetadataIds.RemoveAt(currentPosition); - } - } - - if (entry.Value != null) + Id = _context.Sequences.KeyValueStoreEntryHistorySequence++, + FromStateVersion = change.Key.StateVersion, + EntityMetadataEntryDefinitionId = _existingEntryDefinitions[new MetadataEntryDbLookup(change.Key.EntityId, newEntry.Key!.Name)].Id, + Value = isDeleted ? null : newEntry.Value!.DataStruct.StructData.Hex.ConvertFromHex(), + IsDeleted = isDeleted, + IsLocked = newEntry.IsLocked, + }); + + switch (entry) { - aggregate.MetadataIds.Insert(0, entryHistory.Id); + case { PreviousValue: null, NewValue.Value: null }: + newTotals.TotalEntriesIncludingDeleted++; + break; + case { PreviousValue: null, NewValue.Value: not null }: + newTotals.TotalEntriesIncludingDeleted++; + newTotals.TotalEntriesExcludingDeleted++; + break; + case { PreviousValue.Value: not null, NewValue.Value: null }: + newTotals.TotalEntriesExcludingDeleted--; + break; + case { PreviousValue.Value: null, NewValue.Value: not null }: + newTotals.TotalEntriesExcludingDeleted++; + break; } - _mostRecentEntries[entryLookup] = entryHistory; + _existingTotalsHistory[change.Key.EntityId] = newTotals; } + + _totalsHistoryToAdd.Add(newTotals); } } public async Task SaveEntities() { var rowsInserted = 0; - - rowsInserted += await CopyEntityMetadataHistory(); - rowsInserted += await CopyEntityMetadataAggregateHistory(); - + rowsInserted += await CopyEntityMetadataEntryHistory(); + rowsInserted += await CopyEntityMetadataEntryDefinition(); + rowsInserted += await CopyEntityMetadataTotalsHistory(); return rowsInserted; } - private async Task> MostRecentEntityMetadataHistory() + private async Task> ExistingMetadataEntryDefinitions() { - var lookupSet = new HashSet(); - - foreach (var (lookup, change) in _changes.AsEnumerable()) + if (!_observedEntryDefinitions + .Keys + .ToHashSet() + .Unzip( + x => x.EntityId, + x => x.Key, + out var entityIds, + out var keys)) { - foreach (var entry in change.Entries) - { - lookupSet.Add(new MetadataEntryDbLookup(lookup.EntityId, entry.Key.Name)); - } + return ImmutableDictionary.Empty; } - if (!lookupSet.Unzip(x => x.EntityId, x => x.Key, out var entityIds, out var keys)) - { - return ImmutableDictionary.Empty; - } - - return await _context.ReadHelper.LoadDependencies( + return await _context.ReadHelper.LoadDependencies( @$" WITH variables (entity_id, key) AS ( SELECT UNNEST({entityIds}), UNNEST({keys}) ) -SELECT emh.* -FROM variables -INNER JOIN LATERAL ( - SELECT * - FROM entity_metadata_history - WHERE entity_id = variables.entity_id AND key = variables.key - ORDER BY from_state_version DESC - LIMIT 1 -) emh ON true;", +SELECT * +FROM entity_metadata_entry_definition +WHERE (entity_id, key) IN (SELECT * FROM variables)", e => new MetadataEntryDbLookup(e.EntityId, e.Key)); } - private async Task> MostRecentEntityAggregateMetadataHistory() + private async Task> ExistingMetadataTotalsHistory() { - var entityIds = _changes.Keys.Select(x => x.EntityId).ToHashSet().ToList(); + var entityIds = _observedEntryDefinitions + .Keys + .Select(x => x.EntityId) + .ToHashSet(); - if (!entityIds.Any()) + if (entityIds.Count == 0) { - return ImmutableDictionary.Empty; + return ImmutableDictionary.Empty; } - return await _context.ReadHelper.LoadDependencies( + return await _context.ReadHelper.LoadDependencies( @$" WITH variables (entity_id) AS ( SELECT UNNEST({entityIds}) ) -SELECT emah.* +SELECT emth.* FROM variables INNER JOIN LATERAL ( SELECT * - FROM entity_metadata_aggregate_history + FROM entity_metadata_totals_history WHERE entity_id = variables.entity_id ORDER BY from_state_version DESC LIMIT 1 -) emah ON true;", +) emth ON true;", e => e.EntityId); } - private Task CopyEntityMetadataHistory() => _context.WriteHelper.Copy( - _entriesToAdd, - "COPY entity_metadata_history (id, from_state_version, entity_id, key, value, is_deleted, is_locked) FROM STDIN (FORMAT BINARY)", + private Task CopyEntityMetadataEntryDefinition() => _context.WriteHelper.Copy( + _entryDefinitionsToAdd.Values, + "COPY entity_metadata_entry_definition (id, from_state_version, entity_id, key) FROM STDIN (FORMAT BINARY)", async (writer, e, token) => { await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.EntityId, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.Key, NpgsqlDbType.Text, token); + }); + + private Task CopyEntityMetadataEntryHistory() => _context.WriteHelper.Copy( + _entryHistoryToAdd, + "COPY entity_metadata_entry_history (id, from_state_version, entity_metadata_entry_definition_id, value, is_deleted, is_locked) FROM STDIN (FORMAT BINARY)", + async (writer, e, token) => + { + await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.EntityMetadataEntryDefinitionId, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.Value, NpgsqlDbType.Bytea, token); await writer.WriteAsync(e.IsDeleted, NpgsqlDbType.Boolean, token); await writer.WriteAsync(e.IsLocked, NpgsqlDbType.Boolean, token); }); - private Task CopyEntityMetadataAggregateHistory() => _context.WriteHelper.Copy( - _aggregatesToAdd, - "COPY entity_metadata_aggregate_history (id, from_state_version, entity_id, metadata_ids) FROM STDIN (FORMAT BINARY)", + private Task CopyEntityMetadataTotalsHistory() => _context.WriteHelper.Copy( + _totalsHistoryToAdd, + "COPY entity_metadata_totals_history (id, from_state_version, entity_id, total_entries_including_deleted, total_entries_excluding_deleted) FROM STDIN (FORMAT BINARY)", async (writer, e, token) => { await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.EntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.MetadataIds.ToArray(), NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.TotalEntriesIncludingDeleted, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.TotalEntriesExcludingDeleted, NpgsqlDbType.Bigint, token); }); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index 844ae41d7..68c25559a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -906,7 +906,7 @@ private async Task ProcessTransactions(ReadWriteDbContext db } entityStateProcessor.VisitUpsert(substate, referencedEntity, stateVersion); - entityMetadataProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); + entityMetadataProcessor.VisitUpsert(substate, referencedEntity, stateVersion); entitySchemaProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); componentMethodRoyaltyProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); entityRoleAssignmentProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs index 670780ca5..7c1b31adc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs @@ -408,8 +408,9 @@ public async Task LoadSequences(CancellationToken token) nextval('account_resource_preference_rule_aggregate_history_id_seq') AS AccountResourcePreferenceRuleAggregateHistorySequence, nextval('state_history_id_seq') AS StateHistorySequence, nextval('entities_id_seq') AS EntitySequence, - nextval('entity_metadata_history_id_seq') AS EntityMetadataHistorySequence, - nextval('entity_metadata_aggregate_history_id_seq') AS EntityMetadataAggregateHistorySequence, + nextval('entity_metadata_entry_history_id_seq') AS EntityMetadataEntryHistorySequence, + nextval('entity_metadata_entry_definition_id_seq') AS EntityMetadataEntryDefinitionSequence, + nextval('entity_metadata_totals_history_id_seq') AS EntityMetadataTotalsHistorySequence, nextval('entity_resource_aggregated_vaults_history_id_seq') AS EntityResourceAggregatedVaultsHistorySequence, nextval('entity_resource_aggregate_history_id_seq') AS EntityResourceAggregateHistorySequence, nextval('entity_resource_vault_aggregate_history_id_seq') AS EntityResourceVaultAggregateHistorySequence, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/SequencesHolder.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/SequencesHolder.cs index cfdbc737c..27a2da10e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/SequencesHolder.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/SequencesHolder.cs @@ -86,9 +86,11 @@ internal class SequencesHolder public long EntitySequence { get; set; } - public long EntityMetadataHistorySequence { get; set; } + public long EntityMetadataEntryHistorySequence { get; set; } - public long EntityMetadataAggregateHistorySequence { get; set; } + public long EntityMetadataEntryDefinitionSequence { get; set; } + + public long EntityMetadataTotalsHistorySequence { get; set; } public long EntityResourceAggregatedVaultsHistorySequence { get; set; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs index b5014abf8..97d4b7b38 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs @@ -644,7 +644,9 @@ public async Task CopyNonFungibleIdLocationHistory(List +// using System; using System.Collections.Generic; using System.Numerics; @@ -81,7 +81,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Migrations { [DbContext(typeof(MigrationsDbContext))] - [Migration("20240819084018_InitialCreate")] + [Migration("20240830081152_InitialCreate")] partial class InitialCreate { /// @@ -497,7 +497,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.UseTphMappingStrategy(); }); - modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataAggregateHistory", b => + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataEntryDefinition", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -514,19 +514,21 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("bigint") .HasColumnName("from_state_version"); - b.Property>("MetadataIds") + b.Property("Key") .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("metadata_ids"); + .HasColumnType("text") + .HasColumnName("key"); b.HasKey("Id"); b.HasIndex("EntityId", "FromStateVersion"); - b.ToTable("entity_metadata_aggregate_history"); + b.HasIndex("EntityId", "Key"); + + b.ToTable("entity_metadata_entry_definition"); }); - modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataHistory", b => + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataEntryHistory", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -535,9 +537,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - b.Property("EntityId") + b.Property("EntityMetadataEntryDefinitionId") .HasColumnType("bigint") - .HasColumnName("entity_id"); + .HasColumnName("entity_metadata_entry_definition_id"); b.Property("FromStateVersion") .HasColumnType("bigint") @@ -551,20 +553,47 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("boolean") .HasColumnName("is_locked"); - b.Property("Key") - .IsRequired() - .HasColumnType("text") - .HasColumnName("key"); - b.Property("Value") .HasColumnType("bytea") .HasColumnName("value"); b.HasKey("Id"); - b.HasIndex("EntityId", "Key", "FromStateVersion"); + b.HasIndex("EntityMetadataEntryDefinitionId", "FromStateVersion"); + + b.ToTable("entity_metadata_entry_history"); + }); + + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataTotalsHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("EntityId") + .HasColumnType("bigint") + .HasColumnName("entity_id"); + + b.Property("FromStateVersion") + .HasColumnType("bigint") + .HasColumnName("from_state_version"); + + b.Property("TotalEntriesExcludingDeleted") + .HasColumnType("bigint") + .HasColumnName("total_entries_excluding_deleted"); + + b.Property("TotalEntriesIncludingDeleted") + .HasColumnType("bigint") + .HasColumnName("total_entries_including_deleted"); + + b.HasKey("Id"); + + b.HasIndex("EntityId", "FromStateVersion"); - b.ToTable("entity_metadata_history"); + b.ToTable("entity_metadata_totals_history"); }); modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityResourceAggregateHistory", b => diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240819084018_InitialCreate.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240830081152_InitialCreate.cs similarity index 97% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240819084018_InitialCreate.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240830081152_InitialCreate.cs index be8617b0c..59dbfeb31 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240819084018_InitialCreate.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240830081152_InitialCreate.cs @@ -62,7 +62,7 @@ * permissions under this License. */ -using System; +using System; using System.Collections.Generic; using System.Numerics; using Microsoft.EntityFrameworkCore.Migrations; @@ -292,36 +292,51 @@ protected override void Up(MigrationBuilder migrationBuilder) }); migrationBuilder.CreateTable( - name: "entity_metadata_aggregate_history", + name: "entity_metadata_entry_definition", columns: table => new { id = table.Column(type: "bigint", nullable: false) .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), from_state_version = table.Column(type: "bigint", nullable: false), entity_id = table.Column(type: "bigint", nullable: false), - metadata_ids = table.Column>(type: "bigint[]", nullable: false) + key = table.Column(type: "text", nullable: false) }, constraints: table => { - table.PrimaryKey("PK_entity_metadata_aggregate_history", x => x.id); + table.PrimaryKey("PK_entity_metadata_entry_definition", x => x.id); }); migrationBuilder.CreateTable( - name: "entity_metadata_history", + name: "entity_metadata_entry_history", columns: table => new { id = table.Column(type: "bigint", nullable: false) .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), from_state_version = table.Column(type: "bigint", nullable: false), - entity_id = table.Column(type: "bigint", nullable: false), - key = table.Column(type: "text", nullable: false), + entity_metadata_entry_definition_id = table.Column(type: "bigint", nullable: false), value = table.Column(type: "bytea", nullable: true), is_deleted = table.Column(type: "boolean", nullable: false), is_locked = table.Column(type: "boolean", nullable: false) }, constraints: table => { - table.PrimaryKey("PK_entity_metadata_history", x => x.id); + table.PrimaryKey("PK_entity_metadata_entry_history", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "entity_metadata_totals_history", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + from_state_version = table.Column(type: "bigint", nullable: false), + entity_id = table.Column(type: "bigint", nullable: false), + total_entries_including_deleted = table.Column(type: "bigint", nullable: false), + total_entries_excluding_deleted = table.Column(type: "bigint", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_entity_metadata_totals_history", x => x.id); }); migrationBuilder.CreateTable( @@ -1014,14 +1029,24 @@ protected override void Up(MigrationBuilder migrationBuilder) filter: "discriminator = 'global_validator'"); migrationBuilder.CreateIndex( - name: "IX_entity_metadata_aggregate_history_entity_id_from_state_vers~", - table: "entity_metadata_aggregate_history", + name: "IX_entity_metadata_entry_definition_entity_id_from_state_versi~", + table: "entity_metadata_entry_definition", columns: new[] { "entity_id", "from_state_version" }); migrationBuilder.CreateIndex( - name: "IX_entity_metadata_history_entity_id_key_from_state_version", - table: "entity_metadata_history", - columns: new[] { "entity_id", "key", "from_state_version" }); + name: "IX_entity_metadata_entry_definition_entity_id_key", + table: "entity_metadata_entry_definition", + columns: new[] { "entity_id", "key" }); + + migrationBuilder.CreateIndex( + name: "IX_entity_metadata_entry_history_entity_metadata_entry_definit~", + table: "entity_metadata_entry_history", + columns: new[] { "entity_metadata_entry_definition_id", "from_state_version" }); + + migrationBuilder.CreateIndex( + name: "IX_entity_metadata_totals_history_entity_id_from_state_version", + table: "entity_metadata_totals_history", + columns: new[] { "entity_id", "from_state_version" }); migrationBuilder.CreateIndex( name: "IX_entity_resource_aggregate_history_entity_id_from_state_vers~", @@ -1358,10 +1383,13 @@ protected override void Down(MigrationBuilder migrationBuilder) name: "entities"); migrationBuilder.DropTable( - name: "entity_metadata_aggregate_history"); + name: "entity_metadata_entry_definition"); + + migrationBuilder.DropTable( + name: "entity_metadata_entry_history"); migrationBuilder.DropTable( - name: "entity_metadata_history"); + name: "entity_metadata_totals_history"); migrationBuilder.DropTable( name: "entity_resource_aggregate_history"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql index 51e4c956f..f1b500bfd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql @@ -9,7 +9,7 @@ START TRANSACTION; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TYPE account_default_deposit_rule AS ENUM ('accept', 'reject', 'allow_existing'); CREATE TYPE account_resource_preference_rule AS ENUM ('allowed', 'disallowed'); CREATE TYPE authorized_depositor_badge_type AS ENUM ('resource', 'non_fungible'); @@ -37,7 +37,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_authorized_depositor_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -50,7 +50,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_authorized_depositor_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -66,7 +66,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_default_deposit_rule_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -79,7 +79,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_locker_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -93,7 +93,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_locker_entry_resource_vault_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -107,7 +107,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_locker_entry_touch_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -119,7 +119,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -132,7 +132,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -147,7 +147,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE component_method_royalty_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -160,7 +160,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE component_method_royalty_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -175,7 +175,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entities ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -201,36 +201,49 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN - CREATE TABLE entity_metadata_aggregate_history ( + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE TABLE entity_metadata_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, entity_id bigint NOT NULL, - metadata_ids bigint[] NOT NULL, - CONSTRAINT "PK_entity_metadata_aggregate_history" PRIMARY KEY (id) + key text NOT NULL, + CONSTRAINT "PK_entity_metadata_entry_definition" PRIMARY KEY (id) ); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN - CREATE TABLE entity_metadata_history ( + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE TABLE entity_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, - entity_id bigint NOT NULL, - key text NOT NULL, + entity_metadata_entry_definition_id bigint NOT NULL, value bytea, is_deleted boolean NOT NULL, is_locked boolean NOT NULL, - CONSTRAINT "PK_entity_metadata_history" PRIMARY KEY (id) + CONSTRAINT "PK_entity_metadata_entry_history" PRIMARY KEY (id) + ); + END IF; +END $EF$; + +DO $EF$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE TABLE entity_metadata_totals_history ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + from_state_version bigint NOT NULL, + entity_id bigint NOT NULL, + total_entries_including_deleted bigint NOT NULL, + total_entries_excluding_deleted bigint NOT NULL, + CONSTRAINT "PK_entity_metadata_totals_history" PRIMARY KEY (id) ); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_resource_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -246,7 +259,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_resource_aggregated_vaults_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -262,7 +275,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_resource_vault_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -276,7 +289,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_role_assignments_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -290,7 +303,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_role_assignments_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -306,7 +319,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_role_assignments_owner_role_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -319,7 +332,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE entity_vault_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -338,7 +351,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE key_value_store_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -351,7 +364,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE key_value_store_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -366,7 +379,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE key_value_store_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -386,7 +399,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE ledger_transaction_markers ( id bigint GENERATED BY DEFAULT AS IDENTITY, state_version bigint NOT NULL, @@ -406,7 +419,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE ledger_transactions ( state_version bigint NOT NULL, epoch bigint NOT NULL, @@ -454,7 +467,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE non_fungible_id_data_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -469,7 +482,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE non_fungible_id_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -482,7 +495,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE non_fungible_id_location_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -495,7 +508,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE non_fungible_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -511,7 +524,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE package_blueprint_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -524,7 +537,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE package_blueprint_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -544,7 +557,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE package_code_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -557,7 +570,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE package_code_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -573,7 +586,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE pending_transactions ( id bigint GENERATED BY DEFAULT AS IDENTITY, payload_hash text NOT NULL, @@ -601,7 +614,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE resource_entity_supply_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -616,7 +629,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE resource_holders ( id bigint GENERATED BY DEFAULT AS IDENTITY, entity_id bigint NOT NULL, @@ -630,7 +643,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE schema_entry_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -643,7 +656,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE schema_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -657,7 +670,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE state_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -676,7 +689,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -689,7 +702,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -706,7 +719,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE validator_cumulative_emission_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -722,7 +735,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE validator_public_key_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -736,7 +749,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE pending_transaction_payloads ( id bigint GENERATED BY DEFAULT AS IDENTITY, pending_transaction_id bigint, @@ -749,7 +762,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE TABLE validator_active_set_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -764,520 +777,534 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_aggregate_history_account_enti~" ON account_authorized_depositor_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~1" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, non_fungible_id, from_state_version) WHERE discriminator = 'non_fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~2" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, from_state_version) WHERE discriminator = 'resource'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_i~" ON account_authorized_depositor_entry_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_default_deposit_rule_history_account_entity_id_from~" ON account_default_deposit_rule_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_account_locker_entry_definition_account_locker_entity_id_ac~" ON account_locker_entry_definition (account_locker_entity_id, account_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_account_locker_entry_resource_vault_definition_account_lock~" ON account_locker_entry_resource_vault_definition (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_locker_entry_touch_history_account_locker_definitio~" ON account_locker_entry_touch_history (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_aggregate_history_account_~" ON account_resource_preference_rule_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_entry_history_account_enti~" ON account_resource_preference_rule_entry_history (account_entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_aggregate_history_entity_id_from_s~" ON component_method_royalty_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_from_state~" ON component_method_royalty_entry_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_method_nam~" ON component_method_royalty_entry_history (entity_id, method_name, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_entities_address" ON entities (address); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entities_from_state_version" ON entities (from_state_version) WHERE discriminator = 'global_validator'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN - CREATE INDEX "IX_entity_metadata_aggregate_history_entity_id_from_state_vers~" ON entity_metadata_aggregate_history (entity_id, from_state_version); + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_from_state_versi~" ON entity_metadata_entry_definition (entity_id, from_state_version); + END IF; +END $EF$; + +DO $EF$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_key" ON entity_metadata_entry_definition (entity_id, key); + END IF; +END $EF$; + +DO $EF$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE INDEX "IX_entity_metadata_entry_history_entity_metadata_entry_definit~" ON entity_metadata_entry_history (entity_metadata_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN - CREATE INDEX "IX_entity_metadata_history_entity_id_key_from_state_version" ON entity_metadata_history (entity_id, key, from_state_version); + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN + CREATE INDEX "IX_entity_metadata_totals_history_entity_id_from_state_version" ON entity_metadata_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_resource_aggregate_history_entity_id_from_state_vers~" ON entity_resource_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_resource_aggregated_vaults_history_entity_id_resourc~" ON entity_resource_aggregated_vaults_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_resource_vault_aggregate_history_entity_id_resource_~" ON entity_resource_vault_aggregate_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_aggregate_history_entity_id_from_st~" ON entity_role_assignments_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_entry_history_entity_id_key_role_ke~" ON entity_role_assignments_entry_history (entity_id, key_role, key_module, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_owner_role_history_entity_id_from_s~" ON entity_role_assignments_owner_role_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_global_entity_id_from_state_version" ON entity_vault_history (global_entity_id, from_state_version) WHERE is_royalty_vault = true; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_global_entity_id_vault_entity_id_from_~" ON entity_vault_history (global_entity_id, vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_id_resource_entity_id_from_state_versi~" ON entity_vault_history (id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_owner_entity_id_from_state_version" ON entity_vault_history (owner_entity_id, from_state_version) WHERE is_royalty_vault = true; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_owner_entity_id_vault_entity_id_from_s~" ON entity_vault_history (owner_entity_id, vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_entity_vault_history_vault_entity_id_from_state_version" ON entity_vault_history (vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id_~" ON key_value_store_entry_definition (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id~1" ON key_value_store_entry_definition (key_value_store_entity_id, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_history_key_value_store_entry_definit~" ON key_value_store_entry_history (key_value_store_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_key_value_store_schema_history_key_value_store_entity_id_fr~" ON key_value_store_schema_history (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'event_global_emitter'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version1" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'affected_global_entity'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_event_type_entity_id_state_versi~" ON ledger_transaction_markers (event_type, entity_id, state_version) WHERE discriminator = 'event'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class_is_most_specific" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class' and is_most_specific = true; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_operation_type_entity_id_state_v~" ON ledger_transaction_markers (operation_type, entity_id, state_version) WHERE discriminator = 'manifest_address'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_origin_type_state_version" ON ledger_transaction_markers (origin_type, state_version) WHERE discriminator = 'origin'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_state_version" ON ledger_transaction_markers (state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_ledger_transactions_epoch_round_in_epoch" ON ledger_transactions (epoch, round_in_epoch) WHERE index_in_round = 0; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_intent_hash" ON ledger_transactions USING hash (intent_hash) WHERE intent_hash IS NOT NULL; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_round_timestamp" ON ledger_transactions (round_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_data_history_non_fungible_id_definition_id_~" ON non_fungible_id_data_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id_~" ON non_fungible_id_definition (non_fungible_resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id~1" ON non_fungible_id_definition (non_fungible_resource_entity_id, non_fungible_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_location_history_non_fungible_id_definition~" ON non_fungible_id_location_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_non_fungible_schema_history_resource_entity_id_from_state_v~" ON non_fungible_schema_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_aggregate_history_package_entity_id_from_~" ON package_blueprint_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_from_state_vers~" ON package_blueprint_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_name_version_fr~" ON package_blueprint_history (package_entity_id, name, version, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_package_code_aggregate_history_package_entity_id_from_state~" ON package_code_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_package_code_history_package_entity_id_from_state_version" ON package_code_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transaction_payloads_pending_transaction_id" ON pending_transaction_payloads (pending_transaction_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_first_submitted_to_gateway_timestamp" ON pending_transactions (first_submitted_to_gateway_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_intent_hash" ON pending_transactions (intent_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transactions_payload_hash" ON pending_transactions (payload_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_resubmit_from_timestamp" ON pending_transactions (resubmit_from_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_resource_entity_supply_history_resource_entity_id_from_stat~" ON resource_entity_supply_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE UNIQUE INDEX "IX_resource_holders_entity_id_resource_entity_id" ON resource_holders (entity_id, resource_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_resource_holders_entity_id_resource_entity_id_balance" ON resource_holders (entity_id, resource_entity_id, balance); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_schema_entry_aggregate_history_entity_id_from_state_version" ON schema_entry_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_schema_entry_definition_entity_id_schema_hash" ON schema_entry_definition (entity_id, schema_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_state_history_entity_id_from_state_version" ON state_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_aggregate_history_entity_id_fr~" ON unverified_standard_metadata_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_entry_history_entity_id_discri~" ON unverified_standard_metadata_entry_history (entity_id, discriminator, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_epoch" ON validator_active_set_history (epoch); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_from_state_version" ON validator_active_set_history (from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_validator_public_key_history_id" ON validator_active_set_history (validator_public_key_history_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_cumulative_emission_history_validator_entity_id_e~" ON validator_cumulative_emission_history (validator_entity_id, epoch_number); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_from_state~" ON validator_public_key_history (validator_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_key_type_k~" ON validator_public_key_history (validator_entity_id, key_type, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240819084018_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240830081152_InitialCreate') THEN INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") - VALUES ('20240819084018_InitialCreate', '8.0.2'); + VALUES ('20240830081152_InitialCreate', '8.0.2'); END IF; END $EF$; COMMIT; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs index 8ee876e95..bc7c7e29b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs @@ -62,17 +62,14 @@ * permissions under this License. */ -// +// using System; using System.Collections.Generic; using System.Numerics; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.StandardMetadata; -using RadixDlt.NetworkGateway.PostgresIntegration; using RadixDlt.NetworkGateway.PostgresIntegration.Models; #nullable disable @@ -494,7 +491,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.UseTphMappingStrategy(); }); - modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataAggregateHistory", b => + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataEntryDefinition", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -511,19 +508,21 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("bigint") .HasColumnName("from_state_version"); - b.Property>("MetadataIds") + b.Property("Key") .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("metadata_ids"); + .HasColumnType("text") + .HasColumnName("key"); b.HasKey("Id"); b.HasIndex("EntityId", "FromStateVersion"); - b.ToTable("entity_metadata_aggregate_history"); + b.HasIndex("EntityId", "Key"); + + b.ToTable("entity_metadata_entry_definition"); }); - modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataHistory", b => + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataEntryHistory", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -532,9 +531,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - b.Property("EntityId") + b.Property("EntityMetadataEntryDefinitionId") .HasColumnType("bigint") - .HasColumnName("entity_id"); + .HasColumnName("entity_metadata_entry_definition_id"); b.Property("FromStateVersion") .HasColumnType("bigint") @@ -548,20 +547,47 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("boolean") .HasColumnName("is_locked"); - b.Property("Key") - .IsRequired() - .HasColumnType("text") - .HasColumnName("key"); - b.Property("Value") .HasColumnType("bytea") .HasColumnName("value"); b.HasKey("Id"); - b.HasIndex("EntityId", "Key", "FromStateVersion"); + b.HasIndex("EntityMetadataEntryDefinitionId", "FromStateVersion"); + + b.ToTable("entity_metadata_entry_history"); + }); + + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityMetadataTotalsHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("EntityId") + .HasColumnType("bigint") + .HasColumnName("entity_id"); + + b.Property("FromStateVersion") + .HasColumnType("bigint") + .HasColumnName("from_state_version"); + + b.Property("TotalEntriesExcludingDeleted") + .HasColumnType("bigint") + .HasColumnName("total_entries_excluding_deleted"); + + b.Property("TotalEntriesIncludingDeleted") + .HasColumnType("bigint") + .HasColumnName("total_entries_including_deleted"); + + b.HasKey("Id"); + + b.HasIndex("EntityId", "FromStateVersion"); - b.ToTable("entity_metadata_history"); + b.ToTable("entity_metadata_totals_history"); }); modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.EntityResourceAggregateHistory", b => diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/CustomTypes/IdBoundaryCursor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/CustomTypes/IdBoundaryCursor.cs new file mode 100644 index 000000000..a44c941f7 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/CustomTypes/IdBoundaryCursor.cs @@ -0,0 +1,3 @@ +namespace RadixDlt.NetworkGateway.PostgresIntegration.Models.CustomTypes; + +public readonly record struct IdBoundaryCursor(long StateVersion, long Id); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryDefinition.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryDefinition.cs new file mode 100644 index 000000000..9a1e43e84 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryDefinition.cs @@ -0,0 +1,85 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Models; + +[Table("entity_metadata_entry_definition")] +internal class EntityMetadataEntryDefinition +{ + [Key] + [Column("id")] + public long Id { get; set; } + + [Column("from_state_version")] + public long FromStateVersion { get; set; } + + [Column("entity_id")] + public long EntityId { get; set; } + + [Column("key")] + public string Key { get; set; } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataHistory.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryHistory.cs similarity index 95% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataHistory.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryHistory.cs index 0294cec01..b1960b920 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataHistory.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataEntryHistory.cs @@ -68,8 +68,8 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Models; -[Table("entity_metadata_history")] -internal class EntityMetadataHistory +[Table("entity_metadata_entry_history")] +internal class EntityMetadataEntryHistory { [Key] [Column("id")] @@ -78,11 +78,8 @@ internal class EntityMetadataHistory [Column("from_state_version")] public long FromStateVersion { get; set; } - [Column("entity_id")] - public long EntityId { get; set; } - - [Column("key")] - public string Key { get; set; } + [Column("entity_metadata_entry_definition_id")] + public long EntityMetadataEntryDefinitionId { get; set; } [Column("value")] public byte[]? Value { get; set; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataAggregateHistory.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataTotalsHistory.cs similarity index 91% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataAggregateHistory.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataTotalsHistory.cs index ed1ebba11..ad630f1a9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataAggregateHistory.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/EntityMetadataTotalsHistory.cs @@ -62,14 +62,13 @@ * permissions under this License. */ -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; namespace RadixDlt.NetworkGateway.PostgresIntegration.Models; -[Table("entity_metadata_aggregate_history")] -internal class EntityMetadataAggregateHistory : IAggregateHolder +[Table("entity_metadata_totals_history")] +internal class EntityMetadataTotalsHistory { [Key] [Column("id")] @@ -81,11 +80,9 @@ internal class EntityMetadataAggregateHistory : IAggregateHolder [Column("entity_id")] public long EntityId { get; set; } - [Column("metadata_ids")] - public List MetadataIds { get; set; } + [Column("total_entries_including_deleted")] + public long TotalEntriesIncludingDeleted { get; set; } - IEnumerable<(string Name, int TotalCount)> IAggregateHolder.AggregateCounts() - { - yield return (nameof(MetadataIds), MetadataIds.Count); - } + [Column("total_entries_excluding_deleted")] + public long TotalEntriesExcludingDeleted { get; set; } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs index d40a7c197..44d1eecb8 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs @@ -63,8 +63,8 @@ */ using Dapper; -using Microsoft.EntityFrameworkCore; using System.Collections.Generic; +using System.Data.Common; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading; @@ -74,11 +74,11 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; internal static class DapperWrapperExtensions { - public static async Task> ToList(this IDapperWrapper dapperWrapper, DbContext dbContext, [StringSyntax("postgresql")] string sql, object parameters, CancellationToken token = default) + public static async Task> ToListAsync(this IDapperWrapper dapperWrapper, DbConnection dbConnection, [StringSyntax("postgresql")] string sql, object parameters, CancellationToken token = default) { var commandDefinition = new CommandDefinition(commandText: sql, parameters: parameters, cancellationToken: token); - var result = await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), commandDefinition); + var result = await dapperWrapper.QueryAsync(dbConnection, commandDefinition); return result.ToList(); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index d409ada7d..b4f8292bf 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -76,6 +76,7 @@ using RadixDlt.NetworkGateway.GatewayApi.Exceptions; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Services.Queries; using System; using System.Collections.Generic; using System.Collections.Immutable; @@ -103,8 +104,6 @@ private record RoyaltyVaultBalanceViewModel(long RoyaltyVaultEntityId, string Ba private record NonFungibleIdsViewModel(long Id, long FromStateVersion, string NonFungibleId); - private record struct ExplicitMetadataLookup(long EntityId, string MetadataKey); - private readonly TokenAmount _tokenAmount100 = TokenAmount.FromDecimalString("100"); private readonly INetworkConfigurationProvider _networkConfigurationProvider; private readonly IOptionsSnapshot _endpointConfiguration; @@ -150,7 +149,7 @@ public EntityStateQuerier( var globalPersistedComponentEntities = persistedComponentEntities.Where(x => x.IsGlobal).ToList(); // TODO ideally we'd like to run all those in parallel - var metadata = await GetMetadataSlices(entities.Select(e => e.Id).ToArray(), 0, defaultPageSize, ledgerState, token); + var metadata = await GetMetadataSlices(entities.Select(e => e.Id).ToArray(), null, defaultPageSize, ledgerState, token); var roleAssignmentsHistory = await _roleAssignmentQuerier.GetRoleAssignmentsHistory(globalPersistedComponentEntities, ledgerState, token); var resourcesSupplyData = await GetResourcesSupplyData(resourceEntities.Select(x => x.Id).ToArray(), ledgerState, token); var packageBlueprintHistory = await GetPackageBlueprintHistory(packageEntities.Select(e => e.Id).ToArray(), 0, packagePageSize, ledgerState, token); @@ -180,12 +179,25 @@ public EntityStateQuerier( // fetch [non-]fungible resource collections for all components, even internal ones, for backwards compability var fungibleResources = await EntityFungibleResourcesPageSlice(componentEntities.Select(e => e.Id).ToArray(), aggregatePerVault, 0, defaultPageSize, ledgerState, token); - var nonFungibleResources = await EntityNonFungibleResourcesPageSlice(componentEntities.Select(e => e.Id).ToArray(), aggregatePerVault, optIns.NonFungibleIncludeNfids, 0, - defaultPageSize, ledgerState, token); + var nonFungibleResources = await EntityNonFungibleResourcesPageSlice( + componentEntities.Select(e => e.Id).ToArray(), + aggregatePerVault, + optIns.NonFungibleIncludeNfids, + 0, + defaultPageSize, + ledgerState, + token); var resourceAddressToEntityId = await ResolveResourceEntityIds(fungibleResources.Values, nonFungibleResources.Values, token); var explicitMetadata = optIns.ExplicitMetadata?.Any() == true - ? await GetExplicitMetadata(entities.Select(e => e.Id).Concat(resourceAddressToEntityId.Values).ToArray(), optIns.ExplicitMetadata.ToArray(), ledgerState, token) + ? await ExplicitMetadataQuery.Read( + _dbContext.Database.GetDbConnection(), + _dapperWrapper, + entities.Select(e => e.Id).Concat(resourceAddressToEntityId.Values).ToArray(), + optIns.ExplicitMetadata.ToArray(), + ledgerState, + networkConfiguration.Id, + token) : null; var items = new List(); @@ -381,14 +393,15 @@ public EntityStateQuerier( } } - items.Add(new GatewayModel.StateEntityDetailsResponseItem( - address: entity.Address, - fungibleResources: haveFungibles ? fungibles : null, - nonFungibleResources: haveNonFungibles ? nonFungibles : null, - ancestorIdentities: ancestorIdentities, - metadata: metadata[entity.Id], - explicitMetadata: explicitMetadata?[entity.Id], - details: details)); + items.Add( + new GatewayModel.StateEntityDetailsResponseItem( + address: entity.Address, + fungibleResources: haveFungibles ? fungibles : null, + nonFungibleResources: haveNonFungibles ? nonFungibles : null, + ancestorIdentities: ancestorIdentities, + metadata: metadata[entity.Id], + explicitMetadata: explicitMetadata?[entity.Id], + details: details)); } return new GatewayModel.StateEntityDetailsResponse(ledgerState, items); @@ -435,16 +448,31 @@ public EntityStateQuerier( { var entity = await GetEntity(pageRequest.Address, ledgerState, token); var result = (await EntityNonFungibleResourcesPageSlice( - new[] { entity.Id }, aggregatePerVault, optIns.NonFungibleIncludeNfids, pageRequest.Offset, pageRequest.Limit, ledgerState, token))[entity.Id]; - - if (optIns.ExplicitMetadata?.Any() == true) + new[] { entity.Id }, + aggregatePerVault, + optIns.NonFungibleIncludeNfids, + pageRequest.Offset, + pageRequest.Limit, + ledgerState, + token))[entity.Id]; + + if (optIns.ExplicitMetadata?.Any() != true) { - var resourceAddressToEntityId = await ResolveResourceEntityIds(null, new[] { result }, token); - var explicitMetadata = await GetExplicitMetadata(resourceAddressToEntityId.Values.ToArray(), optIns.ExplicitMetadata.ToArray(), ledgerState, token); - - result.Items.ForEach(nfr => nfr.ExplicitMetadata = explicitMetadata[resourceAddressToEntityId[(EntityAddress)nfr.ResourceAddress]]); + return new GatewayModel.StateEntityNonFungiblesPageResponse(ledgerState, result.TotalCount, result.NextCursor, result.Items, pageRequest.Address); } + var resourceAddressToEntityId = await ResolveResourceEntityIds(null, new[] { result }, token); + var explicitMetadata = await ExplicitMetadataQuery.Read( + _dbContext.Database.GetDbConnection(), + _dapperWrapper, + resourceAddressToEntityId.Values.ToArray(), + optIns.ExplicitMetadata.ToArray(), + ledgerState, + (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id, + token); + + result.Items.ForEach(nfr => nfr.ExplicitMetadata = explicitMetadata[resourceAddressToEntityId[(EntityAddress)nfr.ResourceAddress]]); + return new GatewayModel.StateEntityNonFungiblesPageResponse(ledgerState, result.TotalCount, result.NextCursor, result.Items, pageRequest.Address); } @@ -480,7 +508,8 @@ public EntityStateQuerier( } var mapped = nonFungibles - .Select(x => + .Select( + x => { List? items = null; string? nextCursor = null; @@ -518,14 +547,23 @@ public EntityStateQuerier( var entity = await GetEntity(pageRequest.Address, ledgerState, token); var result = await EntityFungibleResourcesPageSlice(entity.Id, aggregatePerVault, pageRequest.Offset, pageRequest.Limit, ledgerState, token); - if (optIns.ExplicitMetadata?.Any() == true) + if (optIns.ExplicitMetadata?.Any() != true) { - var resourceAddressToEntityId = await ResolveResourceEntityIds(new[] { result }, null, token); - var explicitMetadata = await GetExplicitMetadata(resourceAddressToEntityId.Values.ToArray(), optIns.ExplicitMetadata.ToArray(), ledgerState, token); - - result.Items.ForEach(fr => fr.ExplicitMetadata = explicitMetadata[resourceAddressToEntityId[(EntityAddress)fr.ResourceAddress]]); + return new GatewayModel.StateEntityFungiblesPageResponse(ledgerState, result.TotalCount, result.NextCursor, result.Items, pageRequest.Address); } + var resourceAddressToEntityId = await ResolveResourceEntityIds(new[] { result }, null, token); + var explicitMetadata = await ExplicitMetadataQuery.Read( + _dbContext.Database.GetDbConnection(), + _dapperWrapper, + resourceAddressToEntityId.Values.ToArray(), + optIns.ExplicitMetadata.ToArray(), + ledgerState, + (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id, + token); + + result.Items.ForEach(fr => fr.ExplicitMetadata = explicitMetadata[resourceAddressToEntityId[(EntityAddress)fr.ResourceAddress]]); + return new GatewayModel.StateEntityFungiblesPageResponse(ledgerState, result.TotalCount, result.NextCursor, result.Items, pageRequest.Address); } @@ -545,7 +583,7 @@ public EntityStateQuerier( } public async Task EntityMetadata( - IEntityStateQuerier.PageRequest request, + IEntityStateQuerier.MetadataPageRequest request, GatewayModel.LedgerState ledgerState, CancellationToken token = default) { @@ -559,7 +597,7 @@ public EntityStateQuerier( } else { - metadata = (await GetMetadataSlices(new[] { entity.Id }, request.Offset, request.Limit, ledgerState, token))[entity.Id]; + metadata = (await GetMetadataSlices(new[] { entity.Id }, request.Cursor, request.Limit, ledgerState, token))[entity.Id]; } return new GatewayModel.StateEntityMetadataPageResponse(ledgerState, metadata.TotalCount, metadata.NextCursor, metadata.Items, entity.Address); @@ -702,7 +740,9 @@ FROM non_fungible_schema_history nfsh cancellationToken: token); var nonFungibleDataSchema = await _dapperWrapper.QueryFirstOrDefaultAsync( - _dbContext.Database.GetDbConnection(), nonFungibleDataSchemaQuery, "GetNonFungibleDataSchema" + _dbContext.Database.GetDbConnection(), + nonFungibleDataSchemaQuery, + "GetNonFungibleDataSchema" ); if (nonFungibleDataSchema == null) @@ -742,15 +782,20 @@ ORDER BY nfid.from_state_version DESC foreach (var vm in result) { var programmaticJson = !vm.IsDeleted - ? ScryptoSborUtils.DataToProgrammaticJson(vm.Data, nonFungibleDataSchema.Schema, - nonFungibleDataSchema.SborTypeKind, nonFungibleDataSchema.TypeIndex, (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id) + ? ScryptoSborUtils.DataToProgrammaticJson( + vm.Data, + nonFungibleDataSchema.Schema, + nonFungibleDataSchema.SborTypeKind, + nonFungibleDataSchema.TypeIndex, + (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id) : null; - items.Add(new GatewayModel.StateNonFungibleDetailsResponseItem( - nonFungibleId: vm.NonFungibleId, - isBurned: vm.IsDeleted, - data: !vm.IsDeleted ? new GatewayModel.ScryptoSborValue(vm.Data.ToHex(), programmaticJson) : null, - lastUpdatedAtStateVersion: vm.DataLastUpdatedAtStateVersion)); + items.Add( + new GatewayModel.StateNonFungibleDetailsResponseItem( + nonFungibleId: vm.NonFungibleId, + isBurned: vm.IsDeleted, + data: !vm.IsDeleted ? new GatewayModel.ScryptoSborValue(vm.Data.ToHex(), programmaticJson) : null, + lastUpdatedAtStateVersion: vm.DataLastUpdatedAtStateVersion)); } return new GatewayModel.StateNonFungibleDataResponse( @@ -838,14 +883,15 @@ FROM entities e ledgerState: ledgerState, resourceAddress: resourceAddress.ToString(), nonFungibleIds: vaultLocationResults - .Select(x => new GatewayModel.StateNonFungibleLocationResponseItem( - nonFungibleId: x.NonFungibleId, - owningVaultAddress: !x.IsDeleted ? x.OwnerVaultAddress : null, - owningVaultParentAncestorAddress: !x.IsDeleted ? vaultAncestorResults[x.OwnerVaultId].VaultParentAncestorAddress : null, - owningVaultGlobalAncestorAddress: !x.IsDeleted ? vaultAncestorResults[x.OwnerVaultId].VaultGlobalAncestorAddress : null, - isBurned: x.IsDeleted, - lastUpdatedAtStateVersion: x.FromStateVersion - )) + .Select( + x => new GatewayModel.StateNonFungibleLocationResponseItem( + nonFungibleId: x.NonFungibleId, + owningVaultAddress: !x.IsDeleted ? x.OwnerVaultAddress : null, + owningVaultParentAncestorAddress: !x.IsDeleted ? vaultAncestorResults[x.OwnerVaultId].VaultParentAncestorAddress : null, + owningVaultGlobalAncestorAddress: !x.IsDeleted ? vaultAncestorResults[x.OwnerVaultId].VaultGlobalAncestorAddress : null, + isBurned: x.IsDeleted, + lastUpdatedAtStateVersion: x.FromStateVersion + )) .ToList()); } @@ -895,20 +941,23 @@ FROM entities e var validatorIds = validatorsAndOneMore.Take(validatorsPageSize).Select(e => e.Id).ToArray(); var validatorVaultIds = validatorsAndOneMore .Take(validatorsPageSize) - .Aggregate(new List(), (aggregated, validator) => - { - aggregated.Add(validator.GetStakeVaultEntityId()); - aggregated.Add(validator.GetPendingXrdWithdrawVaultEntityId()); - aggregated.Add(validator.GetLockedOwnerStakeUnitVaultEntityId()); - aggregated.Add(validator.GetPendingOwnerStakeUnitUnlockVaultEntityId()); + .Aggregate( + new List(), + (aggregated, validator) => + { + aggregated.Add(validator.GetStakeVaultEntityId()); + aggregated.Add(validator.GetPendingXrdWithdrawVaultEntityId()); + aggregated.Add(validator.GetLockedOwnerStakeUnitVaultEntityId()); + aggregated.Add(validator.GetPendingOwnerStakeUnitUnlockVaultEntityId()); - return aggregated; - }) + return aggregated; + }) .ToList(); var stateHistory = await _dbContext .StateHistory - .FromSqlInterpolated($@" + .FromSqlInterpolated( + $@" WITH variables (validator_entity_id) AS (SELECT UNNEST({validatorIds})) SELECT esh.* FROM variables v @@ -924,7 +973,8 @@ LIMIT 1 var vaultHistory = await _dbContext .EntityVaultHistory - .FromSqlInterpolated($@" + .FromSqlInterpolated( + $@" WITH variables (vault_entity_id) AS (SELECT UNNEST({validatorVaultIds})) SELECT evh.* FROM variables v @@ -945,43 +995,53 @@ LIMIT 1 .AnnotateMetricName("GetVaultAddresses") .ToDictionaryAsync(e => e.Id, e => e.Address, token); - var metadataById = await GetMetadataSlices(validatorIds, 0, _endpointConfiguration.Value.DefaultPageSize, ledgerState, token); + var metadataById = await GetMetadataSlices(validatorIds, null, _endpointConfiguration.Value.DefaultPageSize, ledgerState, token); var items = validatorsAndOneMore .Take(validatorsPageSize) - .Select(v => - { - GatewayModel.ValidatorCollectionItemActiveInEpoch? activeInEpoch = null; - - if (activeSetById.TryGetValue(v.Id, out var validatorActiveSetHistory)) + .Select( + v => { - var stake = validatorActiveSetHistory.Stake.ToString(); - var stakePercentage = (validatorActiveSetHistory.Stake * _tokenAmount100 / totalStake).ToString(); + GatewayModel.ValidatorCollectionItemActiveInEpoch? activeInEpoch = null; - activeInEpoch = new GatewayModel.ValidatorCollectionItemActiveInEpoch( - stake, - double.Parse(stakePercentage, NumberFormatInfo.InvariantInfo), - validatorActiveSetHistory.PublicKey.ToGatewayPublicKey()); - } + if (activeSetById.TryGetValue(v.Id, out var validatorActiveSetHistory)) + { + var stake = validatorActiveSetHistory.Stake.ToString(); + var stakePercentage = (validatorActiveSetHistory.Stake * _tokenAmount100 / totalStake).ToString(); - var stakeVault = vaultHistory[v.GetStakeVaultEntityId()]; - var pendingXrdWithdrawVaultVault = vaultHistory[v.GetPendingXrdWithdrawVaultEntityId()]; - var lockedOwnerStakeUnitVault = vaultHistory[v.GetLockedOwnerStakeUnitVaultEntityId()]; - var pendingOwnerStakeUnitUnlockVault = vaultHistory[v.GetPendingOwnerStakeUnitUnlockVaultEntityId()]; - var effectiveFeeFactor = ValidatorEffectiveFeeFactorProvider.ExtractFeeFactorFromValidatorState(stateHistory[v.Id].JsonState, ledgerState.Epoch); - - return new GatewayModel.ValidatorCollectionItem( - v.Address, - new GatewayModel.ValidatorVaultItem(stakeVault.Balance.ToString(), stakeVault.FromStateVersion, vaultAddresses[stakeVault.VaultEntityId]), - new GatewayModel.ValidatorVaultItem(pendingXrdWithdrawVaultVault.Balance.ToString(), pendingXrdWithdrawVaultVault.FromStateVersion, vaultAddresses[pendingXrdWithdrawVaultVault.VaultEntityId]), - new GatewayModel.ValidatorVaultItem(lockedOwnerStakeUnitVault.Balance.ToString(), lockedOwnerStakeUnitVault.FromStateVersion, vaultAddresses[lockedOwnerStakeUnitVault.VaultEntityId]), - new GatewayModel.ValidatorVaultItem(pendingOwnerStakeUnitUnlockVault.Balance.ToString(), pendingOwnerStakeUnitUnlockVault.FromStateVersion, vaultAddresses[pendingOwnerStakeUnitUnlockVault.VaultEntityId]), - new JRaw(stateHistory[v.Id].JsonState), - activeInEpoch, - metadataById[v.Id], - effectiveFeeFactor - ); - }) + activeInEpoch = new GatewayModel.ValidatorCollectionItemActiveInEpoch( + stake, + double.Parse(stakePercentage, NumberFormatInfo.InvariantInfo), + validatorActiveSetHistory.PublicKey.ToGatewayPublicKey()); + } + + var stakeVault = vaultHistory[v.GetStakeVaultEntityId()]; + var pendingXrdWithdrawVaultVault = vaultHistory[v.GetPendingXrdWithdrawVaultEntityId()]; + var lockedOwnerStakeUnitVault = vaultHistory[v.GetLockedOwnerStakeUnitVaultEntityId()]; + var pendingOwnerStakeUnitUnlockVault = vaultHistory[v.GetPendingOwnerStakeUnitUnlockVaultEntityId()]; + var effectiveFeeFactor = ValidatorEffectiveFeeFactorProvider.ExtractFeeFactorFromValidatorState(stateHistory[v.Id].JsonState, ledgerState.Epoch); + + return new GatewayModel.ValidatorCollectionItem( + v.Address, + new GatewayModel.ValidatorVaultItem(stakeVault.Balance.ToString(), stakeVault.FromStateVersion, vaultAddresses[stakeVault.VaultEntityId]), + new GatewayModel.ValidatorVaultItem( + pendingXrdWithdrawVaultVault.Balance.ToString(), + pendingXrdWithdrawVaultVault.FromStateVersion, + vaultAddresses[pendingXrdWithdrawVaultVault.VaultEntityId]), + new GatewayModel.ValidatorVaultItem( + lockedOwnerStakeUnitVault.Balance.ToString(), + lockedOwnerStakeUnitVault.FromStateVersion, + vaultAddresses[lockedOwnerStakeUnitVault.VaultEntityId]), + new GatewayModel.ValidatorVaultItem( + pendingOwnerStakeUnitUnlockVault.Balance.ToString(), + pendingOwnerStakeUnitUnlockVault.FromStateVersion, + vaultAddresses[pendingOwnerStakeUnitUnlockVault.VaultEntityId]), + new JRaw(stateHistory[v.Id].JsonState), + activeInEpoch, + metadataById[v.Id], + effectiveFeeFactor + ); + }) .ToList(); var nextCursor = validatorsAndOneMore.Count == validatorsPageSize + 1 @@ -1021,139 +1081,31 @@ LIMIT 1 return result.ToList(); } - private async Task> GetMetadataSlices( + private async Task> GetMetadataSlices( long[] entityIds, - int offset, + GatewayModel.IdBoundaryCoursor? cursor, int limit, GatewayModel.LedgerState ledgerState, CancellationToken token) { - var result = new Dictionary(); - - var cd = new CommandDefinition( - commandText: @" -WITH variables (entity_id) AS ( - SELECT UNNEST(@entityIds) -), -metadata_slices AS ( - SELECT variables.entity_id, emah.metadata_slice, emah.metadata_total_count - FROM variables - INNER JOIN LATERAL ( - SELECT metadata_ids[@startIndex:@endIndex] AS metadata_slice, cardinality(metadata_ids) AS metadata_total_count - FROM entity_metadata_aggregate_history - WHERE entity_id = variables.entity_id AND from_state_version <= @stateVersion - ORDER BY from_state_version DESC - LIMIT 1 - ) emah ON TRUE -) -SELECT emh.from_state_version AS FromStateVersion, emh.entity_id AS EntityId, emh.key AS Key, emh.value AS Value, emh.is_locked AS IsLocked, ms.metadata_total_count AS TotalCount -FROM metadata_slices AS ms -INNER JOIN LATERAL UNNEST(metadata_slice) WITH ORDINALITY AS metadata_join(id, ordinality) ON TRUE -INNER JOIN entity_metadata_history emh ON emh.id = metadata_join.id AND emh.is_deleted = FALSE -ORDER BY metadata_join.ordinality ASC;", - parameters: new - { - entityIds = entityIds, - stateVersion = ledgerState.StateVersion, - startIndex = offset + 1, - endIndex = offset + limit, - }, - cancellationToken: token); - - foreach (var vm in await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) - { - if (!result.ContainsKey(vm.EntityId)) - { - result[vm.EntityId] = new GatewayModel.EntityMetadataCollection(vm.TotalCount, CursorGenerator.GenerateOffsetCursor(offset, limit, vm.TotalCount), new List()); - } - - var networkId = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id; - var value = ScryptoSborUtils.DecodeToGatewayMetadataItemValue(vm.Value, networkId); - var programmaticJson = ScryptoSborUtils.DataToProgrammaticJson(vm.Value, networkId); - var entityMetadataItemValue = new GatewayModel.EntityMetadataItemValue(vm.Value.ToHex(), programmaticJson, value); - - result[vm.EntityId].Items.Add(new GatewayModel.EntityMetadataItem(vm.Key, entityMetadataItemValue, vm.IsLocked, vm.FromStateVersion)); - } - - foreach (var missing in entityIds.Except(result.Keys)) - { - result[missing] = GatewayModel.EntityMetadataCollection.Empty; - } - - return result; - } - - private async Task> GetExplicitMetadata( - long[] entityIds, - string[] metadataKeys, - GatewayModel.LedgerState ledgerState, - CancellationToken token) - { - var lookup = new HashSet(); - var entityIdsParameter = new List(); - var metadataKeysParameter = new List(); - - foreach (var entityId in entityIds) - { - foreach (var metadataKey in metadataKeys) - { - lookup.Add(new ExplicitMetadataLookup(entityId, metadataKey)); - } - } - - foreach (var (entityId, metadataKey) in lookup) + if (!entityIds.Any()) { - entityIdsParameter.Add(entityId); - metadataKeysParameter.Add(metadataKey); + return ImmutableDictionary.Empty; } - var metadataHistory = await _dbContext - .EntityMetadataHistory - .FromSqlInterpolated(@$" -WITH variables (entity_id, metadata_key) AS ( - SELECT UNNEST({entityIdsParameter}), UNNEST({metadataKeysParameter}) -) -SELECT emh.* -FROM variables -INNER JOIN LATERAL ( - SELECT * - FROM entity_metadata_history - WHERE entity_id = variables.entity_id AND key = variables.metadata_key AND from_state_version <= {ledgerState.StateVersion} - ORDER BY from_state_version DESC - LIMIT 1 -) emh ON TRUE;") - .AnnotateMetricName() - .ToListAsync(token); - - var result = new Dictionary(); - - foreach (var mh in metadataHistory) - { - if (mh.IsDeleted) - { - continue; - } - - if (!result.ContainsKey(mh.EntityId)) + return await MetadataPageQuery.ReadPages( + _dbContext.Database.GetDbConnection(), + _dapperWrapper, + ledgerState, + entityIds, + new MetadataPageQuery.QueryConfiguration { - result[mh.EntityId] = new GatewayModel.EntityMetadataCollection(items: new List()); - } - - var networkId = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id; - var value = ScryptoSborUtils.DecodeToGatewayMetadataItemValue(mh.Value, networkId); - var programmaticJson = ScryptoSborUtils.DataToProgrammaticJson(mh.Value, networkId); - var entityMetadataItemValue = new GatewayModel.EntityMetadataItemValue(mh.Value.ToHex(), programmaticJson, value); - - result[mh.EntityId].Items.Add(new GatewayModel.EntityMetadataItem(mh.Key, entityMetadataItemValue, mh.IsLocked, mh.FromStateVersion)); - result[mh.EntityId].TotalCount = result[mh.EntityId].TotalCount.HasValue ? result[mh.EntityId].TotalCount + 1 : 1; - } - - foreach (var missing in entityIds.Except(result.Keys)) - { - result[missing] = GatewayModel.EntityMetadataCollection.Empty; - } - - return result; + Cursor = cursor, + MaxPageSize = limit, + MaxDefinitionsToRead = _endpointConfiguration.Value.MaxDefinitionsLookupLimit, + }, + (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id, + token); } private async Task> GetEntitySchemaHistory(long[] entityIds, int offset, int limit, GatewayModel.LedgerState ledgerState, CancellationToken token) @@ -1252,10 +1204,11 @@ order by ah.ord { totalCount = vm.ResourcesTotalCount; - items.Add(new GatewayModel.FungibleResourcesCollectionItemGloballyAggregated( - resourceAddress: vm.ResourceEntityAddress, - amount: TokenAmount.FromSubUnitsString(vm.Balance).ToString(), - lastUpdatedAtStateVersion: vm.LastUpdatedAtStateVersion)); + items.Add( + new GatewayModel.FungibleResourcesCollectionItemGloballyAggregated( + resourceAddress: vm.ResourceEntityAddress, + amount: TokenAmount.FromSubUnitsString(vm.Balance).ToString(), + lastUpdatedAtStateVersion: vm.LastUpdatedAtStateVersion)); } return new GatewayModel.FungibleResourcesCollection(totalCount, CursorGenerator.GenerateOffsetCursor(offset, limit, totalCount), items); @@ -1276,7 +1229,8 @@ private async Task> GetResourcesSu var result = await _dbContext .ResourceEntitySupplyHistory - .FromSqlInterpolated($@" + .FromSqlInterpolated( + $@" WITH variables (entity_id) AS (SELECT UNNEST({entityIds})) SELECT resh.* FROM variables @@ -1382,7 +1336,8 @@ private async Task> GetStateHistory(ICollection> GetCo return (await _dbContext .ComponentEntityMethodRoyaltyEntryHistory - .FromSqlInterpolated($@" + .FromSqlInterpolated( + $@" WITH variables (component_entity_id) AS ( SELECT UNNEST({componentEntityIds}) ), diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ExtensionsQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ExtensionsQuerier.cs index 6dd4d8859..6bbb9d115 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ExtensionsQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ExtensionsQuerier.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Numerics; @@ -113,8 +112,8 @@ public ExtensionsQuerier(ReadOnlyDbContext dbContext, IDapperWrapper dapperWrapp var totalCount = await _dbContext.ResourceHolders.CountAsync(x => x.ResourceEntityId == resourceEntity.Id, token); - var entriesAndOneMore = await _dapperWrapper.ToList( - _dbContext, + var entriesAndOneMore = await _dapperWrapper.ToListAsync( + _dbContext.Database.GetDbConnection(), @" SELECT ro.id as Id, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs index f65123548..219c6256b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs @@ -62,6 +62,7 @@ * permissions under this License. */ +using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; @@ -122,8 +123,8 @@ private record struct DbRow(EntityRelationship BaseRelationship, string BaseEnti private async Task> GetFromDatabase(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token) { - var rows = await _dapperWrapper.ToList( - _dbContext, + var rows = await _dapperWrapper.ToListAsync( + _dbContext.Database.GetDbConnection(), @"WITH variables AS ( SELECT diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/ExplicitMetadataQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/ExplicitMetadataQuery.cs new file mode 100644 index 000000000..9d133d3f6 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/ExplicitMetadataQuery.cs @@ -0,0 +1,186 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services.Queries; + +internal static class ExplicitMetadataQuery +{ + private readonly record struct ExplicitMetadataLookup(long EntityId, string MetadataKey); + + private readonly record struct QueryResultRow( + long EntityId, + string Key, + long KeyFirstSeenStateVersion, + byte[] Value, + bool IsLocked, + long LastUpdatedStateVersion, + bool FilterOut + ); + + internal static async Task> Read( + DbConnection dbConnection, + IDapperWrapper dapperWrapper, + long[] entityIds, + string[] metadataKeys, + GatewayApiSdk.Model.LedgerState ledgerState, + byte networkId, + CancellationToken token = default) + { + var lookup = new HashSet(); + var entityIdsParameter = new List(); + var metadataKeysParameter = new List(); + + foreach (var entityId in entityIds) + { + foreach (var metadataKey in metadataKeys) + { + lookup.Add(new ExplicitMetadataLookup(entityId, metadataKey)); + } + } + + foreach (var (entityId, metadataKey) in lookup) + { + entityIdsParameter.Add(entityId); + metadataKeysParameter.Add(metadataKey); + } + + var queryParameters = new + { + entityIds = entityIdsParameter, + metadataKeys = metadataKeysParameter, + atLedgerState = ledgerState.StateVersion, + }; + + const string QueryText = @" +WITH variables (entity_id, metadata_key) AS ( + SELECT UNNEST(@entityIds), UNNEST(@metadataKeys) +) +SELECT + variables.entity_id as EntityId, + entries_with_definitions.key, + entries_with_definitions.KeyFirstSeenStateVersion, + entries_with_definitions.value, + entries_with_definitions.is_locked AS IsLocked, + entries_with_definitions.LastUpdatedStateVersion, + COALESCE(entries_with_definitions.FilterOut, TRUE) AS FilterOut +FROM variables +LEFT JOIN LATERAL ( + SELECT + definition.entity_id, + definition.key, + definition.from_state_version AS KeyFirstSeenStateVersion, + history.value, + history.is_locked, + history.is_deleted AS FilterOut, + history.from_state_version AS LastUpdatedStateVersion + FROM entity_metadata_entry_definition definition + INNER JOIN LATERAL ( + SELECT * + FROM entity_metadata_entry_history history + WHERE history.entity_metadata_entry_definition_id = definition.id AND from_state_version <= @atLedgerState + ORDER BY history.from_state_version DESC + LIMIT 1 + ) history on true + WHERE definition.entity_id = variables.entity_id AND definition.key = variables.metadata_key + ) entries_with_definitions on TRUE +;"; + + var queryResult = (await dapperWrapper.ToListAsync( + dbConnection, + QueryText, + queryParameters, + token)).ToList(); + + var result = queryResult + .GroupBy(r => r.EntityId) + .ToDictionary( + g => g.Key, + g => + { + var items = g + .Where(x => !x.FilterOut) + .Select( + x => + { + var value = ScryptoSborUtils.DecodeToGatewayMetadataItemValue(x.Value, networkId); + var programmaticJson = ScryptoSborUtils.DataToProgrammaticJson(x.Value, networkId); + var entityMetadataItemValue = new GatewayModel.EntityMetadataItemValue(x.Value.ToHex(), programmaticJson, value); + return new GatewayModel.EntityMetadataItem(x.Key, entityMetadataItemValue, x.IsLocked, x.LastUpdatedStateVersion); + }) + .ToList(); + + return new GatewayModel.EntityMetadataCollection(items.Count, items: items); + }); + + return result; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/MetadataPageQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/MetadataPageQuery.cs new file mode 100644 index 000000000..4fd41b797 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/Queries/MetadataPageQuery.cs @@ -0,0 +1,255 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.PostgresIntegration.Models.CustomTypes; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services.Queries; + +internal static class MetadataPageQuery +{ + internal readonly record struct QueryConfiguration( + GatewayModel.IdBoundaryCoursor? Cursor, + int MaxPageSize, + int MaxDefinitionsToRead + ); + + private readonly record struct QueryResultRow( + long EntityId, + long TotalEntriesExcludingDeleted, + long TotalEntriesIncludingDeleted, + long DefinitionId, + string Key, + long KeyFirstSeenStateVersion, + byte[] Value, + bool IsLocked, + bool IsDeleted, + long LastUpdatedStateVersion, + bool FilterOut, + bool IsLastCandidate, + IdBoundaryCursor? NextCursorExclusive + ); + + internal static async Task> ReadPages( + DbConnection dbConnection, + IDapperWrapper dapperWrapper, + GatewayApiSdk.Model.LedgerState ledgerState, + long[] entityIds, + QueryConfiguration queryConfiguration, + byte networkId, + CancellationToken token = default) + { + var queryParameters = new + { + entityIds = entityIds, + useCursor = queryConfiguration.Cursor is not null, + atLedgerState = ledgerState.StateVersion, + cursorStateVersion = queryConfiguration.Cursor?.StateVersionBoundary ?? 0, + cursorDefinitionId = queryConfiguration.Cursor?.IdBoundary ?? 0, + pageLimit = queryConfiguration.MaxPageSize, + definitionReadLimit = Math.Floor(queryConfiguration.MaxDefinitionsToRead / (decimal)entityIds.Length), + }; + + const string QueryText = @" +WITH vars AS ( + SELECT + unnest(@entityIds) AS entity_id, + CAST(@useCursor AS bool) AS use_cursor, + ROW(CAST(@cursorStateVersion AS bigint), CAST(@cursorDefinitionId AS bigint)) AS start_cursor_exclusive, + CAST(@atLedgerState AS bigint) AS at_ledger_state +), +definitions_with_cursor AS ( + SELECT + d.*, + (d.from_state_version, d.id) AS cursor + FROM entity_metadata_entry_definition d +) +SELECT + -- entity data + vars.entity_id, + + -- totals + COALESCE(entity_totals.total_entries_excluding_deleted, 0) AS TotalEntriesExcludingDeleted, + COALESCE(entity_totals.total_entries_including_deleted, 0) AS TotalEntriesIncludingDeleted, + + -- definitions + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.definition_id END AS DefinitionId, + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.key END AS Key, + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.key_first_seen_state_version END AS KeyFirstSeenStateVersion, + + -- history + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.value END AS Value, + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.is_locked END AS IsLocked, + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.is_deleted END AS IsDeleted, + CASE WHEN COALESCE(filter_out, TRUE) THEN NULL ELSE entries_per_entity.last_updated_state_version END AS LastUpdatedStateVersion, + + -- cursor + COALESCE(entries_per_entity.filter_out, TRUE) AS FilterOut, + COALESCE(entries_per_entity.is_last_candidate, TRUE) AS IsLastCandidate, + next_cursor_exclusive AS NextCursorExclusive +FROM vars + +-- Totals +LEFT JOIN LATERAL ( + SELECT + t.total_entries_excluding_deleted AS total_entries_excluding_deleted, + t.total_entries_including_deleted AS total_entries_including_deleted + FROM entity_metadata_totals_history t + WHERE t.entity_id = vars.entity_id AND t.from_state_version <= vars.at_ledger_state + ORDER BY t.from_state_version DESC + LIMIT 1 +) entity_totals ON TRUE + +-- entries_per_entity +LEFT JOIN LATERAL ( + SELECT + definitions.id as definition_id, + definitions.key, + definitions.key_first_seen_state_version, + definitions.is_last_candidate, + entries.*, + CASE WHEN (ROW_NUMBER() OVER (ORDER BY definitions.cursor DESC)) = @pageLimit OR definitions.is_last_candidate + THEN definitions.cursor + END AS next_cursor_exclusive + FROM ( + SELECT + d.id, + d.key, + d.cursor, + d.from_state_version AS key_first_seen_state_version, + (ROW_NUMBER() OVER (ORDER BY d.cursor DESC)) = @definitionReadLimit AS is_last_candidate + FROM definitions_with_cursor d + WHERE d.entity_id = vars.entity_id AND ((NOT vars.use_cursor) OR d.cursor < vars.start_cursor_exclusive) + ORDER BY d.cursor DESC + LIMIT @definitionReadLimit + ) definitions + INNER JOIN LATERAL ( + SELECT + h.from_state_version AS last_updated_state_version, + h.value, + h.is_locked, + h.is_deleted, + h.is_deleted AS filter_out + FROM entity_metadata_entry_history h + WHERE h.entity_metadata_entry_definition_id = definitions.id AND h.from_state_version <= vars.at_ledger_state + ORDER BY h.from_state_version DESC + LIMIT 1 + ) entries ON TRUE +WHERE entries.filter_out = FALSE OR definitions.is_last_candidate +ORDER BY definitions.cursor DESC +LIMIT @pageLimit +) entries_per_entity ON TRUE +;"; + + var queryResult = (await dapperWrapper.ToListAsync( + dbConnection, + QueryText, + queryParameters, + token)).ToList(); + + var result = queryResult + .GroupBy(r => r.EntityId) + .ToDictionary( + g => g.Key, + g => + { + var rows = g.ToList(); + var totalEntries = rows.First().TotalEntriesExcludingDeleted; + var elementWithCursor = rows.SingleOrDefault(x => x.NextCursorExclusive.HasValue); + + var items = rows + .Where(x => !x.FilterOut) + .Select( + x => + { + var value = ScryptoSborUtils.DecodeToGatewayMetadataItemValue(x.Value, networkId); + var programmaticJson = ScryptoSborUtils.DataToProgrammaticJson(x.Value, networkId); + var entityMetadataItemValue = new GatewayModel.EntityMetadataItemValue(x.Value.ToHex(), programmaticJson, value); + return new GatewayModel.EntityMetadataItem(x.Key, entityMetadataItemValue, x.IsLocked, x.LastUpdatedStateVersion); + }) + .ToList(); + + var nextCursor = items.Count < totalEntries && elementWithCursor.NextCursorExclusive != null + ? new GatewayModel.IdBoundaryCoursor( + elementWithCursor.NextCursorExclusive.Value.StateVersion, + elementWithCursor.NextCursorExclusive.Value.Id) + .ToCursorString() + : null; + + return new GatewayModel.EntityMetadataCollection( + totalEntries, + nextCursor, + items); + }); + + return result; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs index 66b228672..a9b96058a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs @@ -62,6 +62,7 @@ * permissions under this License. */ +using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.StandardMetadata; @@ -121,8 +122,8 @@ public async Task>> R return ImmutableDictionary>.Empty; } - var partiallyValidatedEntries = await _dapperWrapper.ToList( - _dbContext, + var partiallyValidatedEntries = await _dapperWrapper.ToListAsync( + _dbContext.Database.GetDbConnection(), @" WITH variables (entity_id) AS (SELECT UNNEST(@entityIds)), diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/ValueConverters/IdBoundaryCursorHandler.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/ValueConverters/IdBoundaryCursorHandler.cs new file mode 100644 index 000000000..63bfea2cc --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/ValueConverters/IdBoundaryCursorHandler.cs @@ -0,0 +1,24 @@ +using Dapper; +using RadixDlt.NetworkGateway.PostgresIntegration.Models.CustomTypes; +using System; +using System.Data; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.ValueConverters; + +public sealed class IdBoundaryCursorHandler : SqlMapper.TypeHandler +{ + public override void SetValue(IDbDataParameter parameter, IdBoundaryCursor value) + { + throw new NotImplementedException(); + } + + public override IdBoundaryCursor Parse(object value) + { + if (value is object[] cursor && cursor[0] is long && cursor[1] is long) + { + return new IdBoundaryCursor((long)cursor[0], (long)cursor[1]); + } + + throw new ArgumentException($"Not supported value type: {value.GetType()}"); + } +}