Skip to content

Commit

Permalink
Merge pull request #536 from radixdlt/get-tx-opt
Browse files Browse the repository at this point in the history
Optimized TransactionQuerier.GetTransactions not to fetch unnecessary data from database
  • Loading branch information
krzlabrdx authored Oct 11, 2023
2 parents 97e0ee7 + ebc1f63 commit 748f4f8
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 127 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
Release Date: _unreleased_

- Changed default configuration value of MaxPageSize for endpoints to 100. Validate if max page size is higher than DefaultPageSize.
- added new optIn `receipt_output` to `/stream/transactions`, and `/transaction/committed-details` endpoints. Temporarily set by default to true, to allow client's migration.
- Added new optIn `receipt_output` to `/stream/transactions`, and `/transaction/committed-details` endpoints. Temporarily set by default to true, to allow client's migration.
- Optimized `TransactionQuerier.GetTransactions` not to fetch unnecessary data from underlying database.

## 1.0.1 - Babylon
Release Date: 10.10.2023
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -508,12 +508,9 @@ private async Task<ExtendLedgerReport> ProcessTransactions(ReadWriteDbContext db
};

ledgerTransaction.StateVersion = stateVersion;
ledgerTransaction.LedgerHashes = new LedgerHashes
{
TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash,
ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash,
StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash,
};
ledgerTransaction.TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash;
ledgerTransaction.ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash;
ledgerTransaction.StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash;
ledgerTransaction.Epoch = summary.Epoch;
ledgerTransaction.RoundInEpoch = summary.RoundInEpoch;
ledgerTransaction.IndexInEpoch = summary.IndexInEpoch;
Expand All @@ -524,20 +521,14 @@ private async Task<ExtendLedgerReport> ProcessTransactions(ReadWriteDbContext db
ledgerTransaction.RoundTimestamp = summary.RoundTimestamp;
ledgerTransaction.CreatedTimestamp = summary.CreatedTimestamp;
ledgerTransaction.NormalizedRoundTimestamp = summary.NormalizedRoundTimestamp;
ledgerTransaction.EngineReceipt = new TransactionReceipt
{
StateUpdates = committedTransaction.Receipt.StateUpdates.ToJson(),
Status = committedTransaction.Receipt.Status.ToModel(),
FeeSummary = committedTransaction.Receipt.FeeSummary.ToJson(),
ErrorMessage = committedTransaction.Receipt.ErrorMessage,
Output = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null,
NextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson(),
CostingParameters = committedTransaction.Receipt.CostingParameters.ToJson(),
FeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson(),
FeeSource = committedTransaction.Receipt.FeeSource?.ToJson(),
Events = default!, // will be filled later on.
};

ledgerTransaction.ReceiptStateUpdates = committedTransaction.Receipt.StateUpdates.ToJson();
ledgerTransaction.ReceiptStatus = committedTransaction.Receipt.Status.ToModel();
ledgerTransaction.ReceiptFeeSummary = committedTransaction.Receipt.FeeSummary.ToJson();
ledgerTransaction.ReceiptErrorMessage = committedTransaction.Receipt.ErrorMessage;
ledgerTransaction.ReceiptOutput = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null;
ledgerTransaction.ReceiptNextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson();
ledgerTransaction.ReceiptCostingParameters = committedTransaction.Receipt.CostingParameters.ToJson();
ledgerTransaction.ReceiptFeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson();
ledgerTransactionsToAdd.Add(ledgerTransaction);

if (committedTransaction.Receipt.NextEpoch != null)
Expand Down Expand Up @@ -1175,16 +1166,13 @@ private async Task<ExtendLedgerReport> ProcessTransactions(ReadWriteDbContext db
var transaction = ledgerTransactionsToAdd.Single(x => x.StateVersion == stateVersion);

transaction.AffectedGlobalEntities = affectedGlobalEntities.ToArray();
transaction.EngineReceipt.Events = new ReceiptEvents
{
Emitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray(),
Names = events.Select(e => e.Type.Name).ToArray(),
Sbors = events.Select(e => e.Data.GetDataBytes()).ToArray(),
SchemaEntityIds = events.Select(e => referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray(),
SchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray(),
TypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray(),
SborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray(),
};
transaction.ReceiptEventEmitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray();
transaction.ReceiptEventNames = events.Select(e => e.Type.Name).ToArray();
transaction.ReceiptEventSbors = events.Select(e => e.Data.GetDataBytes()).ToArray();
transaction.ReceiptEventSchemaEntityIds = events.Select(e => referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray();
transaction.ReceiptEventSchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray();
transaction.ReceiptEventTypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray();
transaction.ReceiptEventSborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray();

ledgerTransactionMarkersToAdd.AddRange(affectedGlobalEntities.Select(affectedEntity => new AffectedGlobalEntityTransactionMarker
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,9 +221,9 @@ public async Task<int> CopyLedgerTransaction(ICollection<LedgerTransaction> enti

await writer.StartRowAsync(token);
await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token);
await writer.WriteAsync(lt.LedgerHashes.TransactionTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.LedgerHashes.ReceiptTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.LedgerHashes.StateTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.TransactionTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.ReceiptTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.StateTreeHash, NpgsqlDbType.Text, token);
await writer.WriteAsync(lt.Epoch, NpgsqlDbType.Bigint, token);
await writer.WriteAsync(lt.RoundInEpoch, NpgsqlDbType.Bigint, token);
await writer.WriteAsync(lt.IndexInEpoch, NpgsqlDbType.Bigint, token);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
* permissions under this License.
*/

using Microsoft.EntityFrameworkCore;
using RadixDlt.NetworkGateway.Abstractions;
using RadixDlt.NetworkGateway.Abstractions.Model;
using RadixDlt.NetworkGateway.Abstractions.Numerics;
Expand All @@ -83,6 +82,8 @@ internal record TransactionReceiptEvent(string Name, string Emitter, byte[] Data
[Table("ledger_transactions")]
internal abstract class LedgerTransaction
{
private TransactionReceipt? _engineReceipt;

[Key]
[DatabaseGenerated(DatabaseGeneratedOption.None)]
[Column("state_version")]
Expand Down Expand Up @@ -131,114 +132,69 @@ internal abstract class LedgerTransaction
[Column("normalized_round_timestamp")]
public DateTime NormalizedRoundTimestamp { get; set; }

public TransactionReceipt EngineReceipt { get; set; }
[Column("receipt_status")]
public LedgerTransactionStatus ReceiptStatus { get; set; }

public LedgerHashes LedgerHashes { get; set; }
}
[Column("receipt_fee_summary", TypeName = "jsonb")]
public string ReceiptFeeSummary { get; set; }

[Owned]
internal class LedgerHashes
{
[Column("transaction_tree_hash")]
public string TransactionTreeHash { get; set; }
[Column("receipt_state_updates", TypeName = "jsonb")]
public string ReceiptStateUpdates { get; set; }

[Column("receipt_tree_hash")]
public string ReceiptTreeHash { get; set; }
[Column("receipt_costing_parameters", TypeName = "jsonb")]
public string ReceiptCostingParameters { get; set; }

[Column("state_tree_hash")]
public string StateTreeHash { get; set; }
}
[Column("receipt_fee_source", TypeName = "jsonb")]
public string? ReceiptFeeSource { get; set; }

[Column("receipt_fee_destination", TypeName = "jsonb")]
public string? ReceiptFeeDestination { get; set; }

[Column("receipt_next_epoch", TypeName = "jsonb")]
public string? ReceiptNextEpoch { get; set; }

[Column("receipt_output", TypeName = "jsonb")]
public string? ReceiptOutput { get; set; }

[Column("receipt_error_message")]
public string? ReceiptErrorMessage { get; set; }

[Owned]
internal class ReceiptEvents
{
[Column("receipt_event_emitters", TypeName = "jsonb[]")]
public string[] Emitters { get; set; }
public string[] ReceiptEventEmitters { get; set; }

[Column("receipt_event_names", TypeName = "text[]")]
public string[] Names { get; set; }
public string[] ReceiptEventNames { get; set; }

[Column("receipt_event_sbors")]
public byte[][] Sbors { get; set; }
public byte[][] ReceiptEventSbors { get; set; }

[Column("receipt_event_schema_entity_ids")]
public long[] SchemaEntityIds { get; set; }
public long[] ReceiptEventSchemaEntityIds { get; set; }

[Column("receipt_event_schema_hashes")]
public byte[][] SchemaHashes { get; set; }
public byte[][] ReceiptEventSchemaHashes { get; set; }

[Column("receipt_event_type_indexes")]
public long[] TypeIndexes { get; set; }
public long[] ReceiptEventTypeIndexes { get; set; }

[Column("receipt_event_sbor_type_kinds")]
public SborTypeKind[] SborTypeKinds { get; set; }
public SborTypeKind[] ReceiptEventSborTypeKinds { get; set; }

public List<TransactionReceiptEventLookup> GetEventLookups()
{
var result = new List<TransactionReceiptEventLookup>();
[Column("transaction_tree_hash")]
public string TransactionTreeHash { get; set; }

for (var i = 0; i < Sbors.Length; ++i)
{
result.Add(new TransactionReceiptEventLookup(SchemaEntityIds[i], SchemaHashes[i]));
}
[Column("receipt_tree_hash")]
public string ReceiptTreeHash { get; set; }

return result;
}
[Column("state_tree_hash")]
public string StateTreeHash { get; set; }

public List<TransactionReceiptEvent> GetEvents()
public TransactionReceipt EngineReceipt
{
var result = new List<TransactionReceiptEvent>();

for (var i = 0; i < Sbors.Length; ++i)
{
var eventData = Sbors[i];
var entityId = SchemaEntityIds[i];
var schemaHash = SchemaHashes[i];
var index = TypeIndexes[i];
var typeKind = SborTypeKinds[i];
var emitter = Emitters[i];
var name = Names[i];

result.Add(new TransactionReceiptEvent(name, emitter, eventData, entityId, schemaHash, index, typeKind));
}

return result;
get => _engineReceipt ??= new TransactionReceipt(this);
}
}

[Owned]
internal class TransactionReceipt
{
[Column("receipt_status")]
public LedgerTransactionStatus Status { get; set; }

[Column("receipt_fee_summary", TypeName = "jsonb")]
public string FeeSummary { get; set; }

[Column("receipt_state_updates", TypeName = "jsonb")]
public string StateUpdates { get; set; }

[Column("receipt_costing_parameters", TypeName = "jsonb")]
public string CostingParameters { get; set; }

[Column("receipt_fee_source", TypeName = "jsonb")]
public string? FeeSource { get; set; }

[Column("receipt_fee_destination", TypeName = "jsonb")]
public string? FeeDestination { get; set; }

[Column("receipt_next_epoch", TypeName = "jsonb")]
public string? NextEpoch { get; set; }

[Column("receipt_output", TypeName = "jsonb")]
public string? Output { get; set; }

[Column("receipt_error_message")]
public string? ErrorMessage { get; set; }

public ReceiptEvents Events { get; set; }
}

internal class GenesisLedgerTransaction : LedgerTransaction
{
}
Expand Down Expand Up @@ -278,3 +234,91 @@ internal class UserLedgerTransaction : LedgerTransaction
internal class RoundUpdateLedgerTransaction : LedgerTransaction
{
}

internal class TransactionReceipt
{
private readonly LedgerTransaction _ledgerTransaction;

public TransactionReceipt(LedgerTransaction ledgerTransaction)
{
_ledgerTransaction = ledgerTransaction;

Events = new ReceiptEvents(ledgerTransaction);
}

public string? ErrorMessage => _ledgerTransaction.ReceiptErrorMessage;

public LedgerTransactionStatus Status => _ledgerTransaction.ReceiptStatus;

public string? Output => _ledgerTransaction.ReceiptOutput;

public string FeeSummary => _ledgerTransaction.ReceiptFeeSummary;

public string? FeeDestination => _ledgerTransaction.ReceiptFeeDestination;

public string? FeeSource => _ledgerTransaction.ReceiptFeeSource;

public string? NextEpoch => _ledgerTransaction.ReceiptNextEpoch;

public string CostingParameters => _ledgerTransaction.ReceiptCostingParameters;

public string StateUpdates => _ledgerTransaction.ReceiptStateUpdates;

public ReceiptEvents Events { get; }
}

internal class ReceiptEvents
{
private LedgerTransaction _ledgerTransaction;

public ReceiptEvents(LedgerTransaction ledgerTransaction)
{
_ledgerTransaction = ledgerTransaction;
}

public string[] Emitters => _ledgerTransaction.ReceiptEventEmitters;

public string[] Names => _ledgerTransaction.ReceiptEventNames;

public byte[][] Sbors => _ledgerTransaction.ReceiptEventSbors;

public long[] SchemaEntityIds => _ledgerTransaction.ReceiptEventSchemaEntityIds;

public byte[][] SchemaHashes => _ledgerTransaction.ReceiptEventSchemaHashes;

public long[] TypeIndexes => _ledgerTransaction.ReceiptEventTypeIndexes;

public SborTypeKind[] SborTypeKinds => _ledgerTransaction.ReceiptEventSborTypeKinds;

public List<TransactionReceiptEventLookup> GetEventLookups()
{
var result = new List<TransactionReceiptEventLookup>();

for (var i = 0; i < _ledgerTransaction.ReceiptEventSbors.Length; ++i)
{
result.Add(new TransactionReceiptEventLookup(_ledgerTransaction.ReceiptEventSchemaEntityIds[i], _ledgerTransaction.ReceiptEventSchemaHashes[i]));
}

return result;
}

public List<TransactionReceiptEvent> GetEvents()
{
var result = new List<TransactionReceiptEvent>();

for (var i = 0; i < _ledgerTransaction.ReceiptEventSbors.Length; ++i)
{
var eventData = _ledgerTransaction.ReceiptEventSbors[i];
var entityId = _ledgerTransaction.ReceiptEventSchemaEntityIds[i];
var schemaHash = _ledgerTransaction.ReceiptEventSchemaHashes[i];
var index = _ledgerTransaction.ReceiptEventTypeIndexes[i];
var typeKind = _ledgerTransaction.ReceiptEventSborTypeKinds[i];
var emitter = _ledgerTransaction.ReceiptEventEmitters[i];
var name = _ledgerTransaction.ReceiptEventNames[i];

result.Add(new TransactionReceiptEvent(name, emitter, eventData, entityId, schemaHash, index, typeKind));
}

return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,9 @@ public CommittedStateIdentifiersReader(IDbContextFactory<ReadWriteDbContext> dbC
.Select(e => new
{
e.StateVersion,
e.LedgerHashes,
e.StateTreeHash,
e.TransactionTreeHash,
e.ReceiptTreeHash,
})
.AsNoTracking()
.AnnotateMetricName()
Expand All @@ -102,8 +104,8 @@ public CommittedStateIdentifiersReader(IDbContextFactory<ReadWriteDbContext> dbC

return new GatewayModel.CommittedStateIdentifiers(
transaction.StateVersion,
transaction.LedgerHashes.StateTreeHash,
transaction.LedgerHashes.TransactionTreeHash,
transaction.LedgerHashes.ReceiptTreeHash);
transaction.StateTreeHash,
transaction.TransactionTreeHash,
transaction.ReceiptTreeHash);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ public async Task<TransactionSummary> GetTopOfLedger(CancellationToken token)
.Select(lt => new
{
lt.StateVersion,
lt.LedgerHashes,
lt.StateTreeHash,
lt.TransactionTreeHash,
lt.ReceiptTreeHash,
lt.RoundTimestamp,
lt.NormalizedRoundTimestamp,
lt.CreatedTimestamp,
Expand All @@ -118,9 +120,9 @@ public async Task<TransactionSummary> GetTopOfLedger(CancellationToken token)
? PreGenesisTransactionSummary()
: new TransactionSummary(
StateVersion: lastTransaction.StateVersion,
TransactionTreeHash: lastTransaction.LedgerHashes.TransactionTreeHash,
ReceiptTreeHash: lastTransaction.LedgerHashes.ReceiptTreeHash,
StateTreeHash: lastTransaction.LedgerHashes.StateTreeHash,
TransactionTreeHash: lastTransaction.TransactionTreeHash,
ReceiptTreeHash: lastTransaction.ReceiptTreeHash,
StateTreeHash: lastTransaction.StateTreeHash,
RoundTimestamp: lastTransaction.RoundTimestamp,
NormalizedRoundTimestamp: lastTransaction.NormalizedRoundTimestamp,
CreatedTimestamp: lastTransaction.CreatedTimestamp,
Expand Down
Loading

0 comments on commit 748f4f8

Please sign in to comment.