Do not index manifest for failed transactions (#799)

* index manifest addresses only for successful transactions.

CHANGELOG.md

@@ -3,6 +3,13 @@ Release built: _not released yet_
 
 > [!CAUTION]
 > **Breaking Changes:**
+> - Manifest addresses are no longer indexed in the `/stream/transactions` endpoint for failed transactions. Affected filters:
+>   - `manifest_accounts_withdrawn_from_filter`
+>   - `manifest_accounts_deposited_into_filter`
+>   - `manifest_badges_presented_filter`
+>   - `manifest_resources_filter`
+>   - `accounts_with_manifest_owner_method_calls`
+>   - `accounts_without_manifest_owner_method_calls`
 > - Changed ordering of entity metadata. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending. Affected endpoints:
 >   - `/state/entity/metadata`
 >   - `/state/entity/page/metadata`

babylon-gateway.sln.DotSettings

@@ -1,4 +1,5 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Bech/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=mempool/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Querier/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=radixfoundation/@EntryIndexedValue">True</s:Boolean>

src/RadixDlt.NetworkGateway.Abstractions/EntityAddress.cs

@@ -62,6 +62,7 @@
  * permissions under this License.
  */
 
+using RadixDlt.NetworkGateway.Abstractions.Network;
 using System.Diagnostics;
 
 namespace RadixDlt.NetworkGateway.Abstractions;
@@ -82,6 +83,8 @@ private EntityAddress(string address)
 
     public bool IsAccount => _address.StartsWith("account_");
 
+    public bool IsIdentity => _address.StartsWith("identity_");
+
     public bool IsResource => _address.StartsWith("resource_");
 
     public static implicit operator string(EntityAddress ra) => ra._address;
@@ -92,4 +95,6 @@ public override string ToString()
     {
         return _address;
     }
+
+    public DecodedRadixAddress Decode() => RadixAddressCodec.DecodeEntityAddress(_address);
 }

src/RadixDlt.NetworkGateway.Abstractions/Network/DecodedRadixAddress.cs

@@ -0,0 +1,92 @@
+/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands).
+ *
+ * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this
+ * file except in compliance with the License. You may obtain a copy of the License at:
+ *
+ * radixfoundation.org/licenses/LICENSE-v1
+ *
+ * The Licensor hereby grants permission for the Canonical version of the Work to be
+ * published, distributed and used under or by reference to the Licensor’s trademark
+ * Radix ® and use of any unregistered trade names, logos or get-up.
+ *
+ * The Licensor provides the Work (and each Contributor provides its Contributions) on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+ * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT,
+ * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create
+ * a distributed ledger it is your responsibility to test and validate the code, together
+ * with all logic and performance of that code under all foreseeable scenarios.
+ *
+ * The Licensor does not make or purport to make and hereby excludes liability for all
+ * and any representation, warranty or undertaking in any form whatsoever, whether express
+ * or implied, to any entity or person, including any representation, warranty or
+ * undertaking, as to the functionality security use, value or other characteristics of
+ * any distributed ledger nor in respect the functioning or value of any tokens which may
+ * be created stored or transferred using the Work. The Licensor does not warrant that the
+ * Work or any use of the Work complies with any law or regulation in any territory where
+ * it may be implemented or used or that it will be appropriate for any specific purpose.
+ *
+ * Neither the licensor nor any current or former employees, officers, directors, partners,
+ * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor
+ * shall be liable for any direct or indirect, special, incidental, consequential or other
+ * losses of any kind, in tort, contract or otherwise (including but not limited to loss
+ * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss
+ * of any economic or other opportunity of whatsoever nature or howsoever arising), arising
+ * out of or in connection with (without limitation of any use, misuse, of any ledger system
+ * or use made or its functionality or any performance or operation of any code or protocol
+ * caused by bugs or programming or logic errors or otherwise);
+ *
+ * A. any offer, purchase, holding, use, sale, exchange or transmission of any
+ * cryptographic keys, tokens or assets created, exchanged, stored or arising from any
+ * interaction with the Work;
+ *
+ * B. any failure in a transmission or loss of any token or assets keys or other digital
+ * artefacts due to errors in transmission;
+ *
+ * C. bugs, hacks, logic errors or faults in the Work or any communication;
+ *
+ * D. system software or apparatus including but not limited to losses caused by errors
+ * in holding or transmitting tokens by any third-party;
+ *
+ * E. breaches or failure of security including hacker attacks, loss or disclosure of
+ * password, loss of private key, unauthorised use or misuse of such passwords or keys;
+ *
+ * F. any losses including loss of anticipated savings or other benefits resulting from
+ * use of the Work or any changes to the Work (however implemented).
+ *
+ * You are solely responsible for; testing, validating and evaluation of all operation
+ * logic, functionality, security and appropriateness of using the Work for any commercial
+ * or non-commercial purpose and for any reproduction or redistribution by You of the
+ * Work. You assume all risks associated with Your use of the Work and the exercise of
+ * permissions under this License.
+ */
+
+namespace RadixDlt.NetworkGateway.Abstractions.Network;
+
+public sealed record DecodedRadixAddress(string Hrp, byte[] Data, Bech32Codec.Variant Variant)
+{
+    private const byte Secp256kPreAllocatedAccountPrefix = 209;
+    private const byte Secp256kPreAllocatedIdentityPrefix = 210;
+    private const byte Ed25519PreAllocatedAccountPrefix = 81;
+    private const byte Ed25519PreAllocatedIdentityPrefix = 82;
+
+    public bool IsPreAllocated() => IsPreAllocatedAccountAddress() || IsPreAllocatedIdentityAddress();
+
+    public bool IsSecp256k() => DiscriminatorByte is Secp256kPreAllocatedAccountPrefix or Secp256kPreAllocatedIdentityPrefix;
+
+    public bool IsEd25519() => DiscriminatorByte is Ed25519PreAllocatedAccountPrefix or Ed25519PreAllocatedIdentityPrefix;
+
+    public bool IsPreAllocatedAccountAddress() => DiscriminatorByte is Secp256kPreAllocatedAccountPrefix or Ed25519PreAllocatedAccountPrefix;
+
+    public bool IsPreAllocatedIdentityAddress() => DiscriminatorByte is Secp256kPreAllocatedIdentityPrefix or Ed25519PreAllocatedIdentityPrefix;
+
+    public byte DiscriminatorByte => Data[0];
+
+    public byte[] AddressBytes => Data[1..];
+
+    public override string ToString()
+    {
+        return RadixAddressCodec.Encode(Hrp, Data);
+    }
+}

src/RadixDlt.NetworkGateway.Abstractions/Network/RadixAddressCodec.cs

@@ -62,34 +62,29 @@
  * permissions under this License.
  */
 
-// ReSharper disable CommentTypo
-// ReSharper disable StringLiteralTypo
-// ReSharper disable IdentifierTypo
-/* The above is a fix for ReShaper not liking the work "Bech" */
-
 using System;
 
 namespace RadixDlt.NetworkGateway.Abstractions.Network;
 
-public sealed record DecodedRadixAddress(string Hrp, byte[] Data, Bech32Codec.Variant Variant)
-{
-    public byte DiscriminatorByte => Data[0];
-
-    public byte[] AddressBytes => Data[1..];
-
-    public override string ToString()
-    {
-        return RadixAddressCodec.Encode(Hrp, Data);
-    }
-}
-
 public static class RadixAddressCodec
 {
     public static string Encode(string hrp, ReadOnlySpan<byte> addressData)
     {
         return Bech32Codec.Encode(hrp, EncodeAddressDataInBase32(addressData), Bech32Codec.Variant.Bech32M);
     }
 
+    public static DecodedRadixAddress DecodeEntityAddress(string encoded)
+    {
+        var decoded = Decode(encoded);
+
+        if (decoded.Data.Length != 30)
+        {
+            throw new AddressException($"Entity address is expected to be 30 bytes in length. But was {decoded.Data.Length}");
+        }
+
+        return decoded;
+    }
+
     public static DecodedRadixAddress Decode(string encoded)
     {
         var (hrp, rawBase32Data, variant) = Bech32Codec.Decode(encoded);
@@ -102,7 +97,7 @@ public static DecodedRadixAddress Decode(string encoded)
 
         if (variant != Bech32Codec.Variant.Bech32M)
         {
-            throw new AddressException("Only Bech32M addresses are supported");
+            throw new AddressException($"Only Bech32M addresses are supported, decoded variant: {variant}");
         }
 
         return new DecodedRadixAddress(hrp, addressData, variant);

src/RadixDlt.NetworkGateway.GatewayApi/Validators/RadixAddressValidator.cs

@@ -63,6 +63,7 @@
  */
 
 using FluentValidation;
+using RadixDlt.NetworkGateway.Abstractions;
 using RadixDlt.NetworkGateway.Abstractions.Network;
 using System;
 

src/RadixDlt.NetworkGateway.PostgresIntegration/IoC/GatewayApiBuilderExtensions.cs

@@ -106,7 +106,7 @@ public static GatewayApiBuilder AddPostgresPersistenceCore(this GatewayApiBuilde
             .AddScoped<IValidatorQuerier, ValidatorQuerier>()
             .AddScoped<IPackageQuerier, PackageQuerier>()
             .AddScoped<IEntityQuerier, EntityQuerier>()
-            .AddScoped<IVirtualEntityDataProvider, VirtualEntityDataProvider>()
+            .AddScoped<IPreAllocatedEntityDataProvider, PreAllocatedEntityDataProvider>()
             .AddScoped<ISubmissionTrackingService, SubmissionTrackingService>()
             .AddScoped<IDapperWrapper, DapperWrapper>()
             .AddSingleton<MetricsInterceptor>();

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ManifestAddressesExtractor.cs

@@ -114,22 +114,36 @@ public static ManifestAddresses ExtractAddresses(ToolkitModel.TransactionManifes
         var accountsDepositedInto = manifestSummary.accountsDepositedInto.Select(x => (EntityAddress)x.AddressString()).ToList();
         var identitiesRequiringAuth = manifestSummary.identitiesRequiringAuth.Select(x => (EntityAddress)x.AddressString()).ToList();
 
-        var packageAddresses = allAddresses.Where(x => x.Key == ToolkitModel.EntityType.GlobalPackage).SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString())).ToList();
+        var packageAddresses = allAddresses
+            .Where(x => x.Key == ToolkitModel.EntityType.GlobalPackage)
+            .SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString()))
+            .ToList();
+
         var componentAddresses = allAddresses
             .Where(x => x.Key is ToolkitModel.EntityType.GlobalGenericComponent or ToolkitModel.EntityType.InternalGenericComponent)
             .SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString()))
             .ToList();
+
         var resourceAddresses = allAddresses
             .Where(x => x.Key is ToolkitModel.EntityType.GlobalFungibleResourceManager or ToolkitModel.EntityType.GlobalNonFungibleResourceManager)
             .SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString()))
             .ToList();
+
         var accountAddresses = allAddresses
-            .Where(x => x.Key is ToolkitModel.EntityType.GlobalAccount or ToolkitModel.EntityType.GlobalVirtualEd25519Account
-                or ToolkitModel.EntityType.GlobalVirtualSecp256k1Account)
+            .Where(
+                x => x.Key
+                    is ToolkitModel.EntityType.GlobalAccount
+                    or ToolkitModel.EntityType.GlobalVirtualEd25519Account
+                    or ToolkitModel.EntityType.GlobalVirtualSecp256k1Account)
             .SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString()))
             .ToList();
+
         var identityAddresses = allAddresses
-            .Where(x => x.Key is ToolkitModel.EntityType.GlobalIdentity or ToolkitModel.EntityType.GlobalVirtualEd25519Identity or ToolkitModel.EntityType.GlobalVirtualSecp256k1Identity)
+            .Where(
+                x => x.Key
+                    is ToolkitModel.EntityType.GlobalIdentity
+                    or ToolkitModel.EntityType.GlobalVirtualEd25519Identity
+                    or ToolkitModel.EntityType.GlobalVirtualSecp256k1Identity)
             .SelectMany(x => x.Value.Select(y => (EntityAddress)y.AddressString()))
             .ToList();
 
@@ -159,18 +173,18 @@ private static List<PresentedProof> ExtractProofs(Dictionary<string, ToolkitMode
                 switch (proof)
                 {
                     case ToolkitModel.ResourceSpecifier.Amount fungibleProof:
-                        {
-                            var resourceAddress = (EntityAddress)fungibleProof.resourceAddress.AddressString();
-                            mapped.Add(new PresentedProof(accountAddress, resourceAddress));
-                            break;
-                        }
+                    {
+                        var resourceAddress = (EntityAddress)fungibleProof.resourceAddress.AddressString();
+                        mapped.Add(new PresentedProof(accountAddress, resourceAddress));
+                        break;
+                    }
 
                     case ToolkitModel.ResourceSpecifier.Ids nonFungibleProof:
-                        {
-                            var resourceAddress = (EntityAddress)nonFungibleProof.resourceAddress.AddressString();
-                            mapped.Add(new PresentedProof(accountAddress, resourceAddress));
-                            break;
-                        }
+                    {
+                        var resourceAddress = (EntityAddress)nonFungibleProof.resourceAddress.AddressString();
+                        mapped.Add(new PresentedProof(accountAddress, resourceAddress));
+                        break;
+                    }
 
                     default:
                         throw new UnreachableException($"Unexpected proof type {proof.GetType()}");