Merge branch 'main' into dependabot/npm_and_yarn/discord-bot/express-…

…4.19.2

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "explorer-api",
     "description": "API for Tangle Explorer",
-    "version": "3.3.6",
+    "version": "3.3.8-rc.2",
     "author": "Martyn Janes <[email protected]>",
     "repository": {
         "type": "git",

api/src/models/api/INetworkGetResponse.ts

@@ -75,4 +75,8 @@ interface INetworkView {
      * If Identity Resolver tool should be supported.
      */
     identityResolverEnabled?: boolean;
+    /**
+     * Max results for API requests (only used in legacy for now).
+     */
+    apiMaxResults?: number;
 }

api/src/models/api/stardust/IAssociationsResponse.ts

@@ -2,6 +2,7 @@ import { IResponse } from "../IResponse";
 
 export enum AssociationType {
     BASIC_ADDRESS,
+    BASIC_ADDRESS_EXPIRED,
     BASIC_SENDER,
     BASIC_EXPIRATION_RETURN,
     BASIC_STORAGE_RETURN,
@@ -12,6 +13,7 @@ export enum AssociationType {
     ALIAS_ID,
     FOUNDRY_ALIAS,
     NFT_ADDRESS,
+    NFT_ADDRESS_EXPIRED,
     NFT_STORAGE_RETURN,
     NFT_EXPIRATION_RETURN,
     NFT_ISSUER,

api/src/models/db/INetwork.ts

@@ -172,4 +172,9 @@ export interface INetwork {
      * If Identity Resolver tool should be supported.
      */
     identityResolverEnabled?: boolean;
+
+    /**
+     * Max results for API requests (only used in legacy for now).
+     */
+    apiMaxResults?: number;
 }

api/src/models/db/networkType.ts

@@ -5,10 +5,23 @@ export const MAINNET = "mainnet";
 const DEVNET = "devnet";
 export const SHIMMER = "shimmer";
 const TESTNET = "testnet";
+const IOTA_TESTNET = "iota-testnet";
+const SHIMMER_TESTNET = "shimmer-testnet";
 const ALPHANET = "alphanet";
 const CUSTOM = "custom";
 
-const networkTypes = [LEGACY_MAINNET, CHRYSALIS_MAINNET, MAINNET, DEVNET, SHIMMER, TESTNET, ALPHANET, CUSTOM] as const;
+const networkTypes = [
+    LEGACY_MAINNET,
+    CHRYSALIS_MAINNET,
+    MAINNET,
+    DEVNET,
+    SHIMMER,
+    TESTNET,
+    IOTA_TESTNET,
+    SHIMMER_TESTNET,
+    ALPHANET,
+    CUSTOM,
+] as const;
 
 /**
  * The network type.

api/src/routes/networks/get.ts

@@ -50,6 +50,7 @@ export async function get(_: IConfiguration): Promise<INetworkGetResponse> {
                     circulatingSupply: circulatingSupplyFromSupplyTracker ?? n.circulatingSupply,
                     identityResolverEnabled: n.identityResolverEnabled,
                     tokenRegistryEndpoint: n.tokenRegistryEndpoint,
+                    apiMaxResults: n.apiMaxResults,
                 };
             }),
     };

api/src/routes/stardust/output/associated/post.ts

@@ -1,7 +1,7 @@
 import { ServiceFactory } from "../../../../factories/serviceFactory";
 import { IAssociationsRequest } from "../../../../models/api/stardust/IAssociationsRequest";
 import { IAssociationsRequestBody } from "../../../../models/api/stardust/IAssociationsRequestBody";
-import { IAssociation, IAssociationsResponse } from "../../../../models/api/stardust/IAssociationsResponse";
+import { IAssociationsResponse } from "../../../../models/api/stardust/IAssociationsResponse";
 import { IConfiguration } from "../../../../models/configuration/IConfiguration";
 import { STARDUST } from "../../../../models/db/protocolVersion";
 import { NetworkService } from "../../../../services/networkService";
@@ -33,12 +33,7 @@ export async function post(
 
     const helper = new AssociatedOutputsHelper(networkConfig, body.addressDetails);
     await helper.fetch();
-    const result = helper.associationToOutputIds;
-
-    const associations: IAssociation[] = [];
-    for (const [type, outputIds] of result.entries()) {
-        associations.push({ type, outputIds: outputIds.reverse() });
-    }
+    const associations = helper.getAssociations();
 
     return {
         associations,

api/src/services/stardust/stardustApiService.ts

@@ -249,18 +249,21 @@ export class StardustApiService {
     public async basicOutputDetailsByAddress(addressBech32: string): Promise<IAddressDetailsResponse> {
         let cursor: string | undefined;
         let outputIds: string[] = [];
+        const expiredIds = await this.basicExpiredOutputIdsByAddress(addressBech32);
+        const notClaimedIds = await this.basicNotClaimedOutputIdsByAddress(addressBech32);
 
         do {
             try {
                 const outputIdsResponse = await this.client.basicOutputIds([{ address: addressBech32 }, { cursor: cursor ?? "" }]);
 
-                outputIds = outputIds.concat(outputIdsResponse.items);
+                outputIds = outputIds.concat(outputIdsResponse.items.filter((id) => !expiredIds.includes(id)));
                 cursor = outputIdsResponse.cursor;
             } catch (e) {
                 logger.error(`Fetching basic output ids failed. Cause: ${e}`);
             }
         } while (cursor);
 
+        outputIds = outputIds.concat(notClaimedIds);
         const outputResponses = await this.outputsDetails(outputIds);
 
         return {
@@ -303,18 +306,21 @@ export class StardustApiService {
     public async nftOutputDetailsByAddress(addressBech32: string): Promise<IAddressDetailsResponse> {
         let cursor: string | undefined;
         let outputIds: string[] = [];
+        const expiredIds = await this.nftExpiredOutputIdsByAddress(addressBech32);
+        const notClaimedIds = await this.nftNotClaimedOutputIdsByAddress(addressBech32);
 
         do {
             try {
                 const outputIdsResponse = await this.client.nftOutputIds([{ address: addressBech32 }, { cursor: cursor ?? "" }]);
 
-                outputIds = outputIds.concat(outputIdsResponse.items);
+                outputIds = outputIds.concat(outputIdsResponse.items.filter((id) => !expiredIds.includes(id)));
                 cursor = outputIdsResponse.cursor;
             } catch (e) {
                 logger.error(`Fetching nft output ids failed. Cause: ${e}`);
             }
         } while (cursor);
 
+        outputIds = outputIds.concat(notClaimedIds);
         const outputResponses = await this.outputsDetails(outputIds);
         return {
             outputs: outputResponses,
@@ -499,6 +505,116 @@ export class StardustApiService {
         return new SearchExecutor(this.network, new SearchQueryBuilder(query, this.network.bechHrp).build()).run();
     }
 
+    /**
+     * Get the expired basic output ids for an address (outputs no longer owned by the address but by the expirationReturnAddress).
+     * @param addressBech32 The address in bech32 format.
+     * @returns The basic output ids.
+     */
+    private async basicExpiredOutputIdsByAddress(addressBech32: string): Promise<string[]> {
+        let cursor: string | undefined;
+        let outputIds: string[] = [];
+        const currentTimestamp = Math.floor(Date.now() / 1000);
+        do {
+            try {
+                const outputIdsResponse = await this.client.basicOutputIds([
+                    { address: addressBech32 },
+                    { expiresBefore: currentTimestamp },
+                    { cursor: cursor ?? "" },
+                ]);
+
+                outputIds = outputIds.concat(outputIdsResponse.items);
+                cursor = outputIdsResponse.cursor;
+            } catch (e) {
+                logger.error(`Fetching expired basic output ids failed. Cause: ${e}`);
+            }
+        } while (cursor);
+
+        return outputIds;
+    }
+
+    /**
+     * Get the expired ntf output ids for an address (outputs no longer owned by the address but by the expirationReturnAddress).
+     * @param addressBech32 The address in bech32 format.
+     * @returns The nft output ids.
+     */
+    private async nftExpiredOutputIdsByAddress(addressBech32: string): Promise<string[]> {
+        let cursor: string | undefined;
+        let outputIds: string[] = [];
+        const currentTimestamp = Math.floor(Date.now() / 1000);
+        do {
+            try {
+                const outputIdsResponse = await this.client.nftOutputIds([
+                    { address: addressBech32 },
+                    { expiresBefore: currentTimestamp },
+                    { cursor: cursor ?? "" },
+                ]);
+
+                outputIds = outputIds.concat(outputIdsResponse.items);
+                cursor = outputIdsResponse.cursor;
+            } catch (e) {
+                logger.error(`Fetching expired nft output ids failed. Cause: ${e}`);
+            }
+        } while (cursor);
+
+        return outputIds;
+    }
+
+    /**
+     * Get the not claimed basic output ids for an address (outputs owned by the expirationReturnAddress).
+     * @param expirationReturnAddress The address in bech32 format.
+     * @returns The nft output ids.
+     */
+    private async basicNotClaimedOutputIdsByAddress(expirationReturnAddress: string): Promise<string[]> {
+        let cursor: string | undefined;
+        let outputIds: string[] = [];
+        const currentTimestamp = Math.floor(Date.now() / 1000);
+
+        do {
+            try {
+                const outputIdsResponse = await this.client.basicOutputIds([
+                    { expirationReturnAddress },
+                    { expiresBefore: currentTimestamp },
+                    { cursor: cursor ?? "" },
+                ]);
+
+                outputIds = outputIds.concat(outputIdsResponse.items);
+                cursor = outputIdsResponse.cursor;
+            } catch (e) {
+                logger.error(`Fetching not claimed nft output ids failed. Cause: ${e}`);
+            }
+        } while (cursor);
+
+        return outputIds;
+    }
+
+    /**
+     * Get the not claimed ntf output ids for an address (outputs owned by the expirationReturnAddress).
+     * @param expirationReturnAddress The address in bech32 format.
+     * @returns The nft output ids.
+     */
+    private async nftNotClaimedOutputIdsByAddress(expirationReturnAddress: string): Promise<string[]> {
+        let cursor: string | undefined;
+        let outputIds: string[] = [];
+        const currentTimestamp = Math.floor(Date.now() / 1000);
+
+        do {
+            try {
+                const outputIdsResponse = await this.client.nftOutputIds([
+                    { expirationReturnAddress },
+                    { expiresBefore: currentTimestamp },
+                    { cursor: cursor ?? "" },
+                ]);
+
+                outputIds = outputIds.concat(outputIdsResponse.items);
+                cursor = outputIdsResponse.cursor;
+            } catch (e) {
+                logger.error(`Fetching not claimed nft output ids failed. Cause: ${e}`);
+            }
+        } while (cursor);
+
+        return outputIds;
+    }
+
     /**
      * Extension method which provides request methods for plugins.
      * @param basePluginPath The base path for the plugin eg indexer/v1/ .

api/src/utils/legacy/legacyTangleHelper.ts

@@ -41,17 +41,22 @@ export class LegacyTangleHelper {
         try {
             const client = new LegacyClient(network.provider, network.user, network.password);
 
+            let maxResults = limit || network.apiMaxResults;
+            if (maxResults > network.apiMaxResults) {
+                maxResults = network.apiMaxResults;
+            }
+
             const response = await client.findTransactions({
                 ...findReq,
-                maxresults: 5000,
+                maxresults: maxResults,
             });
 
             if (response?.txHashes && response.txHashes.length > 0) {
                 hashes = response.txHashes;
                 cursor.node = hashes.length;
 
                 if (limit === undefined) {
-                    cursor.hasMore = hashes.length === 5000;
+                    cursor.hasMore = hashes.length === maxResults;
                 } else {
                     cursor.hasMore = hashes.length > limit;
                     if (hashes.length > limit) {

api/src/utils/stardust/associatedOutputsHelper.ts

@@ -8,7 +8,7 @@ import {
     AddressType,
 } from "@iota/sdk";
 import { ServiceFactory } from "../../factories/serviceFactory";
-import { AssociationType } from "../../models/api/stardust/IAssociationsResponse";
+import { AssociationType, IAssociation } from "../../models/api/stardust/IAssociationsResponse";
 import { IBech32AddressDetails } from "../../models/api/stardust/IBech32AddressDetails";
 import { INetwork } from "../../models/db/INetwork";
 
@@ -43,6 +43,15 @@ export class AssociatedOutputsHelper {
             ),
         );
 
+        promises.push(
+            // Basic output -> owner address expired outputs
+            this.fetchAssociatedOutputIds<QueryParameter[]>(
+                async (query) => client.basicOutputIds(query),
+                [{ address }, { expiresBefore: Math.floor(Date.now() / 1000) }],
+                AssociationType.BASIC_ADDRESS_EXPIRED,
+            ),
+        );
+
         promises.push(
             // Basic output -> storage return address
             this.fetchAssociatedOutputIds<QueryParameter>(
@@ -142,6 +151,15 @@ export class AssociatedOutputsHelper {
             ),
         );
 
+        promises.push(
+            // Nft output -> owner address expired outputs
+            this.fetchAssociatedOutputIds<NftQueryParameter[]>(
+                async (query) => client.nftOutputIds(query),
+                [{ address }, { expiresBefore: Math.floor(Date.now() / 1000) }],
+                AssociationType.NFT_ADDRESS_EXPIRED,
+            ),
+        );
+
         promises.push(
             // Nft output -> storage return address
             this.fetchAssociatedOutputIds<NftQueryParameter>(
@@ -181,6 +199,38 @@ export class AssociatedOutputsHelper {
         await Promise.all(promises);
     }
 
+    /**
+     * Retrieves the associations between output types and output IDs.
+     * @returns An array of associations.
+     */
+    public getAssociations(): IAssociation[] {
+        const associations: IAssociation[] = [];
+        for (const [type, outputIds] of this.associationToOutputIds.entries()) {
+            if (type !== AssociationType.BASIC_ADDRESS_EXPIRED && type !== AssociationType.NFT_ADDRESS_EXPIRED) {
+                if (
+                    type === AssociationType.BASIC_ADDRESS &&
+                    this.associationToOutputIds.get(AssociationType.BASIC_ADDRESS_EXPIRED)?.length > 0
+                ) {
+                    // remove expired basic outputs from basic address associations if they exist
+                    const expiredIds = this.associationToOutputIds.get(AssociationType.BASIC_ADDRESS_EXPIRED);
+                    const filteredOutputIds = outputIds.filter((id) => !expiredIds?.includes(id));
+                    associations.push({ type, outputIds: filteredOutputIds.reverse() });
+                } else if (
+                    type === AssociationType.NFT_ADDRESS &&
+                    this.associationToOutputIds.get(AssociationType.NFT_ADDRESS_EXPIRED)?.length > 0
+                ) {
+                    // remove expired nft outputs from nft address associations if they exist
+                    const expiredIds = this.associationToOutputIds.get(AssociationType.NFT_ADDRESS_EXPIRED);
+                    const filteredOutputIds = outputIds.filter((id) => !expiredIds?.includes(id));
+                    associations.push({ type, outputIds: filteredOutputIds.reverse() });
+                } else {
+                    associations.push({ type, outputIds: outputIds.reverse() });
+                }
+            }
+        }
+        return associations;
+    }
+
     /**
      * Generic helper function for fetching associated outputs.
      * @param fetch The function for the API call
@@ -197,7 +247,15 @@ export class AssociatedOutputsHelper {
 
         do {
             try {
-                const response = typeof args === "string" ? await fetch(args) : await fetch({ ...args, cursor });
+                let requestArgs: T = null;
+                if (typeof args === "string") {
+                    requestArgs = args;
+                } else if (Array.isArray(args)) {
+                    requestArgs = cursor ? ([...args, { cursor }] as T) : args;
+                } else {
+                    requestArgs = { ...args, cursor };
+                }
+                const response = await fetch(requestArgs);
 
                 if (typeof response === "string") {
                     const outputIds = associationToOutputIds.get(association);