diff --git a/CHANGELOG.md b/CHANGELOG.md index d956854..37aaf39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.135](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.134...v0.0.1-alpha.135) (2024-10-05) + + +### Features + +* bottleneck fullnode requests to 100 requests per min ([85c7529](https://github.com/DIG-Network/dig-chia-sdk/commit/85c7529acf36b1df6c1dff127d9a8c6770abb3f0)) + + +### Bug Fixes + +* ping peer message ([a18ccae](https://github.com/DIG-Network/dig-chia-sdk/commit/a18ccae01f1685e7ebc73381599e77b334e9e7d5)) + ### [0.0.1-alpha.134](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.133...v0.0.1-alpha.134) (2024-10-05) diff --git a/dig.crt b/dig.crt new file mode 100644 index 0000000..73a2f53 --- /dev/null +++ b/dig.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDLDCCAhSgAwIBAgIUP/RkfZQ+VLvSfQ2qOf7peUnDd/gwDQYJKoZIhvcNAQEL +BQAwRDENMAsGA1UECgwEQ2hpYTEQMA4GA1UEAwwHQ2hpYSBDQTEhMB8GA1UECwwY +T3JnYW5pYyBGYXJtaW5nIERpdmlzaW9uMCAXDTI0MTAwNDE4MzQzNVoYDzIxMDAw +ODAyMDAwMDAwWjBBMQ0wCwYDVQQDDARDaGlhMQ0wCwYDVQQKDARDaGlhMSEwHwYD +VQQLDBhPcmdhbmljIEZhcm1pbmcgRGl2aXNpb24wggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCqvzy6I/4sgmPLLA78WVnmbO63PYS/E0EMNFP/OBi3/4ci +XnsYzmXnLUgmvi/C5OJiJz8OKKC4sh3GJ1hZxqDDjwr8l2w4mL94IqzA9cvRLSuj +KlJSlLBk3W9DBat/MGIgFp2bGyIn7EeRC/kg6AChuvWhuQ5LgIB9zjIkIaeIH5Pb +meQEsNHGtnECO/RwXJ/Md//AlEmX70pwdfXaD83aXjacX/iSsIzzfZ3T6Y0DyncT +oC+b/HpFsWCq5l72AKgRsn0zuh9gKYw1EjremRRhex/vEGoidncuVyS2Q7gaU+Zs +6mXKUTYN10C/ffLyaY0j5HtrTAnRu+Sr2SxPqjErAgMBAAGjFzAVMBMGA1UdEQQM +MAqCCGNoaWEubmV0MA0GCSqGSIb3DQEBCwUAA4IBAQAQ+8jR1tx1lTOpT6dkH8cD +gLU/I/r2kMehkAVYPCZA2cASlNN1wW/TTfI+lmXDl93XqhLPVuwDXPUcDK/1gIEq +NtJxBHBgCY4obdiYYWugV9XjwDoUCkqFNLiHBenUsFab6CHTQBh2EwI85bnstyl0 +vyEpeC6bLcCWkhGMX6pVUEGQ1vM6FsxkS/ViGPvLDXTHNRfllHyqvFx/tOwP19cL +1gmuFFBQ1YEKusJ72eqWgmOL/GB/Zwgk1WeTtzhe7uK/HQdzL8s4Bdh/vcs+c8pH +pMU+AuxVxBuyhj5XXZpzXt8hoSI0VWL6NNK/AUx7d7hbEsAVB7do7LlqUyWm4GUl +-----END CERTIFICATE----- diff --git a/dig.key b/dig.key new file mode 100644 index 0000000..0e91a50 --- /dev/null +++ b/dig.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCqvzy6I/4sgmPL +LA78WVnmbO63PYS/E0EMNFP/OBi3/4ciXnsYzmXnLUgmvi/C5OJiJz8OKKC4sh3G +J1hZxqDDjwr8l2w4mL94IqzA9cvRLSujKlJSlLBk3W9DBat/MGIgFp2bGyIn7EeR +C/kg6AChuvWhuQ5LgIB9zjIkIaeIH5PbmeQEsNHGtnECO/RwXJ/Md//AlEmX70pw +dfXaD83aXjacX/iSsIzzfZ3T6Y0DyncToC+b/HpFsWCq5l72AKgRsn0zuh9gKYw1 +EjremRRhex/vEGoidncuVyS2Q7gaU+Zs6mXKUTYN10C/ffLyaY0j5HtrTAnRu+Sr +2SxPqjErAgMBAAECggEAThMZ4od4vsN0fD+OLqdwqyOeWB2CKJjWQo1e5zGwY9os +4XNQWDxwbi/PcR+/2R8twPHvLDs5mpqfcK8nAA4KDsHGpU9cBdkanAVG9withN1h +ub8tW2Nv8P8r0/qwu3zVMZUFxhm3GYg6BUHzTa+oMku565QhzDZbCquRU+Irp9sA +rZ64MVlqG3rfRz8FdTM3TSNXhX1hcwuXFSWxP9LKnm7CNubKnNHYGtQHsvIrnY3s +C3M/T8tGBKnuVew+jKBjDDz518Bdmtwmh1ccX7WHX0aYVYNoXvZaGlUTEyVLCZX3 +D/hoTf8FzCSFOy0qH0nPgT9lNopho+eOp+QEVZTDWQKBgQDjn5kY4hCu2WNk2TxA +E6QIan6ih2EnYdMyrzNHrOX/sTi9iufvOA6yi/P6Ph7mALBdDhfJqFCEZnSqBiN+ +xTkPqqySyq1uy+gToE/tbpqSWZPUXMyNoc6E85jaRX15vngxVNiTOzJlEoykxn/n +1Ng0pscqKBwvC2o9nf2O9gt9HQKBgQDACHp+rV8SM4dbUOqddQ208jIMTuHMf56d +tzmE6R6/S2HmDy3siMAjRKgSft9hfDUo7x07UsF/jkUEWXpNHIiwfwLJ0ElkMCd0 +gIQLpsVfmveFa/qQ0cEp7kacuOOT6CE9XBMEIQtkVXCuV5L3Xo+5bC8Fx4jMyayw +CYm9PCe85wKBgQC50QOt6H4t/pDBNwWUWXRC2ozeKR4KhDVg3t1B2cc4YHgtY0PL +aTu2TcGxuxyGLnHKxUJuANUaAHmkgrZfOqlGPNH8UzAZjqO5wdj9vpi4eB/R8J+b +z0dECYfyR2ATDoYX32edaWnOUMI3kUPBAWQuNyfHJk87qFnmSx8+oWTnkQKBgQCf +VemdtmjOB2dmQ2uIHpmy68rPH5yHO9T2dBMLzwouG3Qtmaa3Pnh+SvdR8WeT0aWi +Q1Tz5iSbnAZ0J3ItDWH1YE2F+ocK1FHIfuIRcN0QCNscH44WH5SxH/4DB/38uXzr +FrzIjkqPg70tS4isLBABAFy75OYDVcstfZyGIaWvPQKBgQCTgp5CX8cGiFD5nJ3k +2qdWSOB/160MdV7EH/FUrzpx/S3mrB+JvlK0J9ei6NH+SUvB7pLHcjzBCHso8h62 +nTzhHyYTo7mFbIOEFhU+Y386EMfIQSplM36a+ZxXC6r+Xz2QyWO4Yh6eI+eS050n +gK+RBgUP/CyZFqG4I5ucWATdFQ== +-----END PRIVATE KEY----- diff --git a/package-lock.json b/package-lock.json index 2e811cf..190f8f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.29", @@ -14,6 +14,7 @@ "archiver": "^7.0.1", "axios": "^1.7.7", "bip39": "^3.1.0", + "bottleneck": "^2.19.5", "chia-bls": "^1.0.2", "chia-config-loader": "^1.0.1", "chia-root-resolver": "^1.0.0", @@ -1222,6 +1223,11 @@ "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", diff --git a/package.json b/package.json index bcb9117..e4f26ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "description": "", "type": "commonjs", "main": "./dist/index.js", @@ -30,6 +30,7 @@ "archiver": "^7.0.1", "axios": "^1.7.7", "bip39": "^3.1.0", + "bottleneck": "^2.19.5", "chia-bls": "^1.0.2", "chia-config-loader": "^1.0.1", "chia-root-resolver": "^1.0.0", diff --git a/src/DigNetwork/PropagationServer.ts b/src/DigNetwork/PropagationServer.ts index 14c8e59..fc7cab1 100644 --- a/src/DigNetwork/PropagationServer.ts +++ b/src/DigNetwork/PropagationServer.ts @@ -129,10 +129,6 @@ export class PropagationServer { headers: { "Content-Type": "application/json", }, - validateStatus: (status) => { - // Accept all status codes to handle them manually - return true; - }, }; // Data to send in the request (storeId and rootHash) @@ -144,48 +140,9 @@ export class PropagationServer { try { const response = await axios.post(url, data, config); - - if (response.status === 200) { - console.log(green(`✔ Peer was up to date: ${this.ipAddress}`)); - return; - } else if ( - response.status === 400 && - response.data?.error?.includes("does not exist") - ) { - console.log( - yellow( - `⚠ Peer ${this.ipAddress} does not have store ${this.storeId}. Notifying for update.` - ) - ); - // You can implement additional logic here if needed, such as retrying or logging - return; - } else { - console.error( - red( - `✖ Unexpected response from peer ${this.ipAddress}: ${response.status} ${response.statusText}` - ) - ); - throw new Error( - `Unexpected response: ${response.status} ${response.statusText}` - ); - } + console.log(green(`✔ Successfully pinged peer: ${this.ipAddress}`)); + return response.data; } catch (error: any) { - if (error.response) { - // Server responded with a status other than 2xx - if ( - error.response.status === 400 && - error.response.data?.error?.includes("does not exist") - ) { - console.log( - yellow( - `⚠ Peer ${this.ipAddress} does not have store ${this.storeId}. Notifying for update.` - ) - ); - // You can implement additional logic here if needed - return; - } - } - console.error(red(`✖ Failed to ping peer: ${this.ipAddress}`)); console.error(red(error.message)); throw error; diff --git a/src/blockchain/DataStore.ts b/src/blockchain/DataStore.ts index 1afc47e..665d1ec 100644 --- a/src/blockchain/DataStore.ts +++ b/src/blockchain/DataStore.ts @@ -298,13 +298,13 @@ export class DataStore { public static getAllStores(): DataStore[] { const storeFolders = fs.readdirSync(STORE_PATH); - const storIds = storeFolders.filter( + const storeIds = storeFolders.filter( (folder) => /^[a-f0-9]{64}$/.test(folder) && fs.lstatSync(path.join(STORE_PATH, folder)).isDirectory() ); - return storIds.map((storeId) => DataStore.from(storeId)); + return storeIds.map((storeId) => DataStore.from(storeId)); } /** diff --git a/src/blockchain/FullNodePeer.ts b/src/blockchain/FullNodePeer.ts index ae8fd5d..3762eb0 100644 --- a/src/blockchain/FullNodePeer.ts +++ b/src/blockchain/FullNodePeer.ts @@ -8,6 +8,7 @@ import { createSpinner } from "nanospinner"; import { MIN_HEIGHT, MIN_HEIGHT_HEADER_HASH } from "../utils/config"; import { Environment } from "../utils/Environment"; import NodeCache from "node-cache"; +import Bottleneck from "bottleneck"; // Constants const FULLNODE_PORT = 8444; @@ -21,9 +22,10 @@ const DNS_HOSTS = [ ]; const CONNECTION_TIMEOUT = 2000; // in milliseconds const CACHE_DURATION = 30000; // in milliseconds -const COOLDOWN_DURATION = 60000; // in milliseconds +const COOLDOWN_DURATION = 300000; // 5 minutes in milliseconds const MAX_PEERS_TO_FETCH = 5; // Maximum number of peers to fetch from DNS const MAX_RETRIES = 3; // Maximum number of retry attempts +const MAX_REQUESTS_PER_MINUTE = 100; // Throttle limit /** * Represents a peer with its reliability weight and address. @@ -32,6 +34,16 @@ interface PeerInfo { peer: Peer; weight: number; address: string; + isConnected: boolean; // Indicates if the peer is currently connected +} + +/** + * Represents a queued method call. + */ +interface QueuedCall { + execute: () => Promise; + resolve: (value: any) => void; + reject: (reason?: any) => void; } /** @@ -59,6 +71,12 @@ export class FullNodePeer { // Cache for fetched peer IPs private static peerIPCache = new NodeCache({ stdTTL: CACHE_DURATION / 1000 }); + // Bottleneck instance for global throttling + private static limiter = new Bottleneck({ + maxConcurrent: 1, // Ensures only one request is processed at a time + minTime: 60000 / MAX_REQUESTS_PER_MINUTE, // Calculated delay between requests + }); + // Private constructor for singleton pattern private constructor(private peer: Peer) {} @@ -336,10 +354,11 @@ export class FullNodePeer { private static async getBestPeer(): Promise { const now = Date.now(); - // Refresh cachedPeer if expired + // Refresh cachedPeer if expired or disconnected if ( FullNodePeer.cachedPeer && - now - FullNodePeer.cachedPeer.timestamp < CACHE_DURATION + now - FullNodePeer.cachedPeer.timestamp < CACHE_DURATION && + FullNodePeer.peerInfos.get(FullNodePeer.extractPeerIP(FullNodePeer.cachedPeer.peer) || "")?.isConnected ) { return FullNodePeer.cachedPeer.peer; } @@ -396,6 +415,7 @@ export class FullNodePeer { peer: proxiedPeer, weight: FullNodePeer.peerWeights.get(selectedPeerIP) || 1, address: selectedPeerIP, + isConnected: true, // Mark as connected }); // Cache the peer @@ -407,76 +427,91 @@ export class FullNodePeer { } /** - * Creates a proxy for the peer to handle errors and implement retries. + * Creates a proxy for the peer to handle errors, implement retries, and enforce throttling. * @param {Peer} peer - The Peer instance. * @param {string} peerIP - The IP address of the peer. * @param {number} [retryCount=0] - The current retry attempt. * @returns {Peer} The proxied Peer instance. */ private static createPeerProxy(peer: Peer, peerIP: string, retryCount: number = 0): Peer { + // Listen for close events if the Peer class supports it + // This assumes that the Peer class emits a 'close' event when the connection is closed + // Adjust accordingly based on the actual Peer implementation + if (typeof (peer as any).on === "function") { + (peer as any).on("close", () => { + console.warn(`Peer ${peerIP} connection closed.`); + FullNodePeer.handlePeerDisconnection(peerIP); + }); + + (peer as any).on("error", (error: any) => { + console.error(`Peer ${peerIP} encountered an error: ${error.message}`); + FullNodePeer.handlePeerDisconnection(peerIP); + }); + } + return new Proxy(peer, { get: (target, prop) => { const originalMethod = (target as any)[prop]; if (typeof originalMethod === "function") { - return async (...args: any[]) => { - try { - const result = await originalMethod.apply(target, args); - // On successful operation, increase the weight slightly - const currentWeight = FullNodePeer.peerWeights.get(peerIP) || 1; - FullNodePeer.peerWeights.set(peerIP, currentWeight + 0.1); // Increment weight - return result; - } catch (error: any) { - console.error(`Peer ${peerIP} encountered an error: ${error.message}`); - - // Check if the error is related to WebSocket or Operation timed out - if ( - error.message.includes("WebSocket") || - error.message.includes("Operation timed out") - ) { - // Add the faulty peer to the cooldown cache - FullNodePeer.cooldownCache.set(peerIP, true); - - // Decrease weight or remove peer - const currentWeight = FullNodePeer.peerWeights.get(peerIP) || 1; - if (currentWeight > 1) { - FullNodePeer.peerWeights.set(peerIP, currentWeight - 1); - } else { - FullNodePeer.peerWeights.delete(peerIP); - } + return (...args: any[]) => { + // Wrap the method call with Bottleneck's scheduling + return FullNodePeer.limiter.schedule(async () => { + const peerInfo = FullNodePeer.peerInfos.get(peerIP); + if (!peerInfo || !peerInfo.isConnected) { + throw new Error(`Cannot perform operation: Peer ${peerIP} is disconnected.`); + } - // If maximum retries reached, throw the error - if (retryCount >= MAX_RETRIES) { - console.error(`Max retries reached for method ${String(prop)} on peer ${peerIP}.`); - throw error; - } + try { + const result = await originalMethod.apply(target, args); + // On successful operation, increase the weight slightly + const currentWeight = FullNodePeer.peerWeights.get(peerIP) || 1; + FullNodePeer.peerWeights.set(peerIP, currentWeight + 0.1); // Increment weight + return result; + } catch (error: any) { + console.error(`Peer ${peerIP} encountered an error: ${error.message}`); + + // Check if the error is related to WebSocket or Operation timed out + if ( + error.message.includes("WebSocket") || + error.message.includes("Operation timed out") + ) { + // Handle the disconnection and mark the peer accordingly + FullNodePeer.handlePeerDisconnection(peerIP); + + // If maximum retries reached, throw the error + if (retryCount >= MAX_RETRIES) { + console.error(`Max retries reached for method ${String(prop)} on peer ${peerIP}.`); + throw error; + } - // Attempt to select a new peer and retry the method - try { - console.info(`Selecting a new peer to retry method ${String(prop)}...`); - const newPeer = await FullNodePeer.getBestPeer(); + // Attempt to select a new peer and retry the method + try { + console.info(`Selecting a new peer to retry method ${String(prop)}...`); + const newPeer = await FullNodePeer.getBestPeer(); - // Extract new peer's IP address - const newPeerIP = FullNodePeer.extractPeerIP(newPeer); + // Extract new peer's IP address + const newPeerIP = FullNodePeer.extractPeerIP(newPeer); - if (!newPeerIP) { - throw new Error("Unable to extract IP from the new peer."); - } + if (!newPeerIP) { + throw new Error("Unable to extract IP from the new peer."); + } - // Wrap the new peer with a proxy, incrementing the retry count - const proxiedNewPeer = FullNodePeer.createPeerProxy(newPeer, newPeerIP, retryCount + 1); + // Wrap the new peer with a proxy, incrementing the retry count + const proxiedNewPeer = FullNodePeer.createPeerProxy(newPeer, newPeerIP, retryCount + 1); - // Retry the method on the new peer - return await (proxiedNewPeer as any)[prop](...args); - } catch (retryError: any) { - console.error(`Retry failed on a new peer: ${retryError.message}`); - throw retryError; + // Retry the method on the new peer + return await (proxiedNewPeer as any)[prop](...args); + } catch (retryError: any) { + console.error(`Retry failed on a new peer: ${retryError.message}`); + throw retryError; + } + } else { + // For other errors, handle normally + throw error; } - } else { - // For other errors, handle normally - throw error; } - } + }); }; } return originalMethod; @@ -484,6 +519,38 @@ export class FullNodePeer { }); } + /** + * Handles peer disconnection by marking it in cooldown and updating internal states. + * @param {string} peerIP - The IP address of the disconnected peer. + */ + private static handlePeerDisconnection(peerIP: string): void { + // Mark the peer in cooldown + FullNodePeer.cooldownCache.set(peerIP, true); + + // Decrease weight or remove peer + const currentWeight = FullNodePeer.peerWeights.get(peerIP) || 1; + if (currentWeight > 1) { + FullNodePeer.peerWeights.set(peerIP, currentWeight - 1); + } else { + FullNodePeer.peerWeights.delete(peerIP); + } + + // Update the peer's connection status + const peerInfo = FullNodePeer.peerInfos.get(peerIP); + if (peerInfo) { + peerInfo.isConnected = false; + FullNodePeer.peerInfos.set(peerIP, peerInfo); + } + + // If the disconnected peer was the cached peer, invalidate the cache + if ( + FullNodePeer.cachedPeer && + FullNodePeer.extractPeerIP(FullNodePeer.cachedPeer.peer) === peerIP + ) { + FullNodePeer.cachedPeer = null; + } + } + /** * Extracts the IP address from a Peer instance. * @param {Peer} peer - The Peer instance. @@ -498,36 +565,53 @@ export class FullNodePeer { return null; } - /** + /** * Waits for a coin to be confirmed (spent) on the blockchain. * @param {Buffer} parentCoinInfo - The parent coin information. * @returns {Promise} Whether the coin was confirmed. */ - public static async waitForConfirmation( - parentCoinInfo: Buffer - ): Promise { - const spinner = createSpinner("Waiting for confirmation...").start(); - const peer = await FullNodePeer.connect(); - - try { - while (true) { - const confirmed = await peer.isCoinSpent( - parentCoinInfo, - MIN_HEIGHT, - Buffer.from(MIN_HEIGHT_HEADER_HASH, "hex") - ); - - if (confirmed) { - spinner.success({ text: "Coin confirmed!" }); - return true; - } - - await new Promise((resolve) => setTimeout(resolve, 5000)); + public static async waitForConfirmation( + parentCoinInfo: Buffer + ): Promise { + const spinner = createSpinner("Waiting for confirmation...").start(); + let peer: Peer; + + try { + peer = await FullNodePeer.connect(); + } catch (error: any) { + spinner.error({ text: "Failed to connect to a peer." }); + console.error(`waitForConfirmation connection error: ${error.message}`); + throw error; + } + + try { + while (true) { + const confirmed = await peer.isCoinSpent( + parentCoinInfo, + MIN_HEIGHT, + Buffer.from(MIN_HEIGHT_HEADER_HASH, "hex") + ); + + if (confirmed) { + spinner.success({ text: "Coin confirmed!" }); + return true; } - } catch (error: any) { - spinner.error({ text: "Error while waiting for confirmation." }); - console.error(`waitForConfirmation error: ${error.message}`); - throw error; + + await FullNodePeer.delay(5000); } + } catch (error: any) { + spinner.error({ text: "Error while waiting for confirmation." }); + console.error(`waitForConfirmation error: ${error.message}`); + throw error; } + } + + /** + * Delays execution for a specified amount of time. + * @param {number} ms - Milliseconds to delay. + * @returns {Promise} A promise that resolves after the delay. + */ + private static delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } }