From e2519b1cb9fed7889b0026079cfeb3e86c72c363 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 7 Oct 2024 10:17:56 -0400 Subject: [PATCH 1/2] feat: additional caching measure to optimize requests --- src/DigNetwork/ContentServer.ts | 160 +++++++++++++++++++++------- src/DigNetwork/PropagationServer.ts | 42 +++++++- src/utils/PeerRanker.ts | 1 - 3 files changed, 162 insertions(+), 41 deletions(-) diff --git a/src/DigNetwork/ContentServer.ts b/src/DigNetwork/ContentServer.ts index 1f0eda9..0255b5b 100644 --- a/src/DigNetwork/ContentServer.ts +++ b/src/DigNetwork/ContentServer.ts @@ -4,6 +4,10 @@ import { URL } from "url"; import { Readable } from "stream"; import { getOrCreateSSLCerts } from "../utils/ssl"; import { formatHost } from "../utils/network"; +import NodeCache from "node-cache"; + +const hasRootHashCache = new NodeCache({ stdTTL: 86400 }); +const wellKnownCache = new NodeCache({ stdTTL: 86400 }); export class ContentServer { private ipAddress: string; @@ -30,7 +34,9 @@ export class ContentServer { challengeHex?: string ): Promise { // Construct the base URL - let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`; + let url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/chia.${this.storeId}.${rootHash}/${key}`; // If a challenge is provided, append it as a query parameter if (challengeHex) { @@ -40,13 +46,12 @@ export class ContentServer { return this.fetchWithRetries(url); } - // New method to get only the first chunk of the content - public async getKeyChunk( - key: string, - rootHash: string, - ): Promise { + // New method to get only the first chunk of the content + public async getKeyChunk(key: string, rootHash: string): Promise { // Construct the base URL - let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`; + let url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/chia.${this.storeId}.${rootHash}/${key}`; return this.fetchFirstChunk(url); } @@ -65,15 +70,44 @@ export class ContentServer { } } - // Method to get the .well-known information + /** + * Fetches and caches the .well-known information for the store's IP address. + * + * @returns A promise that resolves to the .well-known JSON data. + */ public async getWellKnown(): Promise { - const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/.well-known`; - return this.fetchJson(url); + // Construct the cache key based on ipAddress + const cacheKey = `${this.ipAddress}-wellknown`; + + // Check if the result is already cached + const cachedResult = wellKnownCache.get(cacheKey); + if (cachedResult !== undefined) { + return cachedResult; + } + + // If not cached, proceed to fetch the .well-known information + const url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/.well-known`; + + try { + const data = await this.fetchJson(url); + wellKnownCache.set(cacheKey, data); + return data; + } catch (error: any) { + console.error( + `Error fetching .well-known information for ${this.ipAddress}:`, + error.message + ); + throw error; // Propagate the error after logging + } } // Method to get the list of known stores public async getKnownStores(): Promise { - const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/.well-known/stores`; + const url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/.well-known/stores`; return this.fetchJson(url); } @@ -85,14 +119,23 @@ export class ContentServer { // Method to get the index of keys in a store public async getKeysIndex(rootHash?: string): Promise { - let udi = `chia.${this.storeId}`; + try { + let udi = `chia.${this.storeId}`; - if (rootHash) { - udi += `.${rootHash}`; - } + if (rootHash) { + udi += `.${rootHash}`; + } - const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}`; - return this.fetchJson(url); + const url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/${udi}`; + return this.fetchJson(url); + } catch (error: any) { + if (rootHash) { + hasRootHashCache.del(`${this.storeId}-${rootHash}`); + } + throw error; + } } // Method to check if a specific key exists (HEAD request) @@ -100,14 +143,23 @@ export class ContentServer { key: string, rootHash?: string ): Promise<{ success: boolean; headers?: http.IncomingHttpHeaders }> { - let udi = `chia.${this.storeId}`; + try { + let udi = `chia.${this.storeId}`; - if (rootHash) { - udi += `.${rootHash}`; - } + if (rootHash) { + udi += `.${rootHash}`; + } - const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`; - return this.head(url); + const url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/${udi}/${key}`; + return this.head(url); + } catch (error: any) { + if (rootHash) { + hasRootHashCache.del(`${this.storeId}-${rootHash}`); + } + throw error; + } } // Method to check if a specific store exists (HEAD request) @@ -115,23 +167,55 @@ export class ContentServer { success: boolean; headers?: http.IncomingHttpHeaders; }> { - let url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}`; + try { + let url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/chia.${this.storeId}`; - if (options?.hasRootHash) { - url += `?hasRootHash=${options.hasRootHash}`; - } + if (options?.hasRootHash) { + url += `?hasRootHash=${options.hasRootHash}`; + } - return this.head(url); + return this.head(url); + } catch (error: any) { + if (options?.hasRootHash) { + hasRootHashCache.del(`${this.storeId}-${options.hasRootHash}`); + } + throw error; + } } + /** + * Checks if the store has the specified rootHash. + * Utilizes caching to improve performance. + * + * @param rootHash - The root hash to check. + * @returns A promise that resolves to true if the root hash exists, otherwise false. + */ public async hasRootHash(rootHash: string): Promise { + // Construct the cache key using storeId and rootHash + const cacheKey = `${this.storeId}-${rootHash}`; + + // Check if the result is already cached + const cachedResult = hasRootHashCache.get(cacheKey); + if (cachedResult !== undefined) { + return cachedResult; + } + + // If not cached, perform the headStore request const { success, headers } = await this.headStore({ hasRootHash: rootHash, }); - if (success) { - return headers?.["x-has-root-hash"] === "true"; + + // Determine if the store has the root hash + const hasHash = success && headers?.["x-has-root-hash"] === "true"; + + // Only cache the result if the store has the root hash + if (hasHash) { + hasRootHashCache.set(cacheKey, true); } - return false; + + return hasHash; } public streamKey(key: string, rootHash?: string): Promise { @@ -142,7 +226,9 @@ export class ContentServer { } return new Promise((resolve, reject) => { - const url = `https://${formatHost(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`; + const url = `https://${formatHost(this.ipAddress)}:${ + ContentServer.port + }/${udi}/${key}`; const urlObj = new URL(url); const requestOptions = { @@ -164,6 +250,7 @@ export class ContentServer { reject(new Error("Redirected without a location header")); } } else { + hasRootHashCache.del(`${this.storeId}-${rootHash}`); reject( new Error( `Failed to retrieve data from ${url}. Status code: ${response.statusCode}` @@ -483,13 +570,14 @@ export class ContentServer { response.headers.location ) { // Handle redirects - const redirectUrl = new URL(response.headers.location, url).toString(); // Resolve relative URLs + const redirectUrl = new URL( + response.headers.location, + url + ).toString(); // Resolve relative URLs if (timeout) { clearTimeout(timeout); } - this.fetchFirstChunk(redirectUrl) - .then(resolve) - .catch(reject); + this.fetchFirstChunk(redirectUrl).then(resolve).catch(reject); } else { if (timeout) { clearTimeout(timeout); diff --git a/src/DigNetwork/PropagationServer.ts b/src/DigNetwork/PropagationServer.ts index fc7cab1..6b81407 100644 --- a/src/DigNetwork/PropagationServer.ts +++ b/src/DigNetwork/PropagationServer.ts @@ -20,6 +20,10 @@ import { promptCredentials } from "../utils/credentialsUtils"; import { STORE_PATH } from "../utils/config"; import { Wallet, DataStore } from "../blockchain"; import { formatHost } from "../utils/network"; +import NodeCache from "node-cache"; + +// Initialize cache with a TTL of 1 week (604800 seconds) +const storeExistsCache = new NodeCache({ stdTTL: 86400 }); // Helper function to trim long filenames with ellipsis and ensure consistent padding function formatFilename(filename: string | undefined, maxLength = 30): string { @@ -140,7 +144,10 @@ export class PropagationServer { try { const response = await axios.post(url, data, config); - console.log(green(`✔ Successfully pinged peer: ${this.ipAddress}`)); + console.log( + green(`✔ Successfully pinged peer: ${this.ipAddress}`), + response + ); return response.data; } catch (error: any) { console.error(red(`✖ Failed to ping peer: ${this.ipAddress}`)); @@ -181,14 +188,34 @@ export class PropagationServer { } /** - * Check if the store and optional root hash exist by making a HEAD request. + * Checks if the store exists and optionally if a specific rootHash exists. + * Utilizes caching to improve performance. + * + * @param rootHash - (Optional) The root hash to check. + * @returns A promise that resolves to an object containing store existence and root hash existence. */ - async checkStoreExists( + public async checkStoreExists( rootHash?: string ): Promise<{ storeExists: boolean; rootHashExists: boolean }> { + // Construct the cache key + const cacheKey = rootHash + ? `${this.storeId}-${rootHash}` + : `${this.storeId}-nohash`; + + // Check if the result is already cached + const cachedResult = storeExistsCache.get<{ + storeExists: boolean; + rootHashExists: boolean; + }>(cacheKey); + if (cachedResult !== undefined) { + return cachedResult; + } + + // If not cached, proceed with the HTTP request const spinner = createSpinner( `Checking if store ${this.storeId} exists...` ).start(); + try { const config: AxiosRequestConfig = { httpsAgent: this.createHttpsAgent(), @@ -221,7 +248,11 @@ export class PropagationServer { }); } - return { storeExists, rootHashExists }; + const result = { storeExists, rootHashExists }; + + storeExistsCache.set(cacheKey, result); + + return result; } catch (error: any) { spinner.error({ text: red("Error checking if store exists:") }); console.error(red(error.message)); @@ -625,6 +656,7 @@ export class PropagationServer { return Buffer.concat(dataBuffers); } catch (error) { + storeExistsCache.del(`${this.storeId}-nohash`); throw error; } finally { if (progressBar) { @@ -773,6 +805,8 @@ export class PropagationServer { console.log("integrity check"); } } catch (error) { + storeExistsCache.del(`${this.storeId}-nohash`); + storeExistsCache.del(`${this.storeId}-${rootHash}`); throw error; } finally { if (progressBar) { diff --git a/src/utils/PeerRanker.ts b/src/utils/PeerRanker.ts index dac1f02..f91171e 100644 --- a/src/utils/PeerRanker.ts +++ b/src/utils/PeerRanker.ts @@ -109,7 +109,6 @@ export class PeerRanker { private async measureBandwidth(ip: string): Promise { const cachedMetrics = peerCache.get(ip); if (cachedMetrics && cachedMetrics.bandwidth) { - console.log(`Bandwidth for IP ${ip} retrieved from cache.`); return cachedMetrics.bandwidth; } From 47e25a21647ee21adaf780efdb56ad03c0084cca Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 7 Oct 2024 10:19:49 -0400 Subject: [PATCH 2/2] chore(release): 0.0.1-alpha.156 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff25f3..a7c09c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.156](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.155...v0.0.1-alpha.156) (2024-10-07) + + +### Features + +* additional caching measure to optimize requests ([e2519b1](https://github.com/DIG-Network/dig-chia-sdk/commit/e2519b1cb9fed7889b0026079cfeb3e86c72c363)) + ### [0.0.1-alpha.155](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.154...v0.0.1-alpha.155) (2024-10-06) diff --git a/package-lock.json b/package-lock.json index 94a8d19..ad37fad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.155", + "version": "0.0.1-alpha.156", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.155", + "version": "0.0.1-alpha.156", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.29", diff --git a/package.json b/package.json index f27d34b..e8f889c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.155", + "version": "0.0.1-alpha.156", "description": "", "type": "commonjs", "main": "./dist/index.js",