From 28a3b3d6ea9d7355e2b4260ba2547d57f0c7fd3c Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 19 Sep 2024 21:10:26 -0400 Subject: [PATCH 1/4] feat: temp disable ssl on content server driver while ssl bug figured out --- src/DigNetwork/DigNetwork.ts | 106 ++++++++++++++++++--------------- src/blockchain/FullNodePeer.ts | 2 +- 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/DigNetwork/DigNetwork.ts b/src/DigNetwork/DigNetwork.ts index ffd5e5f..edcf8c5 100644 --- a/src/DigNetwork/DigNetwork.ts +++ b/src/DigNetwork/DigNetwork.ts @@ -7,7 +7,7 @@ import { getFilePathFromSha256 } from "../utils/hashUtils"; import { DataStore, ServerCoin } from "../blockchain"; import { DIG_FOLDER_PATH } from "../utils/config"; import { RootHistoryItem } from "../types"; -import { promisify } from 'util'; +import { promisify } from "util"; const rename = promisify(fs.rename); const unlink = promisify(fs.unlink); @@ -260,32 +260,37 @@ export class DigNetwork { `${rootInfo.root_hash}.dat` ); - fs.writeFileSync(`${this.storeDir}/${rootInfo.root_hash}.dat`, rootResponse); + fs.writeFileSync( + `${this.storeDir}/${rootInfo.root_hash}.dat`, + rootResponse + ); const root = JSON.parse(rootResponse); if (!skipData) { - // Use Object.entries() to iterate over the map (root.files) - await Promise.all( - Object.entries(root.files).map( - async ([storeKey, file]: [string, any]) => { - const filePath = getFilePathFromSha256( - file.sha256, - `${this.storeDir}/data` - ); - if (!fs.existsSync(filePath) || forceDownload) { - console.log( - `Downloading file with sha256: ${file.sha256}...` - ); - await this.downloadFileFromPeers( - `data/${file.sha256.match(/.{1,2}/g)!.join("/")}`, - filePath, - forceDownload - ); - } - } - ) - ); + // Explicitly define the type for file entries + interface FileEntry { + sha256: string; + } + + // Sequential file download + for (const [storeKey, file] of Object.entries( + root.files + )) { + const filePath = getFilePathFromSha256( + file.sha256, + `${this.storeDir}/data` + ); + + if (!fs.existsSync(filePath) || forceDownload) { + console.log(`Downloading file with sha256: ${file.sha256}...`); + await this.downloadFileFromPeers( + `data/${file.sha256.match(/.{1,2}/g)!.join("/")}`, + filePath, + forceDownload + ); + } + } } } @@ -327,7 +332,7 @@ export class DigNetwork { forceDownload ); } - + private async downloadFileFromPeers( dataPath: string, filePath: string, @@ -335,73 +340,78 @@ export class DigNetwork { ): Promise { let digPeers = await this.fetchAvailablePeers(); const tempFilePath = `${filePath}.tmp`; - + while (true) { if (!overwrite && fs.existsSync(filePath)) return; - + const blacklist = this.peerBlacklist.get(dataPath) || new Set(); - + for (const digPeer of digPeers) { if (blacklist.has(digPeer.IpAddress)) continue; - + try { // Create directory if it doesn't exist const directory = path.dirname(tempFilePath); if (!fs.existsSync(directory)) { fs.mkdirSync(directory, { recursive: true }); } - + // Stream the file data to a temporary file const fileStream = fs.createWriteStream(tempFilePath); - + // Start streaming the data from the peer - const peerStream = await digPeer.propagationServer.streamStoreData(dataPath); - + const peerStream = await digPeer.propagationServer.streamStoreData( + dataPath + ); + // Pipe the peer stream to the temp file await new Promise((resolve, reject) => { peerStream.pipe(fileStream); - - peerStream.on('end', resolve); - peerStream.on('error', reject); - fileStream.on('error', reject); + + peerStream.on("end", resolve); + peerStream.on("error", reject); + fileStream.on("error", reject); }); - + // Rename the temp file to the final file path after successful download await rename(tempFilePath, filePath); - - if (process.env.DIG_DEBUG === '1') { + + if (process.env.DIG_DEBUG === "1") { console.log(`Downloaded ${dataPath} from ${digPeer.IpAddress}`); } - + return; // Exit the method if download succeeds } catch (error) { console.warn( `Failed to download ${dataPath} from ${digPeer.IpAddress}, blacklisting peer and trying next...` ); blacklist.add(digPeer.IpAddress); - + // Clean up the temp file in case of failure if (fs.existsSync(tempFilePath)) { await unlink(tempFilePath); } } } - + this.peerBlacklist.set(dataPath, blacklist); - + if (blacklist.size >= digPeers.length) { - if (process.env.DIG_DEBUG === '1') { - console.warn(`All peers blacklisted for ${dataPath}. Refreshing peers...`); + if (process.env.DIG_DEBUG === "1") { + console.warn( + `All peers blacklisted for ${dataPath}. Refreshing peers...` + ); } - + digPeers = await this.fetchAvailablePeers(); if (!digPeers.length) { - throw new Error(`Failed to download ${dataPath}: no peers available.`); + throw new Error( + `Failed to download ${dataPath}: no peers available.` + ); } } } } - private async runProgressBar( total: number, diff --git a/src/blockchain/FullNodePeer.ts b/src/blockchain/FullNodePeer.ts index 7010f17..85c18dc 100644 --- a/src/blockchain/FullNodePeer.ts +++ b/src/blockchain/FullNodePeer.ts @@ -291,7 +291,7 @@ export class FullNodePeer { FullNodePeer.cachedPeer = { peer: bestPeer, timestamp: now }; - console.log(`Using Peer: ${peerIPs[bestPeerIndex]}`); + console.log(`Using Fullnode Peer: ${peerIPs[bestPeerIndex]}`); return bestPeer; } From 0098cbe5ad5507e1e84007ad1f6009e248b3632d Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 19 Sep 2024 21:11:06 -0400 Subject: [PATCH 2/4] chore(release): 0.0.1-alpha.44 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b9cb26..fe9dba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.44](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.43...v0.0.1-alpha.44) (2024-09-20) + + +### Features + +* temp disable ssl on content server driver while ssl bug figured out ([28a3b3d](https://github.com/DIG-Network/dig-chia-sdk/commit/28a3b3d6ea9d7355e2b4260ba2547d57f0c7fd3c)) + ### [0.0.1-alpha.43](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.42...v0.0.1-alpha.43) (2024-09-20) diff --git a/package-lock.json b/package-lock.json index f996614..a5bfefa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.43", + "version": "0.0.1-alpha.44", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.43", + "version": "0.0.1-alpha.44", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.24", diff --git a/package.json b/package.json index eeb2a8f..891761a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.43", + "version": "0.0.1-alpha.44", "description": "", "type": "commonjs", "main": "./dist/index.js", From f47ac42c5c6823b028157fcd0c3f6924dd6d0d96 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 19 Sep 2024 21:18:03 -0400 Subject: [PATCH 3/4] chore: test sequantial download --- src/DigNetwork/DigNetwork.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/DigNetwork/DigNetwork.ts b/src/DigNetwork/DigNetwork.ts index edcf8c5..e76f4a5 100644 --- a/src/DigNetwork/DigNetwork.ts +++ b/src/DigNetwork/DigNetwork.ts @@ -178,7 +178,7 @@ export class DigNetwork { const peerIp = digPeers[0]; const digPeer = new DigPeer(peerIp, storeId); - const storeResponse = await digPeer.propagationServer.headStore({ + const storeResponse = await digPeer.contentServer.headStore({ hasRootHash: rootHash, }); @@ -349,6 +349,12 @@ export class DigNetwork { for (const digPeer of digPeers) { if (blacklist.has(digPeer.IpAddress)) continue; + const response = await digPeer.propagationServer.headStore(); + + if (!response.success) { + continue; + } + try { // Create directory if it doesn't exist const directory = path.dirname(tempFilePath); From 17023411a99b401146999690ff741e8850b3fabe Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Thu, 19 Sep 2024 21:18:47 -0400 Subject: [PATCH 4/4] chore(release): 0.0.1-alpha.45 --- CHANGELOG.md | 2 ++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe9dba6..de4e6a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.45](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.44...v0.0.1-alpha.45) (2024-09-20) + ### [0.0.1-alpha.44](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.43...v0.0.1-alpha.44) (2024-09-20) diff --git a/package-lock.json b/package-lock.json index a5bfefa..6e4fdb8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.44", + "version": "0.0.1-alpha.45", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.44", + "version": "0.0.1-alpha.45", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.24", diff --git a/package.json b/package.json index 891761a..895adda 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.44", + "version": "0.0.1-alpha.45", "description": "", "type": "commonjs", "main": "./dist/index.js",