From 974e839d445046cab7874323044bf0e22810a899 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 23 Sep 2024 19:25:30 -0400 Subject: [PATCH 1/3] chore: update stuff --- src/DigNetwork/DigPeer.ts | 2 +- src/DigNetwork/PropagationServer.ts | 36 ++++++++++++++++++----------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/DigNetwork/DigPeer.ts b/src/DigNetwork/DigPeer.ts index 940331d..16cb95b 100644 --- a/src/DigNetwork/DigPeer.ts +++ b/src/DigNetwork/DigPeer.ts @@ -185,6 +185,6 @@ export class DigPeer { public async downloadData(dataPath: string): Promise { - await this.propagationServer.downloadFile(dataPath); + await this.propagationServer.downloadFile(dataPath, dataPath); } } diff --git a/src/DigNetwork/PropagationServer.ts b/src/DigNetwork/PropagationServer.ts index 2f6bef2..925e80b 100644 --- a/src/DigNetwork/PropagationServer.ts +++ b/src/DigNetwork/PropagationServer.ts @@ -13,7 +13,11 @@ import { getFilePathFromSha256 } from "../utils/hashUtils"; import { createSpinner } from "nanospinner"; // Helper function to trim long filenames with ellipsis and ensure consistent padding -function formatFilename(filename: string, maxLength = 30): string { +function formatFilename(filename: string | undefined, maxLength = 30): string { + if (!filename) { + return "Unknown File".padEnd(maxLength, " "); // Provide a fallback value if filename is undefined + } + if (filename.length > maxLength) { // Trim the filename and add ellipsis return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " "); @@ -414,6 +418,7 @@ export class PropagationServer { * @returns {Promise} - The file content in memory as a Buffer. */ async fetchFile(dataPath: string): Promise { + console.log(cyan(`Fetching file ${dataPath}...`)); const config: AxiosRequestConfig = { responseType: "arraybuffer", // To store the file content in memory httpsAgent: this.createHttpsAgent(), @@ -423,7 +428,9 @@ export class PropagationServer { // Update progress bar const progressBar = PropagationServer.multiBar.create(totalLength, 0, { - dataPath: yellow(dataPath), + filename: yellow( + dataPath.replace("data", "").replace(/\\/g, "").replace(/\//g, "") + ), percentage: 0, }); @@ -493,9 +500,10 @@ export class PropagationServer { /** * Download a file from the server by sending a GET request. * Logs progress using cli-progress. + * @param {string} label - human readable key name. * @param {string} dataPath - The data path of the file to download. */ - async downloadFile(dataPath: string) { + async downloadFile(label: string, dataPath: string) { const config: AxiosRequestConfig = { responseType: "stream", // Make sure the response is streamed httpsAgent: this.createHttpsAgent(), @@ -513,11 +521,9 @@ export class PropagationServer { const response = await axios.get(url, config); const totalLength = parseInt(response.headers["content-length"], 10); - console.log(cyan(`Starting download for ${dataPath}...`)); - // Create a progress bar for the download const progressBar = PropagationServer.multiBar.create(totalLength, 0, { - dataPath: yellow(dataPath), + filename: yellow(label), percentage: 0, }); @@ -538,12 +544,6 @@ export class PropagationServer { fileWriteStream.on("finish", () => { progressBar.update(totalLength, { percentage: 100 }); progressBar.stop(); - console.log( - green( - ` - File ${dataPath} downloaded successfully to ${downloadPath}.` - ) - ); resolve(); }); @@ -582,7 +582,7 @@ export class PropagationServer { throw new Error(`Store ${storeId} does not exist.`); } - await propagationServer.downloadFile("height.json"); + await propagationServer.downloadFile("height.json", "height.json"); const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`); const root = JSON.parse(datFileContent.toString()); @@ -592,9 +592,17 @@ export class PropagationServer { "data" ); - await propagationServer.downloadFile(dataPath); + await propagationServer.downloadFile( + Buffer.from(fileKey, 'hex').toString("utf-8"), + dataPath + ); } + fs.writeFileSync( + path.join(STORE_PATH, storeId, `${rootHash}.dat`), + datFileContent + ); + const dataStore = DataStore.from(storeId); await dataStore.generateManifestFile(); From 35a5c212309217cbfc6677fc83bcc5e0c53cea2a Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 23 Sep 2024 20:45:32 -0400 Subject: [PATCH 2/3] feat: working progress bars --- package-lock.json | 238 ++-------------- package.json | 3 +- src/DigNetwork/PropagationServer.ts | 415 +++++++++++++++------------- src/blockchain/DataStore.ts | 11 - src/utils/asyncPool.ts | 29 ++ 5 files changed, 283 insertions(+), 413 deletions(-) create mode 100644 src/utils/asyncPool.ts diff --git a/package-lock.json b/package-lock.json index 9501f0d..43772f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "nconf": "^0.12.1", "node-cache": "^5.1.2", "p-limit": "^6.1.0", + "progress-stream": "^2.0.0", "superagent": "^10.0.0", "unzipper": "^0.12.3" }, @@ -45,6 +46,7 @@ "@types/mocha": "^10.0.7", "@types/nconf": "^0.10.7", "@types/node": "^22.1.0", + "@types/progress-stream": "^2.0.5", "@types/superagent": "^8.1.8", "@types/unzipper": "^0.10.10", "chai": "^5.1.1", @@ -784,6 +786,15 @@ "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true }, + "node_modules/@types/progress-stream": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/progress-stream/-/progress-stream-2.0.5.tgz", + "integrity": "sha512-5YNriuEZkHlFHHepLIaxzq3atGeav1qCTGzB74HKWpo66qjfostF+rHc785YYYHeBytve8ZG3ejg42jEIfXNiQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/readdir-glob": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", @@ -1320,17 +1331,6 @@ "node": ">=12" } }, - "node_modules/chalk": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", - "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", @@ -1644,20 +1644,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/cli-cursor": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", - "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", - "dependencies": { - "restore-cursor": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/cli-progress": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", @@ -2879,17 +2865,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/get-east-asian-width": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.2.0.tgz", - "integrity": "sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA==", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-func-name": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", @@ -3405,17 +3380,6 @@ "npm": ">=3" } }, - "node_modules/is-interactive": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", - "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -4004,17 +3968,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-function": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", - "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -4398,125 +4351,6 @@ "wrappy": "1" } }, - "node_modules/onetime": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", - "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", - "dependencies": { - "mimic-function": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/ora/-/ora-8.1.0.tgz", - "integrity": "sha512-GQEkNkH/GHOhPFXcqZs3IDahXEQcQxsSjEkK4KvEEST4t7eNzoMjxTzef+EZ+JluDEV+Raoi3WQ2CflnRdSVnQ==", - "dependencies": { - "chalk": "^5.3.0", - "cli-cursor": "^5.0.0", - "cli-spinners": "^2.9.2", - "is-interactive": "^2.0.0", - "is-unicode-supported": "^2.0.0", - "log-symbols": "^6.0.0", - "stdin-discarder": "^0.2.2", - "string-width": "^7.2.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ora/node_modules/emoji-regex": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.4.0.tgz", - "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==" - }, - "node_modules/ora/node_modules/is-unicode-supported": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", - "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/log-symbols": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", - "integrity": "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==", - "dependencies": { - "chalk": "^5.3.0", - "is-unicode-supported": "^1.3.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/log-symbols/node_modules/is-unicode-supported": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", - "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, "node_modules/os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -4729,6 +4563,15 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/progress-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/progress-stream/-/progress-stream-2.0.0.tgz", + "integrity": "sha512-xJwOWR46jcXUq6EH9yYyqp+I52skPySOeHfkxOZ2IY1AiBi/sFJhbhAKHoV3OTw/omQ45KTio9215dRJ2Yxd3Q==", + "dependencies": { + "speedometer": "~1.0.0", + "through2": "~2.0.3" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -4982,21 +4825,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/restore-cursor": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", - "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", - "dependencies": { - "onetime": "^7.0.0", - "signal-exit": "^4.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/run-async": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", @@ -5166,6 +4994,11 @@ "integrity": "sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==", "dev": true }, + "node_modules/speedometer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/speedometer/-/speedometer-1.0.0.tgz", + "integrity": "sha512-lgxErLl/7A5+vgIIXsh9MbeukOaCb2axgQ+bKCdIE+ibNT4XNYGNCR1qFEGq6F+YDASXK3Fh/c5FgtZchFolxw==" + }, "node_modules/split": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", @@ -5403,17 +5236,6 @@ "node": ">=10" } }, - "node_modules/stdin-discarder": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.2.2.tgz", - "integrity": "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/streamx": { "version": "2.20.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", @@ -5650,7 +5472,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", - "dev": true, "dependencies": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" @@ -5659,14 +5480,12 @@ "node_modules/through2/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, "node_modules/through2/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -5680,14 +5499,12 @@ "node_modules/through2/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/through2/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -5983,7 +5800,6 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true, "engines": { "node": ">=0.4" } diff --git a/package.json b/package.json index cfe915c..a235d3d 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "nanospinner": "^1.1.0", "nconf": "^0.12.1", "node-cache": "^5.1.2", - "p-limit": "^6.1.0", + "progress-stream": "^2.0.0", "superagent": "^10.0.0", "unzipper": "^0.12.3" }, @@ -61,6 +61,7 @@ "@types/mocha": "^10.0.7", "@types/nconf": "^0.10.7", "@types/node": "^22.1.0", + "@types/progress-stream": "^2.0.5", "@types/superagent": "^8.1.8", "@types/unzipper": "^0.10.10", "chai": "^5.1.1", diff --git a/src/DigNetwork/PropagationServer.ts b/src/DigNetwork/PropagationServer.ts index 925e80b..383f893 100644 --- a/src/DigNetwork/PropagationServer.ts +++ b/src/DigNetwork/PropagationServer.ts @@ -1,3 +1,4 @@ +// Import necessary modules and dependencies import axios, { AxiosRequestConfig } from "axios"; import fs from "fs"; import path from "path"; @@ -7,22 +8,21 @@ import { getOrCreateSSLCerts } from "../utils/ssl"; import { promptCredentials } from "../utils/credentialsUtils"; import https from "https"; import cliProgress from "cli-progress"; -import { green, red, cyan, yellow, blue } from "colorette"; // For colored output +import { green, red, blue, yellow } from "colorette"; import { STORE_PATH } from "../utils/config"; import { getFilePathFromSha256 } from "../utils/hashUtils"; import { createSpinner } from "nanospinner"; +import ProgressStream from "progress-stream"; // Helper function to trim long filenames with ellipsis and ensure consistent padding function formatFilename(filename: string | undefined, maxLength = 30): string { if (!filename) { - return "Unknown File".padEnd(maxLength, " "); // Provide a fallback value if filename is undefined + return "Unknown File".padEnd(maxLength, " "); } if (filename.length > maxLength) { - // Trim the filename and add ellipsis return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " "); } - // For shorter filenames, just pad to the maxLength return filename.padEnd(maxLength, " "); } @@ -34,56 +34,15 @@ export class PropagationServer { ipAddress: string; certPath: string; keyPath: string; - username: string | undefined; // To store username if needed for credentials - password: string | undefined; // To store password if needed for credentials + username: string | undefined; + password: string | undefined; private static readonly port = 4159; // Static port used for all requests - // Adjust cliProgress settings to align output properly and handle potential undefined values - // MultiBar for handling multiple progress bars with green progress and margin between bars - private static multiBar = new cliProgress.MultiBar( - { - clearOnComplete: false, - hideCursor: true, - format: (options, params, payload) => { - // Bar length and padding settings - const barCompleteChar = green(options.barCompleteChar || "="); // Green bar for completed portion - const barIncompleteChar = options.barIncompleteChar || "-"; // Default incomplete character - const barSize = options.barsize || 40; // Default size of the bar - - // Calculate the bar progress - const progressBar = `${barCompleteChar.repeat( - Math.round(params.progress * barSize) - )}${barIncompleteChar.repeat( - barSize - Math.round(params.progress * barSize) - )}`; - - // Calculate the percentage manually - const percentage = Math.round((params.value / params.total) * 100); - - // Format the filename to a fixed length - const formattedFilename = formatFilename(payload.filename, 30); // Trim to max 30 chars - - // Padding the filename, percentage, and size - const percentageStr = `${percentage}%`.padEnd(4); // Ensure percentage is always 7 characters wide - const size = `${params.value}/${params.total} bytes`.padEnd(20); // Ensure size is always 20 characters wide - - // Return the complete formatted progress bar with padding - return `${progressBar} | ${formattedFilename.padEnd( - 35 - )} | ${percentageStr} | ${size}`; - }, - stopOnComplete: true, - barsize: 40, - align: "left", - }, - cliProgress.Presets.shades_classic - ); - constructor(ipAddress: string, storeId: string) { this.storeId = storeId; - this.sessionId = ""; // Session ID will be set after starting the upload session - this.publicKey = ""; // Public key will be set after initializing the wallet + this.sessionId = ""; + this.publicKey = ""; this.ipAddress = ipAddress; // Get or create SSL certificates @@ -97,7 +56,9 @@ export class PropagationServer { */ async initializeWallet() { this.wallet = await Wallet.load("default"); - this.publicKey = await this.wallet.getPublicSyntheticKey().toString("hex"); + this.publicKey = ( + await this.wallet.getPublicSyntheticKey() + ).toString("hex"); } /** @@ -107,15 +68,12 @@ export class PropagationServer { return new https.Agent({ cert: fs.readFileSync(this.certPath), key: fs.readFileSync(this.keyPath), - rejectUnauthorized: false, // Allow self-signed certificates + rejectUnauthorized: false, }); } /** * Check if the store and optional root hash exist by making a HEAD request. - * - * @param {string} [rootHash] - Optional root hash to check for existence. - * @returns {Promise<{ storeExists: boolean, rootHashExists: boolean }>} - An object indicating if the store and root hash exist. */ async checkStoreExists( rootHash?: string @@ -161,9 +119,6 @@ export class PropagationServer { /** * Start an upload session by sending a POST request with the rootHash.dat file. - * - * @param {string} rootHash - The root hash used to name the .dat file. - * @param {string} datFilePath - The full path to the rootHash.dat file. */ async startUploadSession(rootHash: string) { const spinner = createSpinner( @@ -172,11 +127,7 @@ export class PropagationServer { try { const formData = new FormData(); - const datFilePath = path.join( - STORE_PATH, - this.storeId, - `${rootHash}.dat` - ); + const datFilePath = path.join(STORE_PATH, this.storeId, `${rootHash}.dat`); // Ensure the rootHash.dat file exists if (!fs.existsSync(datFilePath)) { @@ -220,7 +171,6 @@ export class PropagationServer { /** * Request a nonce for a file by sending a HEAD request to the server. - * Checks if the file already exists based on the 'x-file-exists' header. */ async getFileNonce( filename: string @@ -251,7 +201,7 @@ export class PropagationServer { /** * Upload a file to the server by sending a PUT request. - * Logs progress using cli-progress for each file. + * Logs progress using a local cli-progress bar. */ async uploadFile(label: string, dataPath: string) { const filePath = path.join(STORE_PATH, this.storeId, dataPath); @@ -267,58 +217,83 @@ export class PropagationServer { const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce); const publicKey = await wallet.getPublicSyntheticKey(); - const formData = new FormData(); - formData.append("file", fs.createReadStream(filePath)); - + // Get the file size const fileSize = fs.statSync(filePath).size; - // Create a new progress bar for each file - const progressBar = PropagationServer.multiBar.create(fileSize, 0, { - filename: yellow(path.basename(label)), - percentage: 0, - }); + let progressBar: cliProgress.SingleBar | undefined; - let uploadedBytes = 0; + try { + // Create a new progress bar for the file + progressBar = new cliProgress.SingleBar( + { + format: `${blue('[{bar}]')} | ${yellow('{filename}')} | {percentage}% | {value}/{total} bytes`, + hideCursor: true, + barsize: 30, + align: "left", + autopadding: true, + noTTYOutput: false, + stopOnComplete: true, + clearOnComplete: false, + }, + cliProgress.Presets.legacy + ); - const config: AxiosRequestConfig = { - headers: { - "Content-Type": "multipart/form-data", + progressBar.start(fileSize, 0, { + filename: formatFilename(path.basename(label)), + }); + + // Create a progress stream + const progressStream = ProgressStream({ + length: fileSize, + time: 500, // Adjust as needed + }); + + progressStream.on("progress", (progress) => { + progressBar!.update(progress.transferred); + }); + + // Create a read stream and pipe it through the progress stream + const fileReadStream = fs + .createReadStream(filePath) + .pipe(progressStream); + + // Use form-data to construct the request body + const formData = new FormData(); + formData.append("file", fileReadStream); + + const headers = { + ...formData.getHeaders(), "x-nonce": nonce, "x-public-key": publicKey.toString("hex"), "x-key-ownership-sig": keyOwnershipSig, - ...formData.getHeaders(), - }, - httpsAgent: this.createHttpsAgent(), + }; - // Tracking upload progress and updating the progress bar - onUploadProgress: (progressEvent: any) => { - const uploadedBytes = progressEvent.loaded; - const percentage = Math.round( - (100 * progressEvent.loaded) / progressEvent.total - ); - progressBar.update(uploadedBytes, { percentage }); - }, - }; + const config: AxiosRequestConfig = { + headers: headers, + httpsAgent: this.createHttpsAgent(), + maxContentLength: Infinity, + maxBodyLength: Infinity, + }; - try { const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`; const response = await axios.put(url, formData, config); - // Complete the progress bar - progressBar.update(fileSize, { percentage: 100 }); - progressBar.stop(); - - return response.data; + // Wait for the progress stream to finish + await new Promise((resolve, reject) => { + progressStream.on("end", resolve); + progressStream.on("error", reject); + }); } catch (error: any) { - console.error(red(`✖ Error uploading file ${label}:`), error.message); - progressBar.stop(); // Stop the progress bar in case of error throw error; + } finally { + if (progressBar) { + progressBar.stop(); + } } } /** * Commit the upload session by sending a POST request to the server. - * This finalizes the upload and moves files from the temporary session directory to the permanent location. */ async commitUploadSession() { const spinner = createSpinner( @@ -354,10 +329,6 @@ export class PropagationServer { /** * Static function to handle the entire upload process for multiple files based on rootHash. - * @param {string} storeId - The ID of the DataStore. - * @param {string} rootHash - The root hash used to derive the file set. - * @param {string} publicKey - The public key of the user. - * @param {string} ipAddress - The IP address of the server. */ static async uploadStore( storeId: string, @@ -398,10 +369,21 @@ export class PropagationServer { const dataStore = DataStore.from(storeId); const files = await dataStore.getFileSetForRootHash(rootHash); - // Upload each file - for (const file of files) { - await propagationServer.uploadFile(file.name, file.path); - } + // Prepare upload tasks + const uploadTasks = files.map((file) => ({ + label: file.name, + dataPath: file.path, + })); + + // Limit the number of concurrent uploads + const concurrencyLimit = 3; // Adjust this number as needed + + // Import asyncPool from your utilities + const { asyncPool } = await import("../utils/asyncPool"); + + await asyncPool(concurrencyLimit, uploadTasks, async (task) => { + await propagationServer.uploadFile(task.label, task.dataPath); + }); // Commit the session after all files have been uploaded await propagationServer.commitUploadSession(); @@ -413,59 +395,82 @@ export class PropagationServer { /** * Fetch a file from the server by sending a GET request and return its content in memory. - * Logs progress using cli-progress. - * @param {string} dataPath - The data path of the file to download. - * @returns {Promise} - The file content in memory as a Buffer. + * Logs progress using a local cli-progress bar. */ async fetchFile(dataPath: string): Promise { - console.log(cyan(`Fetching file ${dataPath}...`)); + const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`; const config: AxiosRequestConfig = { - responseType: "arraybuffer", // To store the file content in memory + responseType: "stream", httpsAgent: this.createHttpsAgent(), - onDownloadProgress: (progressEvent) => { - const totalLength = progressEvent.total || 0; - const downloadedBytes = progressEvent.loaded; - - // Update progress bar - const progressBar = PropagationServer.multiBar.create(totalLength, 0, { - filename: yellow( - dataPath.replace("data", "").replace(/\\/g, "").replace(/\//g, "") - ), - percentage: 0, - }); - - progressBar.update(downloadedBytes, { - percentage: Math.round((downloadedBytes / totalLength) * 100), - }); - - if (downloadedBytes === totalLength) { - progressBar.update(totalLength, { percentage: 100 }); - progressBar.stop(); - console.log(green(`✔ File ${dataPath} fetched successfully.`)); - } - }, }; - const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`; + let progressBar: cliProgress.SingleBar | undefined; try { const response = await axios.get(url, config); + const totalLengthHeader = response.headers["content-length"]; + const totalLength = totalLengthHeader + ? parseInt(totalLengthHeader, 10) + : null; + + if (!totalLength) { + throw new Error("Content-Length header is missing"); + } + + // Create a new progress bar for the file + progressBar = new cliProgress.SingleBar( + { + format: `${blue('[{bar}]')} | ${yellow('{filename}')} | {percentage}% | {value}/{total} bytes`, + hideCursor: true, + barsize: 30, + align: "left", + autopadding: true, + noTTYOutput: false, + stopOnComplete: true, + clearOnComplete: false, + }, + cliProgress.Presets.legacy + ); + + progressBar.start(totalLength, 0, { + filename: formatFilename(dataPath), + }); + + let dataBuffers: Buffer[] = []; + + const progressStream = ProgressStream({ + length: totalLength, + time: 500, // Adjust as needed + }); + + progressStream.on("progress", (progress) => { + progressBar!.update(progress.transferred); + }); - // Return the file contents as a Buffer - return Buffer.from(response.data); + response.data.pipe(progressStream); + + progressStream.on("data", (chunk: Buffer) => { + dataBuffers.push(chunk); + }); + + // Wait for the progress stream to finish + await new Promise((resolve, reject) => { + progressStream.on("end", resolve); + progressStream.on("error", reject); + }); + + return Buffer.concat(dataBuffers); } catch (error) { - console.error(red(`✖ Error fetching file ${dataPath}:`), error); throw error; + } finally { + if (progressBar) { + progressBar.stop(); + } } } /** * Get details of a file, including whether it exists and its size. - * Makes a HEAD request to the server and checks the response headers. - * - * @param {string} dataPath - The path of the file within the DataStore. - * @param {string} rootHash - The root hash associated with the DataStore. - * @returns {Promise<{ exists: boolean; size: number }>} - An object containing file existence and size information. */ async getFileDetails( dataPath: string, @@ -476,7 +481,6 @@ export class PropagationServer { httpsAgent: this.createHttpsAgent(), }; - // Construct the URL for the HEAD request to check file details const url = `https://${this.ipAddress}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`; const response = await axios.head(url, config); @@ -486,7 +490,7 @@ export class PropagationServer { return { exists: fileExists, - size: fileExists ? fileSize : 0, // Return 0 size if file doesn't exist + size: fileExists ? fileSize : 0, }; } catch (error: any) { console.error( @@ -499,71 +503,87 @@ export class PropagationServer { /** * Download a file from the server by sending a GET request. - * Logs progress using cli-progress. - * @param {string} label - human readable key name. - * @param {string} dataPath - The data path of the file to download. + * Logs progress using a local cli-progress bar. */ async downloadFile(label: string, dataPath: string) { - const config: AxiosRequestConfig = { - responseType: "stream", // Make sure the response is streamed - httpsAgent: this.createHttpsAgent(), - }; - const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`; - const downloadPath = path.join(STORE_PATH, this.storeId, dataPath); // Save to the correct store directory + const downloadPath = path.join(STORE_PATH, this.storeId, dataPath); // Ensure that the directory for the file exists fs.mkdirSync(path.dirname(downloadPath), { recursive: true }); - const fileWriteStream = fs.createWriteStream(downloadPath); + const config: AxiosRequestConfig = { + responseType: "stream", + httpsAgent: this.createHttpsAgent(), + }; + + let progressBar: cliProgress.SingleBar | undefined; try { const response = await axios.get(url, config); - const totalLength = parseInt(response.headers["content-length"], 10); + const totalLengthHeader = response.headers["content-length"]; + const totalLength = totalLengthHeader + ? parseInt(totalLengthHeader, 10) + : null; - // Create a progress bar for the download - const progressBar = PropagationServer.multiBar.create(totalLength, 0, { - filename: yellow(label), - percentage: 0, - }); + if (!totalLength) { + throw new Error("Content-Length header is missing"); + } - let downloadedBytes = 0; + // Create a new progress bar for the file + progressBar = new cliProgress.SingleBar( + { + format: `${blue('[{bar}]')} | ${yellow('{filename}')} | {percentage}% | {value}/{total} bytes`, + hideCursor: true, + barsize: 30, + align: "left", + autopadding: true, + noTTYOutput: false, + stopOnComplete: true, + clearOnComplete: false, + }, + cliProgress.Presets.legacy + ); - // Pipe the response data to the file stream and track progress - response.data.on("data", (chunk: Buffer) => { - downloadedBytes += chunk.length; - progressBar.update(downloadedBytes, { - percentage: Math.round((downloadedBytes / totalLength) * 100), - }); + progressBar.start(totalLength, 0, { + filename: formatFilename(label), }); - // Pipe the data into the file and finalize - response.data.pipe(fileWriteStream); + const fileWriteStream = fs.createWriteStream(downloadPath); - return new Promise((resolve, reject) => { - fileWriteStream.on("finish", () => { - progressBar.update(totalLength, { percentage: 100 }); - progressBar.stop(); - resolve(); - }); + const progressStream = ProgressStream({ + length: totalLength, + time: 500, // Adjust as needed + }); - fileWriteStream.on("error", (error) => { - progressBar.stop(); - console.error(red(`✖ Error downloading file ${dataPath}:`), error); - reject(error); - }); + progressStream.on("progress", (progress) => { + progressBar!.update(progress.transferred); }); + + response.data.pipe(progressStream).pipe(fileWriteStream); + + // Wait for both the file write stream and the progress stream to finish + await Promise.all([ + new Promise((resolve, reject) => { + fileWriteStream.on("finish", resolve); + fileWriteStream.on("error", reject); + }), + new Promise((resolve, reject) => { + progressStream.on("end", resolve); + progressStream.on("error", reject); + }), + ]); } catch (error) { - console.error(red(`✖ Error downloading file ${dataPath}:`), error); throw error; + } finally { + if (progressBar) { + progressBar.stop(); + } } } /** * Static function to handle downloading multiple files from a DataStore based on file paths. - * @param {string} storeId - The ID of the DataStore. - * @param {string[]} dataPaths - The list of data paths to download. - * @param {string} ipAddress - The IP address of the server. */ static async downloadStore( storeId: string, @@ -582,22 +602,37 @@ export class PropagationServer { throw new Error(`Store ${storeId} does not exist.`); } - await propagationServer.downloadFile("height.json", "height.json"); - const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`); + // Fetch the rootHash.dat file + const datFileContent = await propagationServer.fetchFile( + `${rootHash}.dat` + ); const root = JSON.parse(datFileContent.toString()); + // Prepare download tasks + const downloadTasks = []; + for (const [fileKey, fileData] of Object.entries(root.files)) { const dataPath = getFilePathFromSha256( - root.files[fileKey].sha256, + (fileData as any).sha256, "data" ); - await propagationServer.downloadFile( - Buffer.from(fileKey, 'hex').toString("utf-8"), - dataPath - ); + const label = Buffer.from(fileKey, "hex").toString("utf-8"); + + downloadTasks.push({ label, dataPath }); } + // Limit the number of concurrent downloads + const concurrencyLimit = 5; // Adjust this number as needed + + // Import asyncPool from your utilities + const { asyncPool } = await import("../utils/asyncPool"); + + await asyncPool(concurrencyLimit, downloadTasks, async (task) => { + await propagationServer.downloadFile(task.label, task.dataPath); + }); + + // Save the rootHash.dat file fs.writeFileSync( path.join(STORE_PATH, storeId, `${rootHash}.dat`), datFileContent diff --git a/src/blockchain/DataStore.ts b/src/blockchain/DataStore.ts index fd0ccb9..7411c83 100644 --- a/src/blockchain/DataStore.ts +++ b/src/blockchain/DataStore.ts @@ -601,20 +601,9 @@ export class DataStore { const datFilePath = path.join(STORE_PATH, this.storeId, `${rootHash}.dat`); const datFileContent = JSON.parse(fs.readFileSync(datFilePath, "utf-8")); - const heightDatFilePath = path.join( - STORE_PATH, - this.storeId, - "height.json" - ); - // Use a Set to ensure uniqueness const filesInvolved = new Set<{ name: string; path: string }>(); - filesInvolved.add({ - name: "height.json", - path: "height.json", - }); - // Iterate over each file and perform an integrity check for (const [fileKey, fileData] of Object.entries(datFileContent.files)) { const filePath = getFilePathFromSha256( diff --git a/src/utils/asyncPool.ts b/src/utils/asyncPool.ts new file mode 100644 index 0000000..31fe0f2 --- /dev/null +++ b/src/utils/asyncPool.ts @@ -0,0 +1,29 @@ + +/** + * Processes items in sequential batches with a concurrency limit. + * @param {number} limit - The maximum number of concurrent executions per batch. + * @param {Array} items - The array of items to process. + * @param {(item: T) => Promise} iteratorFn - The async function to apply to each item. + * @returns {Promise>} - A promise that resolves when all items have been processed. + */ +export async function asyncPool( + limit: number, + items: T[], + iteratorFn: (item: T) => Promise + ): Promise { + const ret: R[] = []; + + for (let i = 0; i < items.length; i += limit) { + const batchItems = items.slice(i, i + limit); + const batchPromises = batchItems.map((item) => iteratorFn(item)); + + // Wait for the current batch to complete before starting the next one + const batchResults = await Promise.all(batchPromises); + ret.push(...batchResults); + + // Optional: add a cooldown between batches + // await new Promise((resolve) => setTimeout(resolve, 500)); + } + + return ret; + } \ No newline at end of file From 1b16bba007982476ca36539998a70734bde8c086 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 23 Sep 2024 20:46:28 -0400 Subject: [PATCH 3/3] chore(release): 0.0.1-alpha.72 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5e978..98e7bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.72](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.71...v0.0.1-alpha.72) (2024-09-24) + + +### Features + +* working progress bars ([35a5c21](https://github.com/DIG-Network/dig-chia-sdk/commit/35a5c212309217cbfc6677fc83bcc5e0c53cea2a)) + ### [0.0.1-alpha.71](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.70...v0.0.1-alpha.71) (2024-09-23) diff --git a/package-lock.json b/package-lock.json index 43772f9..e12dda8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.71", + "version": "0.0.1-alpha.72", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.71", + "version": "0.0.1-alpha.72", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.24", diff --git a/package.json b/package.json index a235d3d..c259b29 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.71", + "version": "0.0.1-alpha.72", "description": "", "type": "commonjs", "main": "./dist/index.js",