From 6cfb2cc6118de2bda531f7854c35e33f07af744a Mon Sep 17 00:00:00 2001 From: YR Chen Date: Tue, 23 Jul 2024 19:22:53 +0800 Subject: [PATCH] winget-source: migrate from `Promise` to `async/await` (#119) * winget-source: migrate from `Promise` to `async/await` * winget-source: log when no update available --- winget-source/package-lock.json | 13 +++++- winget-source/package.json | 1 + winget-source/sync-repo.js | 82 ++++++++++++++++++++------------- winget-source/utilities.js | 57 +++++++++++------------ 4 files changed, 89 insertions(+), 64 deletions(-) diff --git a/winget-source/package-lock.json b/winget-source/package-lock.json index 5935b72..dcf8987 100644 --- a/winget-source/package-lock.json +++ b/winget-source/package-lock.json @@ -1,18 +1,19 @@ { "name": "@ustcmirror/winget-source", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ustcmirror/winget-source", - "version": "1.0.0", + "version": "1.1.0", "license": "MIT", "dependencies": { "async": "^3.2.4", "fetch-retry": "^5.0.4", "jszip": "^3.10.1", "node-fetch": "^3.3.1", + "promised-sqlite3": "^2.1.0", "sqlite3": "^5.1.5", "winston": "^3.8.2" } @@ -1233,6 +1234,14 @@ "node": ">=10" } }, + "node_modules/promised-sqlite3": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/promised-sqlite3/-/promised-sqlite3-2.1.0.tgz", + "integrity": "sha512-g227r1cE/GrP7UfQdiwi1URAc7HL2LAulNrDC0vYMWcI387Urc5VoIinoXfXWDi546sZUM7gQhxrNr9a9nXN1Q==", + "peerDependencies": { + "sqlite3": "^5" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", diff --git a/winget-source/package.json b/winget-source/package.json index aa01e63..84d9dbe 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -11,6 +11,7 @@ "fetch-retry": "^5.0.4", "jszip": "^3.10.1", "node-fetch": "^3.3.1", + "promised-sqlite3": "^2.1.0", "sqlite3": "^5.1.5", "winston": "^3.8.2" }, diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index e79cd5c..920b464 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -2,12 +2,13 @@ import assert from 'assert' import async from 'async' import { rm } from 'fs/promises' -import { EX_IOERR, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js' +import { AsyncDatabase } from 'promised-sqlite3' +import { EX_IOERR, EX_OK, EX_SOFTWARE, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js' import { buildPathpartMap, buildURIList, - exitOnError, + exitWithCode, extractDatabaseFromBundle, getLocalPath, makeTempDirectory, @@ -17,38 +18,57 @@ import { } from './utilities.js' +const sourceV1Filename = 'source.msix'; +const sourceV2Filename = 'source2.msix'; + +// set up configs and temp directory const { parallelLimit, remote, sqlite3, winston } = setupEnvironment(); +const tempDirectory = await makeTempDirectory('winget-repo-'); winston.info(`start syncing with ${remote}`); -const sourceV1Filename = 'source.msix'; -const sourceV2Filename = 'source2.msix'; +try { + // download V1 index package to buffer + const [indexBuffer, modifiedDate, updated] = await syncFile(sourceV1Filename, true, false); + if (!updated) { + winston.info(`nothing to sync from ${remote}`); + exitWithCode(EX_OK); + } + assert(indexBuffer !== null, "Failed to get the source index buffer!"); -syncFile(sourceV1Filename, true, false).catch(exitOnError(EX_UNAVAILABLE)).then(async result => { - assert(result, "Failed to catch error when syncing source index!"); - const [indexBuffer, modifiedDate, synced] = result; - if (synced) { - assert(indexBuffer !== null, "Failed to get the source index buffer!"); - const temp = await makeTempDirectory('winget-repo-'); - const database = await extractDatabaseFromBundle(indexBuffer, temp); - const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY, exitOnError(EX_IOERR)); - - db.all('SELECT * FROM pathparts', (error, rows) => { - const pathparts = buildPathpartMap(error, rows); - db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC', (error, rows) => { - db.close(); - const uris = buildURIList(error, rows, pathparts); - const download = async (uri) => await syncFile(uri, false); - async.eachLimit(uris, parallelLimit, download, (error) => { - rm(temp, { recursive: true }); - exitOnError(EX_TEMPFAIL)(error); - saveFile(getLocalPath(sourceV1Filename), indexBuffer, modifiedDate).then(_ => - syncFile(sourceV2Filename, true) - ).then(_ => { - winston.info(`successfully synced with ${remote}`); - }); - }); - }); - }); + // unpack, extract and load index database + try { + const databaseFilePath = await extractDatabaseFromBundle(indexBuffer, tempDirectory); + const rawDatabase = new sqlite3.Database(databaseFilePath, sqlite3.OPEN_READONLY); + + // read manifest URIs from index database + try { + const db = new AsyncDatabase(rawDatabase) + const pathparts = buildPathpartMap(await db.all('SELECT * FROM pathparts')); + const uris = buildURIList(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts); + await db.close() + + // sync latest manifests in parallel + try { + await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, false)); + } catch (error) { + exitWithCode(EX_TEMPFAIL, error); + } + } catch (error) { + exitWithCode(EX_SOFTWARE, error); + } + } catch (error) { + exitWithCode(EX_IOERR, error); } -}); + + // update index packages + await saveFile(getLocalPath(sourceV1Filename), indexBuffer, modifiedDate); + await syncFile(sourceV2Filename, true); +} catch (error) { + exitWithCode(EX_UNAVAILABLE, error); +} + +winston.info(`successfully synced with ${remote}`); + +// clean up temp directory +await rm(tempDirectory, { recursive: true }); diff --git a/winget-source/utilities.js b/winget-source/utilities.js index bac769a..4bba127 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -7,10 +7,12 @@ import path from 'path' import process from 'process' import sqlite3 from 'sqlite3' import winston from 'winston' + import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' import { isIP } from 'net' -import { EX_IOERR, EX_SOFTWARE, EX_USAGE } from './sysexits.js' + +import { EX_IOERR, EX_USAGE } from './sysexits.js' /** @@ -89,7 +91,9 @@ function getContentLength(response) { */ function resolvePathpart(id, pathparts) { const pathpart = pathparts.get(id); - if (pathpart === undefined) return ''; + if (pathpart === undefined) { + return ''; + } return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart); } @@ -119,13 +123,11 @@ function setupWinstonLogger() { /** * Build a local storage for path parts from database query. * - * @param {Error?} error Database error thrown from the query, if any. * @param {{ rowid: number, parent: number, pathpart: string }[]} rows Rows returned by the query. * * @returns {Map} In-memory path part storage to query against. */ -export function buildPathpartMap(error, rows) { - exitOnError(EX_SOFTWARE)(error); +export function buildPathpartMap(rows) { return new Map(rows.map(row => [row.rowid, { parent: row.parent, pathpart: row.pathpart }] )); @@ -134,32 +136,29 @@ export function buildPathpartMap(error, rows) { /** * Build a list of all manifest URIs from database query. * - * @param {Error?} error Database error thrown from the query, if any. * @param {{ pathpart: string, [key: string]: string }[]} rows Rows returned by the query. * @param {Map} pathparts Path part storage built from the database. * * @returns {string[]} Manifest URIs to sync. */ -export function buildURIList(error, rows, pathparts) { - exitOnError(EX_SOFTWARE)(error); +export function buildURIList(rows, pathparts) { return rows.map(row => resolvePathpart(row.pathpart, pathparts)); } /** - * Get an error handling function that logs an error and exits with given status if it occurs. + * Exit with given status with error logging. * - * @param {number} code Exit code to use if there's an error. + * @param {number} code Exit code to use. + * @param {Error | string | null | undefined} error Error to log. * - * @returns {(err: Error | string | null | undefined) => void} Function that handles a possible error. + * @returns {never} Exits the process. */ -export function exitOnError(code = 1) { - return (error) => { - if (error) { - winston.exitOnError = false; - winston.error(error); - process.exit(code); - } - }; +export function exitWithCode(code = 0, error = undefined) { + if (error) { + winston.exitOnError = false; + winston.error(error); + } + process.exit(code); } /** @@ -171,16 +170,12 @@ export function exitOnError(code = 1) { * @returns {Promise} Path of the extracted `index.db` file. */ export async function extractDatabaseFromBundle(msixFile, directory) { - try { - const bundle = (msixFile instanceof Buffer) ? msixFile : await readFile(msixFile); - const zip = await JSZip.loadAsync(bundle); - const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array'); - const destination = path.join(directory, 'index.db'); - await writeFile(destination, buffer); - return destination; - } catch (error) { - exitOnError(EX_IOERR)(error); - } + const bundle = (msixFile instanceof Buffer) ? msixFile : await readFile(msixFile); + const zip = await JSZip.loadAsync(bundle); + const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array'); + const destination = path.join(directory, 'index.db'); + await writeFile(destination, buffer); + return destination; } /** @@ -218,7 +213,7 @@ export async function makeTempDirectory(prefix) { try { return await mkdtemp(path.join(os.tmpdir(), prefix)); } catch (error) { - exitOnError(EX_IOERR)(error); + exitWithCode(EX_IOERR, error); } } @@ -230,7 +225,7 @@ export async function makeTempDirectory(prefix) { export function setupEnvironment() { setupWinstonLogger(); if (!local) { - exitOnError(EX_USAGE)("destination path $TO not set!"); + exitWithCode(EX_USAGE, "destination path $TO not set!"); } if (localAddress) { https.globalAgent.options.localAddress = localAddress;