Skip to content

Commit

Permalink
winget-source: migrate from Promise to async/await (#119)
Browse files Browse the repository at this point in the history 
* winget-source: migrate from `Promise` to `async/await`

* winget-source: log when no update available
  • Loading branch information
@stevapple
stevapple authored Jul 23, 2024
1 parent 64f6c57 commit 6cfb2cc
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 64 deletions.
13 changes: 11 additions & 2 deletions winget-source/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions winget-source/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"fetch-retry": "^5.0.4",
"jszip": "^3.10.1",
"node-fetch": "^3.3.1",
"promised-sqlite3": "^2.1.0",
"sqlite3": "^5.1.5",
"winston": "^3.8.2"
},
Expand Down
82 changes: 51 additions & 31 deletions winget-source/sync-repo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ import assert from 'assert'
import async from 'async'

import { rm } from 'fs/promises'
import { EX_IOERR, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js'
import { AsyncDatabase } from 'promised-sqlite3'
import { EX_IOERR, EX_OK, EX_SOFTWARE, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js'

import {
buildPathpartMap,
buildURIList,
exitOnError,
exitWithCode,
extractDatabaseFromBundle,
getLocalPath,
makeTempDirectory,
Expand All @@ -17,38 +18,57 @@ import {
} from './utilities.js'


const sourceV1Filename = 'source.msix';
const sourceV2Filename = 'source2.msix';

// set up configs and temp directory
const { parallelLimit, remote, sqlite3, winston } = setupEnvironment();
const tempDirectory = await makeTempDirectory('winget-repo-');

winston.info(`start syncing with ${remote}`);

const sourceV1Filename = 'source.msix';
const sourceV2Filename = 'source2.msix';
try {
// download V1 index package to buffer
const [indexBuffer, modifiedDate, updated] = await syncFile(sourceV1Filename, true, false);
if (!updated) {
winston.info(`nothing to sync from ${remote}`);
exitWithCode(EX_OK);
}
assert(indexBuffer !== null, "Failed to get the source index buffer!");

syncFile(sourceV1Filename, true, false).catch(exitOnError(EX_UNAVAILABLE)).then(async result => {
assert(result, "Failed to catch error when syncing source index!");
const [indexBuffer, modifiedDate, synced] = result;
if (synced) {
assert(indexBuffer !== null, "Failed to get the source index buffer!");
const temp = await makeTempDirectory('winget-repo-');
const database = await extractDatabaseFromBundle(indexBuffer, temp);
const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY, exitOnError(EX_IOERR));

db.all('SELECT * FROM pathparts', (error, rows) => {
const pathparts = buildPathpartMap(error, rows);
db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC', (error, rows) => {
db.close();
const uris = buildURIList(error, rows, pathparts);
const download = async (uri) => await syncFile(uri, false);
async.eachLimit(uris, parallelLimit, download, (error) => {
rm(temp, { recursive: true });
exitOnError(EX_TEMPFAIL)(error);
saveFile(getLocalPath(sourceV1Filename), indexBuffer, modifiedDate).then(_ =>
syncFile(sourceV2Filename, true)
).then(_ => {
winston.info(`successfully synced with ${remote}`);
});
});
});
});
// unpack, extract and load index database
try {
const databaseFilePath = await extractDatabaseFromBundle(indexBuffer, tempDirectory);
const rawDatabase = new sqlite3.Database(databaseFilePath, sqlite3.OPEN_READONLY);

// read manifest URIs from index database
try {
const db = new AsyncDatabase(rawDatabase)
const pathparts = buildPathpartMap(await db.all('SELECT * FROM pathparts'));
const uris = buildURIList(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts);
await db.close()

// sync latest manifests in parallel
try {
await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, false));
} catch (error) {
exitWithCode(EX_TEMPFAIL, error);
}
} catch (error) {
exitWithCode(EX_SOFTWARE, error);
}
} catch (error) {
exitWithCode(EX_IOERR, error);
}
});

// update index packages
await saveFile(getLocalPath(sourceV1Filename), indexBuffer, modifiedDate);
await syncFile(sourceV2Filename, true);
} catch (error) {
exitWithCode(EX_UNAVAILABLE, error);
}

winston.info(`successfully synced with ${remote}`);

// clean up temp directory
await rm(tempDirectory, { recursive: true });
57 changes: 26 additions & 31 deletions winget-source/utilities.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import path from 'path'
import process from 'process'
import sqlite3 from 'sqlite3'
import winston from 'winston'

import { existsSync } from 'fs'
import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises'
import { isIP } from 'net'
import { EX_IOERR, EX_SOFTWARE, EX_USAGE } from './sysexits.js'

import { EX_IOERR, EX_USAGE } from './sysexits.js'


/**
Expand Down Expand Up @@ -89,7 +91,9 @@ function getContentLength(response) {
*/
function resolvePathpart(id, pathparts) {
const pathpart = pathparts.get(id);
if (pathpart === undefined) return '';
if (pathpart === undefined) {
return '';
}
return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart);
}

Expand Down Expand Up @@ -119,13 +123,11 @@ function setupWinstonLogger() {
/**
* Build a local storage for path parts from database query.
*
* @param {Error?} error Database error thrown from the query, if any.
* @param {{ rowid: number, parent: number, pathpart: string }[]} rows Rows returned by the query.
*
* @returns {Map<number, { parent: number, pathpart: string }>} In-memory path part storage to query against.
*/
export function buildPathpartMap(error, rows) {
exitOnError(EX_SOFTWARE)(error);
export function buildPathpartMap(rows) {
return new Map(rows.map(row =>
[row.rowid, { parent: row.parent, pathpart: row.pathpart }]
));
Expand All @@ -134,32 +136,29 @@ export function buildPathpartMap(error, rows) {
/**
* Build a list of all manifest URIs from database query.
*
* @param {Error?} error Database error thrown from the query, if any.
* @param {{ pathpart: string, [key: string]: string }[]} rows Rows returned by the query.
* @param {Map<number, { parent: number, pathpart: string }>} pathparts Path part storage built from the database.
*
* @returns {string[]} Manifest URIs to sync.
*/
export function buildURIList(error, rows, pathparts) {
exitOnError(EX_SOFTWARE)(error);
export function buildURIList(rows, pathparts) {
return rows.map(row => resolvePathpart(row.pathpart, pathparts));
}

/**
* Get an error handling function that logs an error and exits with given status if it occurs.
* Exit with given status with error logging.
*
* @param {number} code Exit code to use if there's an error.
* @param {number} code Exit code to use.
* @param {Error | string | null | undefined} error Error to log.
*
* @returns {(err: Error | string | null | undefined) => void} Function that handles a possible error.
* @returns {never} Exits the process.
*/
export function exitOnError(code = 1) {
return (error) => {
if (error) {
winston.exitOnError = false;
winston.error(error);
process.exit(code);
}
};
export function exitWithCode(code = 0, error = undefined) {
if (error) {
winston.exitOnError = false;
winston.error(error);
}
process.exit(code);
}

/**
Expand All @@ -171,16 +170,12 @@ export function exitOnError(code = 1) {
* @returns {Promise<string>} Path of the extracted `index.db` file.
*/
export async function extractDatabaseFromBundle(msixFile, directory) {
try {
const bundle = (msixFile instanceof Buffer) ? msixFile : await readFile(msixFile);
const zip = await JSZip.loadAsync(bundle);
const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array');
const destination = path.join(directory, 'index.db');
await writeFile(destination, buffer);
return destination;
} catch (error) {
exitOnError(EX_IOERR)(error);
}
const bundle = (msixFile instanceof Buffer) ? msixFile : await readFile(msixFile);
const zip = await JSZip.loadAsync(bundle);
const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array');
const destination = path.join(directory, 'index.db');
await writeFile(destination, buffer);
return destination;
}

/**
Expand Down Expand Up @@ -218,7 +213,7 @@ export async function makeTempDirectory(prefix) {
try {
return await mkdtemp(path.join(os.tmpdir(), prefix));
} catch (error) {
exitOnError(EX_IOERR)(error);
exitWithCode(EX_IOERR, error);
}
}

Expand All @@ -230,7 +225,7 @@ export async function makeTempDirectory(prefix) {
export function setupEnvironment() {
setupWinstonLogger();
if (!local) {
exitOnError(EX_USAGE)("destination path $TO not set!");
exitWithCode(EX_USAGE, "destination path $TO not set!");
}
if (localAddress) {
https.globalAgent.options.localAddress = localAddress;
Expand Down

0 comments on commit 6cfb2cc

Please sign in to comment.