From fc78adc77c44563c54d2b1b2f1e915bc2a74e954 Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 3 Sep 2024 12:19:29 +1000 Subject: [PATCH] chore: using array of filenames instead of generator of treenodes for serialization [ci skip] --- src/client/handlers/VaultsSecretsGet.ts | 55 ++++++++------------ src/utils/utils.ts | 12 ++--- src/vaults/fileTree.ts | 68 +++++++++---------------- src/vaults/types.ts | 4 +- tests/client/handlers/vaults.test.ts | 34 +++++++++++++ tests/vaults/fileTree.test.ts | 7 ++- 6 files changed, 87 insertions(+), 93 deletions(-) diff --git a/src/client/handlers/VaultsSecretsGet.ts b/src/client/handlers/VaultsSecretsGet.ts index c87f9a01e..afcc393f5 100644 --- a/src/client/handlers/VaultsSecretsGet.ts +++ b/src/client/handlers/VaultsSecretsGet.ts @@ -1,6 +1,7 @@ import type { DB } from '@matrixai/db'; import type { JSONObject, JSONRPCRequest } from '@matrixai/rpc'; import type VaultManager from '../../vaults/VaultManager'; +import { ReadableStream } from 'stream/web'; import { RawHandler } from '@matrixai/rpc'; import { validateSync } from '../../validation'; import { matchSync } from '../../utils'; @@ -16,8 +17,6 @@ class VaultsSecretsGet extends RawHandler<{ }> { public handle = async ( input: [JSONRPCRequest, ReadableStream], - _cancel: any, - _ctx: any, ): Promise<[JSONObject, ReadableStream]> => { const { vaultManager, db } = this.container; const [headerMessage, inputStream] = input; @@ -27,24 +26,26 @@ class VaultsSecretsGet extends RawHandler<{ if (params == undefined) throw new validationErrors.ErrorParse('Input params cannot be undefined'); - const { nameOrId, secretName }: { nameOrId: string; secretName: string } = - validateSync( - (keyPath, value) => { - return matchSync(keyPath)( - [ - ['nameOrId', 'secretName'], - () => { - return value as string; - }, - ], - () => value, - ); - }, - { - nameOrId: params.vaultNameOrId, - secretName: params.secretName, - }, - ); + const { + nameOrId, + secretNames, + }: { nameOrId: string; secretNames: Array } = validateSync( + (keyPath, value) => { + return matchSync(keyPath)( + [ + ['nameOrId'], + () => value as string, + ['secretNames'], + () => value as Array, + ], + () => value, + ); + }, + { + nameOrId: params.nameOrId, + secretNames: params.secretNames, + }, + ); const secretContentsGen = db.withTransactionG( async function* (tran): AsyncGenerator { const vaultIdFromName = await vaultManager.getVaultId(nameOrId, tran); @@ -60,19 +61,7 @@ class VaultsSecretsGet extends RawHandler<{ void, void > { - const contents = fileTree.serializerStreamFactory( - fs, - fileTree.globWalk({ - fs: fs, - basePath: '.', - pattern: secretName, - yieldRoot: false, - yieldStats: false, - yieldFiles: true, - yieldParents: false, - yieldDirectories: false, - }), - ); + const contents = fileTree.serializerStreamFactory(fs, secretNames); for await (const chunk of contents) { yield chunk; } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 0f8bb50cc..338707ac8 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -11,6 +11,7 @@ import process from 'process'; import path from 'path'; import nodesEvents from 'events'; import lexi from 'lexicographic-integer'; +import { ReadableStream } from 'stream/web' import { PromiseCancellable } from '@matrixai/async-cancellable'; import { timedCancellable } from '@matrixai/contexts/dist/functions'; import * as utilsErrors from './errors'; @@ -544,15 +545,8 @@ function setMaxListeners( async function* streamToAsyncGenerator( stream: ReadableStream, ): AsyncGenerator { - const reader = stream.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (value !== undefined) yield value!; - } - } finally { - reader.releaseLock(); + for await (const chunk of stream) { + yield chunk; } } diff --git a/src/vaults/fileTree.ts b/src/vaults/fileTree.ts index b733f1839..5a791d8b0 100644 --- a/src/vaults/fileTree.ts +++ b/src/vaults/fileTree.ts @@ -205,7 +205,7 @@ function generateGenericHeader(headerData: HeaderGeneric): Uint8Array { * Creates the content header which identifies the content with the length. * Data should follow this header. * Formatted as... - * generic_header(10) | total_size(8)[data_size + header_size] | i_node(4) | 'D'(1) + * generic_header(10) | total_size(8)[data_size + header_size] | 'D'(1) */ function generateContentHeader(headerData: HeaderContent): Uint8Array { const contentHeader = new Uint8Array(HeaderSize.CONTENT); @@ -215,8 +215,7 @@ function generateContentHeader(headerData: HeaderContent): Uint8Array { contentHeader.byteLength, ); dataView.setBigUint64(0, headerData.dataSize, false); - dataView.setUint32(8, headerData.iNode, false); - dataView.setUint8(12, HeaderMagic.END); + dataView.setUint8(8, HeaderMagic.END); return contentHeader; } @@ -249,8 +248,7 @@ function parseContentHeader(data: Uint8Array): Parsed { const dataView = new DataView(data.buffer, data.byteOffset, data.byteLength); if (data.byteLength < HeaderSize.CONTENT) return { remainder: data }; const dataSize = dataView.getBigUint64(0, false); - const iNode = dataView.getUint32(8, false); - const magicByte = dataView.getUint8(12); + const magicByte = dataView.getUint8(8); if (magicByte !== HeaderMagic.END) { throw new validationErrors.ErrorParse( `invalid magic byte, should be "${HeaderMagic.END}", found "${magicByte}"`, @@ -259,7 +257,6 @@ function parseContentHeader(data: Uint8Array): Parsed { return { data: { dataSize, - iNode, }, remainder: data.subarray(HeaderSize.CONTENT), }; @@ -276,7 +273,6 @@ function parseContentHeader(data: Uint8Array): Parsed { async function* encodeContent( fs: FileSystem | FileSystemReadable, path: string, - iNode: number, chunkSize: number = 1024 * 4, ): AsyncGenerator { const fd = await fs.promises.open(path, 'r'); @@ -317,7 +313,6 @@ async function* encodeContent( }), generateContentHeader({ dataSize: BigInt(stats.size), - iNode, }), ]); while (true) { @@ -332,52 +327,38 @@ async function* encodeContent( } /** - * Takes an AsyncGenerator and serializes it into a `ReadableStream` + * Takes an Array of file paths and serializes their contents into a `ReadableStream` * @param fs - * @param treeGen - An AsyncGenerator that yields the files and directories of a file tree. + * @param filePaths - An array of file paths to be serialized. */ function serializerStreamFactory( fs: FileSystem | FileSystemReadable, - treeGen: AsyncGenerator, + filePaths: Array, ): ReadableStream { - let contentsGen: AsyncGenerator | undefined; - let fileNode: TreeNode | undefined; - async function getNextContentChunk(): Promise { - while (true) { - if (contentsGen == null) { - // Keep consuming values if the result is not a file - while (true) { - const result = await treeGen.next(); - if (result.done) return undefined; - if (result.value.type === 'FILE') { - fileNode = result.value; - break; - } - } - contentsGen = encodeContent(fs, fileNode.path, fileNode.iNode); - } - const contentChunk = await contentsGen.next(); - if (!contentChunk.done) return contentChunk.value; - contentsGen = undefined; - } - } - async function cleanup(reason: unknown) { - await treeGen?.throw(reason).catch(() => {}); - await contentsGen?.throw(reason).catch(() => {}); - } + let contentsGen: AsyncGenerator | undefined; return new ReadableStream({ pull: async (controller) => { try { - const contentChunk = await getNextContentChunk(); - if (contentChunk == null) return controller.close(); - else controller.enqueue(contentChunk); + while (true) { + if (contentsGen == null) { + const path = filePaths.shift(); + if (path == null) return controller.close(); + contentsGen = encodeContent(fs, path); + } + const { done, value } = await contentsGen.next(); + if (!done) { + controller.enqueue(value); + return; + } + contentsGen = undefined; + } } catch (e) { - await cleanup(e); + await contentsGen?.throw(e).catch(() => {}); return controller.error(e); } }, cancel: async (reason) => { - await cleanup(reason); + await contentsGen?.throw(reason).catch(() => {}); }, }); } @@ -473,9 +454,8 @@ function parserTransformStreamFactory(): TransformStream< const contentHeader = parseContentHeader(genericHeader.remainder); if (contentHeader.data == null) return; - const { dataSize, iNode } = contentHeader.data; - controller.enqueue({ type: 'CONTENT', dataSize, iNode }); - contentLength = dataSize; + contentLength = contentHeader.data.dataSize; + controller.enqueue({ type: 'CONTENT', dataSize: contentLength }); workingBuffer = contentHeader.remainder; } // We yield the whole buffer, or split it for the next header diff --git a/src/vaults/types.ts b/src/vaults/types.ts index a4609718c..b13f0bf29 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -145,7 +145,6 @@ type TreeNode = { type ContentNode = { type: 'CONTENT'; - iNode: number; dataSize: bigint; }; type DoneMessage = { type: 'DONE' }; @@ -181,12 +180,11 @@ type HeaderGeneric = { }; type HeaderContent = { dataSize: bigint; - iNode: number; }; enum HeaderSize { GENERIC = 2, - CONTENT = 13, + CONTENT = 9, } enum HeaderType { diff --git a/tests/client/handlers/vaults.test.ts b/tests/client/handlers/vaults.test.ts index b847ee018..1263fc4bf 100644 --- a/tests/client/handlers/vaults.test.ts +++ b/tests/client/handlers/vaults.test.ts @@ -1470,6 +1470,40 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { vaultsErrors.ErrorSecretsSecretUndefined, ); }); + // TODO: TEST + test('view output', async () => { + const secret = 'test-secret'; + const vaultId = await vaultManager.createVault('test-vault'); + const vaultIdEncoded = vaultsUtils.encodeVaultId(vaultId); + await rpcClient.methods.vaultsSecretsNew({ + nameOrId: vaultIdEncoded, + secretName: secret, + secretContent: Buffer.from('test-secret-contents-1').toString('binary'), + }); + await rpcClient.methods.vaultsSecretsNew({ + nameOrId: vaultIdEncoded, + secretName: 's2', + secretContent: Buffer.from('test-secret-contents-abc').toString('binary'), + }); + const response = await rpcClient.methods.vaultsSecretsGet({ + nameOrId: vaultIdEncoded, + secretNames: ['test-secret','s2'], + }); + // const secretContent = response.meta?.result; + const data: Array = []; + for await (const d of response.readable) data.push(d); + // console.log(new TextDecoder().decode(Buffer.concat(data))); + const output = Buffer.concat(data) + .toString('utf-8') + .split('') + .map(char => { + const code = char.charCodeAt(0); + return code >= 32 && code <= 126 ? char : `\\x${code.toString(16).padStart(2, '0')}`; + }) + .join(''); + console.log(output); + + }) }); describe('vaultsSecretsNewDir and vaultsSecretsList', () => { const logger = new Logger('vaultsSecretsNewDirList test', LogLevel.WARN, [ diff --git a/tests/vaults/fileTree.test.ts b/tests/vaults/fileTree.test.ts index d007f9302..083668135 100644 --- a/tests/vaults/fileTree.test.ts +++ b/tests/vaults/fileTree.test.ts @@ -722,10 +722,9 @@ describe('fileTree', () => { yieldParents: false, yieldDirectories: false, }); - const serializedStream = fileTree.serializerStreamFactory( - fs, - fileTreeGen, - ); + const data: Array = []; + for await (const p of fileTreeGen) data.push(p.path); + const serializedStream = fileTree.serializerStreamFactory(fs, data); const parserTransform = fileTree.parserTransformStreamFactory(); const outputStream = serializedStream.pipeThrough(parserTransform); const output: Array = [];