chore: secrets get mostly done

src/client/handlers/VaultsSecretsGet.ts

@@ -1,7 +1,7 @@
 import type { DB } from '@matrixai/db';
 import type { JSONObject, JSONRPCRequest } from '@matrixai/rpc';
 import type VaultManager from '../../vaults/VaultManager';
-import { ReadableStream } from 'stream/web';
+import type { ReadableStream } from 'stream/web';
 import { RawHandler } from '@matrixai/rpc';
 import { validateSync } from '../../validation';
 import { matchSync } from '../../utils';
@@ -21,10 +21,11 @@ class VaultsSecretsGet extends RawHandler<{
     const { vaultManager, db } = this.container;
     const [headerMessage, inputStream] = input;
     const params = headerMessage.params;
-    inputStream.cancel(); // Close input stream as it's useless for this call
+    await inputStream.cancel();
 
-    if (params == undefined)
+    if (params == null) {
       throw new validationErrors.ErrorParse('Input params cannot be undefined');
+    }
 
     const {
       nameOrId,
@@ -34,9 +35,29 @@ class VaultsSecretsGet extends RawHandler<{
         return matchSync(keyPath)(
           [
             ['nameOrId'],
-            () => value as string,
+            () => {
+              if (typeof value != 'string') {
+                throw new validationErrors.ErrorParse(
+                  'Parameter must be of type string',
+                );
+              }
+              return value as string;
+            },
+          ],
+          [
             ['secretNames'],
-            () => value as Array<string>,
+            () => {
+              if (
+                !Array.isArray(value) ||
+                value.length === 0 ||
+                !value.every((v) => typeof v === 'string')
+              ) {
+                throw new validationErrors.ErrorParse(
+                  'Parameter must be a non-empty array of strings',
+                );
+              }
+              return value as Array<string>;
+            },
           ],
           () => value,
         );
@@ -53,7 +74,6 @@ class VaultsSecretsGet extends RawHandler<{
         if (vaultId == null) {
           throw new vaultsErrors.ErrorVaultsVaultUndefined();
         }
-
         // Get secret contents
         yield* vaultManager.withVaultsG([vaultId], (vault) => {
           return vault.readG(async function* (fs): AsyncGenerator<
@@ -62,8 +82,15 @@ class VaultsSecretsGet extends RawHandler<{
             void
           > {
             const contents = fileTree.serializerStreamFactory(fs, secretNames);
-            for await (const chunk of contents) {
-              yield chunk;
+            try {
+              for await (const chunk of contents) yield chunk;
+            } catch (e) {
+              if (e.name === 'ErrorSecretsSecretUndefined') {
+                throw new vaultsErrors.ErrorSecretsSecretUndefined(e.message, {
+                  cause: e.cause,
+                });
+              }
+              throw e;
             }
           });
         });

src/utils/utils.ts

@@ -10,8 +10,8 @@ import os from 'os';
 import process from 'process';
 import path from 'path';
 import nodesEvents from 'events';
+import { ReadableStream } from 'stream/web';
 import lexi from 'lexicographic-integer';
-import { ReadableStream } from 'stream/web'
 import { PromiseCancellable } from '@matrixai/async-cancellable';
 import { timedCancellable } from '@matrixai/contexts/dist/functions';
 import * as utilsErrors from './errors';

src/vaults/fileTree.ts

@@ -1,5 +1,5 @@
 import type { Stat } from 'encryptedfs';
-import type { FileSystem } from '../types';
+import type { FileHandle, FileSystem } from '../types';
 import type {
   ContentNode,
   DoneMessage,
@@ -11,13 +11,15 @@ import type {
   HeaderGeneric,
   HeaderContent,
 } from './types';
+import type { FdIndex } from 'encryptedfs/dist/fd';
 import path from 'path';
 import { ReadableStream, TransformStream } from 'stream/web';
 import { minimatch } from 'minimatch';
 import * as vaultsUtils from './utils';
 import { HeaderSize, HeaderType, HeaderMagic } from './types';
 import * as utils from '../utils';
 import * as validationErrors from '../validation/errors';
+import * as vaultsErrors from '../vaults/errors';
 
 /**
  * Generates a serializable format of file stats
@@ -275,7 +277,17 @@ async function* encodeContent(
   path: string,
   chunkSize: number = 1024 * 4,
 ): AsyncGenerator<Uint8Array, void, void> {
-  const fd = await fs.promises.open(path, 'r');
+  let fd: FileHandle | FdIndex;
+  try {
+    fd = await fs.promises.open(path, 'r');
+  } catch (e) {
+    if (e.code === 'ENOENT') {
+      throw new vaultsErrors.ErrorSecretsSecretUndefined(e.message, {
+        cause: e,
+      });
+    }
+    throw e;
+  }
   async function read(buffer: Uint8Array): Promise<{
     bytesRead: number;
     buffer: Uint8Array;
@@ -335,13 +347,14 @@ function serializerStreamFactory(
   fs: FileSystem | FileSystemReadable,
   filePaths: Array<string>,
 ): ReadableStream<Uint8Array> {
-  let contentsGen: AsyncGenerator<Uint8Array> | undefined;
+  const paths = filePaths.slice();
+  let contentsGen: AsyncGenerator<Uint8Array> | undefined = undefined;
   return new ReadableStream<Uint8Array>({
     pull: async (controller) => {
       try {
         while (true) {
           if (contentsGen == null) {
-            const path = filePaths.shift();
+            const path = paths.shift();
             if (path == null) return controller.close();
             contentsGen = encodeContent(fs, path);
           }
@@ -440,32 +453,31 @@ function parserTransformStreamFactory(): TransformStream<
     transform: (chunk, controller) => {
       if (chunk.byteLength > 0) processedChunks = true;
       workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]);
-      if (contentLength == null) {
-        const genericHeader = parseGenericHeader(workingBuffer);
-        if (genericHeader.data == null) return;
-        if (genericHeader.data.type !== HeaderType.CONTENT) {
-          controller.error(
-            new validationErrors.ErrorParse(
-              `expected CONTENT message, got "${genericHeader.data.type}"`,
-            ),
-          );
-          return;
+
+      while (true) {
+        // Header parsing until enough data is acquired
+        if (contentLength == null) {
+          const genericHeader = parseGenericHeader(workingBuffer);
+          if (genericHeader.data == null) return;
+          if (genericHeader.data.type !== HeaderType.CONTENT) {
+            controller.error(
+              new validationErrors.ErrorParse(
+                `expected CONTENT message, got "${genericHeader.data.type}"`,
+              ),
+            );
+            return;
+          }
+          const contentHeader = parseContentHeader(genericHeader.remainder);
+          if (contentHeader.data == null) return;
+
+          contentLength = contentHeader.data.dataSize;
+          controller.enqueue({ type: 'CONTENT', dataSize: contentLength });
+          workingBuffer = contentHeader.remainder;
         }
-        const contentHeader = parseContentHeader(genericHeader.remainder);
-        if (contentHeader.data == null) return;
+        // We yield the whole buffer, or split it for the next header
+        if (workingBuffer.byteLength < Number(contentLength)) break;
 
-        contentLength = contentHeader.data.dataSize;
-        controller.enqueue({ type: 'CONTENT', dataSize: contentLength });
-        workingBuffer = contentHeader.remainder;
-      }
-      // We yield the whole buffer, or split it for the next header
-      if (workingBuffer.byteLength === 0) return;
-      if (workingBuffer.byteLength <= contentLength) {
-        contentLength -= BigInt(workingBuffer.byteLength);
-        controller.enqueue(workingBuffer);
-        workingBuffer = new Uint8Array(0);
-        if (contentLength === 0n) contentLength = undefined;
-      } else {
+        // Process the contents after enough data has been accumulated
         controller.enqueue(workingBuffer.subarray(0, Number(contentLength)));
         workingBuffer = workingBuffer.subarray(Number(contentLength));
         contentLength = undefined;

tests/client/handlers/vaults.test.ts

@@ -1,6 +1,7 @@
 import type { TLSConfig } from '@/network/types';
 import type { FileSystem } from '@/types';
 import type { VaultId } from '@/ids';
+import type { ContentNode } from '@/vaults/types';
 import type NodeManager from '@/nodes/NodeManager';
 import type {
   LogEntryMessage,
@@ -15,6 +16,7 @@ import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger';
 import { DB } from '@matrixai/db';
 import { RPCClient } from '@matrixai/rpc';
 import { WebSocketClient } from '@matrixai/ws';
+import { fileTree } from '@/vaults';
 import TaskManager from '@/tasks/TaskManager';
 import ACL from '@/acl/ACL';
 import KeyRing from '@/keys/KeyRing';
@@ -1449,12 +1451,26 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => {
     });
     expect(createResponse.success).toBeTruthy();
     // Get secret
-    const getResponse1 = await rpcClient.methods.vaultsSecretsGet({
+    await testsUtils.expectRemoteError(
+      rpcClient.methods.vaultsSecretsGet({
+        nameOrId: vaultIdEncoded,
+        secretNames: ['doesnt-exist'],
+      }),
+      vaultsErrors.ErrorSecretsSecretUndefined,
+    );
+    const getResponse = await rpcClient.methods.vaultsSecretsGet({
       nameOrId: vaultIdEncoded,
-      secretName: secret,
+      secretNames: [secret],
     });
-    const secretContent = getResponse1.secretContent;
-    expect(secretContent).toStrictEqual(secret);
+    const data: Array<ContentNode | Uint8Array> = [];
+    const dataStream = getResponse.readable.pipeThrough(
+      fileTree.parserTransformStreamFactory(),
+    );
+    for await (const chunk of dataStream) data.push(chunk);
+    const secretContent = data
+      .filter((v) => v instanceof Uint8Array)
+      .map((v) => Buffer.from(v as Uint8Array).toString());
+    expect(secretContent).toStrictEqual([secret]);
     // Delete secret
     const deleteResponse = await rpcClient.methods.vaultsSecretsDelete({
       nameOrId: vaultIdEncoded,
@@ -1465,45 +1481,11 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => {
     await testsUtils.expectRemoteError(
       rpcClient.methods.vaultsSecretsGet({
         nameOrId: vaultIdEncoded,
-        secretName: secret,
+        secretNames: [secret],
       }),
       vaultsErrors.ErrorSecretsSecretUndefined,
     );
   });
-  // TODO: TEST
-  test('view output', async () => {
-    const secret = 'test-secret';
-    const vaultId = await vaultManager.createVault('test-vault');
-    const vaultIdEncoded = vaultsUtils.encodeVaultId(vaultId);
-    await rpcClient.methods.vaultsSecretsNew({
-      nameOrId: vaultIdEncoded,
-      secretName: secret,
-      secretContent: Buffer.from('test-secret-contents-1').toString('binary'),
-    });
-    await rpcClient.methods.vaultsSecretsNew({
-      nameOrId: vaultIdEncoded,
-      secretName: 's2',
-      secretContent: Buffer.from('test-secret-contents-abc').toString('binary'),
-    });
-    const response = await rpcClient.methods.vaultsSecretsGet({
-      nameOrId: vaultIdEncoded,
-      secretNames: ['test-secret','s2'],
-    });
-    // const secretContent = response.meta?.result;
-    const data: Array<Uint8Array> = [];
-    for await (const d of response.readable) data.push(d);
-    // console.log(new TextDecoder().decode(Buffer.concat(data)));
-    const output = Buffer.concat(data)
-      .toString('utf-8')
-      .split('')
-      .map(char => {
-        const code = char.charCodeAt(0);
-        return code >= 32 && code <= 126 ? char : `\\x${code.toString(16).padStart(2, '0')}`;
-      })
-      .join('');
-    console.log(output);
-
-  })
 });
 describe('vaultsSecretsNewDir and vaultsSecretsList', () => {
   const logger = new Logger('vaultsSecretsNewDirList test', LogLevel.WARN, [

tests/vaults/VaultOps.test.ts

@@ -579,21 +579,15 @@ describe('VaultOps', () => {
     });
     test('serializer with content works with efs', async () => {
       const data = await vault.readF(async (fs) => {
-        const fileTreeGen = fileTree.globWalk({
-          fs,
-          yieldStats: false,
-          yieldRoot: false,
-          yieldFiles: true,
-          yieldParents: true,
-          yieldDirectories: true,
-        });
         const data: Array<TreeNode | ContentNode | Uint8Array> = [];
         const parserTransform = fileTree.parserTransformStreamFactory();
-        const serializedStream = fileTree.serializerStreamFactory(
-          fs,
-          fileTreeGen,
-          true,
-        );
+        const serializedStream = fileTree.serializerStreamFactory(fs, [
+          file0b,
+          file1a,
+          file2b,
+          file3a,
+          file4b,
+        ]);
         const outputStream = serializedStream.pipeThrough(parserTransform);
         for await (const output of outputStream) {
           data.push(output);