feat: adding unix-like ls for secrets

wip: working on adding unix-like ls for secrets

chore: added custom type and pattern matching

chore: updated caller for VaultsSecretsList

wip: working on passing filetree via rpc

feat: revamped the entire VaultsSecretsList handler

chore: updated handler name

chore: working on fixing read: 0 error

chore: renamed VaultsSecretsList to VaultsSecretsGetFileTree

fix: fixed problem with `withVaultsG` where `this` wasn't defined

[ci skip]

chore: switched from RawHandler to ServerCaller

chore: cleaned up code and started work on param parser

feat: added parsers for parameters

[ci skip]

chore: updated all tests referencing old serializerStreamFactory

feat: switched over the rpc handler from rawhandler to stremhandler

feat: stopped using globWalk for reading and listing files

chore: added error handling for invalid patterns

chore: added error handling for invalid patterns

chore: added symbolic link information to the file stats

chore: cleaned up irrelevant code

chore: fixed tests

src/client/errors.ts

@@ -18,6 +18,11 @@ class ErrorClientAuthDenied<T> extends ErrorClient<T> {
   exitCode = sysexits.NOPERM;
 }
 
+class ErrorClientFSReadFailed<T> extends ErrorClient<T> {
+  static description = 'Failed to read from filesystem';
+  exitCode = sysexits.IOERR;
+}
+
 class ErrorClientService<T> extends ErrorClient<T> {}
 
 class ErrorClientServiceRunning<T> extends ErrorClientService<T> {
@@ -45,6 +50,7 @@ export {
   ErrorClientAuthMissing,
   ErrorClientAuthFormat,
   ErrorClientAuthDenied,
+  ErrorClientFSReadFailed,
   ErrorClientService,
   ErrorClientServiceRunning,
   ErrorClientServiceNotRunning,

src/client/handlers/VaultsSecretsList.ts

@@ -2,55 +2,65 @@ import type { DB } from '@matrixai/db';
 import type {
   ClientRPCRequestParams,
   ClientRPCResponseResult,
-  SecretNameMessage,
-  VaultIdentifierMessage,
+  SecretFilesMessage,
+  SecretFiles,
 } from '../types';
 import type VaultManager from '../../vaults/VaultManager';
+import path from 'path';
 import { ServerHandler } from '@matrixai/rpc';
 import * as vaultsUtils from '../../vaults/utils';
 import * as vaultsErrors from '../../vaults/errors';
-import * as vaultOps from '../../vaults/VaultOps';
+import * as clientErrors from '../errors';
 
 class VaultsSecretsList extends ServerHandler<
   {
     vaultManager: VaultManager;
     db: DB;
   },
-  ClientRPCRequestParams<VaultIdentifierMessage>,
-  ClientRPCResponseResult<SecretNameMessage>
+  ClientRPCRequestParams<SecretFilesMessage>,
+  ClientRPCResponseResult<SecretFiles>
 > {
   public async *handle(
-    input: ClientRPCRequestParams<VaultIdentifierMessage>,
-    _cancel,
-    _meta,
-    ctx,
-  ): AsyncGenerator<ClientRPCResponseResult<SecretNameMessage>> {
-    if (ctx.signal.aborted) throw ctx.signal.reason;
+    input: ClientRPCRequestParams<SecretFilesMessage>,
+    _cancel: any,
+  ): AsyncGenerator<ClientRPCResponseResult<SecretFiles>, void, void> {
     const { vaultManager, db } = this.container;
-    const secrets = await db.withTransactionF(async (tran) => {
+    const vaultId = await db.withTransactionF(async (tran) => {
       const vaultIdFromName = await vaultManager.getVaultId(
         input.nameOrId,
         tran,
       );
       const vaultId =
         vaultIdFromName ?? vaultsUtils.decodeVaultId(input.nameOrId);
-      if (vaultId == null) {
-        throw new vaultsErrors.ErrorVaultsVaultUndefined();
-      }
-      return await vaultManager.withVaults(
-        [vaultId],
-        async (vault) => {
-          return await vaultOps.listSecrets(vault);
-        },
-        tran,
-      );
+      if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined();
+      return vaultId;
+    });
+
+    yield* vaultManager.withVaultsG([vaultId], (vault) => {
+      return vault.readG(async function* (fs): AsyncGenerator<
+        SecretFiles,
+        void,
+        void
+      > {
+        let files: Array<string>;
+        try {
+          // @ts-ignore: While the types don't fully match, it matches enough for our usage.
+          files = await fs.promises.readdir(input.path);
+        } catch (e) {
+          throw new clientErrors.ErrorClientFSReadFailed(
+            `No matches exist for path: ${input.path}`,
+            { cause: e },
+          );
+        }
+        files = files.map((file) => path.join(input.path, file));
+
+        for await (const file of files) {
+          const stat = await fs.promises.stat(file);
+          const type = stat.isFile() ? 'FILE' : 'DIRECTORY';
+          yield { path: file, type: type };
+        }
+      });
     });
-    for (const secret of secrets) {
-      if (ctx.signal.aborted) throw ctx.signal.reason;
-      yield {
-        secretName: secret,
-      };
-    }
   }
 }
 

src/client/types.ts

@@ -300,6 +300,9 @@ type VaultsLatestVersionMessage = {
 };
 
 // Secrets
+type SecretFilesMessage = VaultIdentifierMessage & {
+  path: string;
+};
 
 type SecretNameMessage = {
   secretName: string;
@@ -327,6 +330,11 @@ type SecretRenameMessage = SecretIdentifierMessage & {
   newSecretName: string;
 };
 
+type SecretFiles = {
+  path: string;
+  type: 'FILE' | 'DIRECTORY';
+};
+
 // Stat is the 'JSON.stringify version of the file stat
 type SecretStatMessage = {
   stat: {
@@ -412,11 +420,13 @@ export type {
   VaultsLatestVersionMessage,
   SecretNameMessage,
   SecretIdentifierMessage,
+  SecretFilesMessage,
   ContentMessage,
   SecretContentMessage,
   SecretMkdirMessage,
   SecretDirMessage,
   SecretRenameMessage,
+  SecretFiles,
   SecretStatMessage,
   SignatureMessage,
   OverrideRPClientType,

src/vaults/VaultManager.ts

@@ -1058,13 +1058,14 @@ class VaultManager {
       },
     );
     // Running the function with locking
+    const vaultThis = this;
     return yield* this.vaultLocks.withG(
       ...vaultLocks,
       async function* (): AsyncGenerator<T, Treturn, Tnext> {
         // Getting the vaults while locked
         const vaults = await Promise.all(
           vaultIds.map(async (vaultId) => {
-            return await this.getVault(vaultId, tran);
+            return await vaultThis.getVault(vaultId, tran);
           }),
         );
         return yield* g(...vaults);

src/vaults/fileTree.ts

@@ -542,8 +542,17 @@ function parserTransformStreamFactory(): TransformStream<
     };
     jsonParser.write(initialChunk);
   };
+  let processed: boolean = false;
   return new TransformStream<Uint8Array, TreeNode | ContentNode | Uint8Array>({
+    flush: (controller) => {
+      if (!processed) {
+        controller.error(
+          new validationErrors.ErrorParse('Stream ended prematurely'),
+        );
+      }
+    },
     transform: (chunk, controller) => {
+      if (chunk.byteLength > 0) processed = true;
       switch (phase) {
         case 'START': {
           workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]);

tests/client/handlers/vaults.test.ts

@@ -69,6 +69,7 @@ import * as nodesUtils from '@/nodes/utils';
 import * as vaultsUtils from '@/vaults/utils';
 import * as vaultsErrors from '@/vaults/errors';
 import * as networkUtils from '@/network/utils';
+import * as clientErrors from '@/client/errors';
 import * as testsUtils from '../../utils';
 
 describe('vaultsClone', () => {
@@ -1587,16 +1588,31 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => {
       dirName: secretDir,
     });
     expect(addResponse.success).toBeTruthy();
-    // List secrets
-    const listResponse = await rpcClient.methods.vaultsSecretsList({
+
+    await expect(async () => {
+      const files = await rpcClient.methods.vaultsSecretsList({
+        nameOrId: vaultsIdEncoded,
+        path: 'doesntExist',
+      });
+      try {
+        for await (const _ of files); // Consume values
+      } catch (e) {
+        throw e.cause;
+      }
+    }).rejects.toThrow(clientErrors.ErrorClientFSReadFailed);
+
+    const secrets = await rpcClient.methods.vaultsSecretsList({
       nameOrId: vaultsIdEncoded,
+      path: 'secretDir',
     });
-    const secrets: Array<string> = [];
-    for await (const secret of listResponse) {
-      secrets.push(secret.secretName);
+
+    // Extract secret file paths
+    const parsedFiles: Array<string> = [];
+    for await (const file of secrets) {
+      parsedFiles.push(file.path);
     }
-    expect(secrets.sort()).toStrictEqual(
-      secretList.map((secret) => path.join('secretDir', secret)).sort(),
+    expect(parsedFiles).toIncludeAllMembers(
+      secretList.map((secret) => path.join('secretDir', secret)),
     );
   });
 });