Skip to content

Commit

Permalink
feat: switched over the rpc handler from rawhandler to stremhandler
Browse files Browse the repository at this point in the history
  • Loading branch information
aryanjassal committed Aug 16, 2024
1 parent b86dfd4 commit 1fc3ba0
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 145 deletions.
11 changes: 9 additions & 2 deletions src/client/callers/vaultsSecretsGetFileTree.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
import { RawCaller } from '@matrixai/rpc';
import type { HandlerTypes } from '@matrixai/rpc';
import type VaultsSecretsGetFileTree from '../handlers/VaultsSecretsGetFileTree';
import { ServerCaller } from '@matrixai/rpc';

const vaultsSecretsGetFileTree = new RawCaller();
type CallerTypes = HandlerTypes<VaultsSecretsGetFileTree>;

const vaultsSecretsGetFileTree = new ServerCaller<
CallerTypes['input'],
CallerTypes['output']
>();

export default vaultsSecretsGetFileTree;
148 changes: 32 additions & 116 deletions src/client/handlers/VaultsSecretsGetFileTree.ts
Original file line number Diff line number Diff line change
@@ -1,140 +1,56 @@
import type { DB } from '@matrixai/db';
import type { JSONObject, JSONRPCRequest } from '@matrixai/rpc';
import type VaultManager from '../../vaults/VaultManager';
import { ReadableStream } from 'stream/web';
import { RawHandler } from '@matrixai/rpc';
import type { ClientRPCRequestParams, ClientRPCResponseResult } from '../types';
import type { TreeNode } from '../../vaults/types';
import type { SecretFilesMessage } from '../types';
import { ServerHandler } from '@matrixai/rpc';
import { fileTree } from '../../vaults';
import * as vaultsUtils from '../../vaults/utils';
import * as vaultsErrors from '../../vaults/errors';
import * as utils from '../../utils';
import { fileTree } from '../../vaults';
import { validateSync } from '../../validation';
import * as validationErrors from '../../validation/errors';

class VaultsSecretsGetFileTree extends RawHandler<{
vaultManager: VaultManager;
db: DB;
}> {
public handle = async (
input: [JSONRPCRequest, ReadableStream<Uint8Array>],
class VaultsSecretsGetFileTree extends ServerHandler<
{
vaultManager: VaultManager;
db: DB;
},
ClientRPCRequestParams<SecretFilesMessage>,
ClientRPCResponseResult<TreeNode>
> {
public async *handle(
input: ClientRPCRequestParams<SecretFilesMessage>,
_cancel: any,
): Promise<[JSONObject, ReadableStream<Uint8Array>]> => {
): AsyncGenerator<ClientRPCResponseResult<TreeNode>, void, void> {
const { vaultManager, db } = this.container;
const [headerMessage, _] = input;

const params = headerMessage.params;
if (params == null || !utils.isObject(params)) utils.never();

// TEST: testing needed
function stringParser(value: any): string {
if (typeof value !== 'string') {
throw new validationErrors.ErrorParse(
'Provided value must be a string',
);
}
return value as string;
}

function globPatternParser(value: any): string {
const allowedCharacters = /^[\w\s\-./*?[\]{}]*$/;
const invalidCharacters = /[\0:]/;
if (!value) {
throw new validationErrors.ErrorParse('Glob pattern must not be empty');
}
if (typeof value !== 'string') {
throw new validationErrors.ErrorParse(
'Glob pattern must be must be a string',
);
}
if (!allowedCharacters.test(value) || invalidCharacters.test(value)) {
throw new validationErrors.ErrorParse('Glob pattern is not valid');
}
return value as string;
}

function booleanParser(value: any): boolean {
if (typeof value !== 'boolean') {
throw new validationErrors.ErrorParse(
'Provided value must be a string',
);
}
return value as boolean;
}

const {
vaultNameOrId,
pattern,
yieldStats,
yieldRoot,
yieldFiles,
yieldParents,
yieldDirectories,
}: {
vaultNameOrId: string;
pattern: string;
yieldStats: boolean;
yieldRoot: boolean;
yieldFiles: boolean;
yieldParents: boolean;
yieldDirectories: boolean;
} = validateSync(
(keyPath, value) => {
return utils.matchSync(keyPath)(
[['vaultNameOrId'], () => stringParser(value)],
[['pattern'], () => globPatternParser(value)],
[
[
'yieldStats',
'yieldRoot',
'yieldFiles',
'yieldParents',
'yieldDirectories',
],
() => booleanParser(value),
],
() => value,
);
},
{
vaultNameOrId: params.vaultNameOrId,
pattern: params.pattern,
yieldStats: params.yieldStats,
yieldRoot: params.yieldRoot,
yieldFiles: params.yieldFiles,
yieldParents: params.yieldParents,
yieldDirectories: params.yieldDirectories,
},
);

const vaultId = await db.withTransactionF(async (tran) => {
const vaultIdFromName = await vaultManager.getVaultId(
vaultNameOrId,
input.nameOrId,
tran,
);
const vaultId =
vaultIdFromName ?? vaultsUtils.decodeVaultId(vaultNameOrId);
vaultIdFromName ?? vaultsUtils.decodeVaultId(input.nameOrId);
if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined();
return vaultId;
});

const filesData = vaultManager.withVaultsG([vaultId], (vault) => {
return vault.readG((fs): AsyncGenerator<Uint8Array, void, void> => {
const fileTreeGen = fileTree.globWalk({
yield* vaultManager.withVaultsG([vaultId], (vault) => {
return vault.readG(async function* (fs): AsyncGenerator<
TreeNode,
void,
void
> {
yield* fileTree.globWalk({
fs: fs,
basePath: '.',
pattern: pattern,
yieldStats: yieldStats,
yieldRoot: yieldRoot,
yieldFiles: yieldFiles,
yieldParents: yieldParents,
yieldDirectories: yieldDirectories,
pattern: input.pattern,
yieldStats: input.yieldStats,
yieldRoot: input.yieldRoot,
yieldFiles: input.yieldFiles,
yieldParents: input.yieldParents,
yieldDirectories: input.yieldDirectories,
});
return fileTree.serializerStreamFactory(fs, fileTreeGen, false);
});
});

const filesDataStream = utils.asyncGeneratorToReadableStream(filesData);
return [{}, filesDataStream];
};
}
}

export default VaultsSecretsGetFileTree;
6 changes: 0 additions & 6 deletions src/client/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,11 +301,6 @@ type VaultsLatestVersionMessage = {

// Secrets

// NOTE: see if we still need this. probably dont, but just be sure.
type SecretFilesList = {
data: string;
};

type SecretFilesMessage = VaultIdentifierMessage & {
pattern: string;
yieldStats: boolean;
Expand Down Expand Up @@ -426,7 +421,6 @@ export type {
VaultsScanMessage,
VaultsVersionMessage,
VaultsLatestVersionMessage,
SecretFilesList,
SecretNameMessage,
SecretIdentifierMessage,
SecretFilesMessage,
Expand Down
3 changes: 1 addition & 2 deletions src/utils/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@ import os from 'os';
import process from 'process';
import path from 'path';
import nodesEvents from 'events';
import lexi from 'lexicographic-integer';
import { ReadableStream } from 'stream/web';
import lexi from 'lexicographic-integer';
import { PromiseCancellable } from '@matrixai/async-cancellable';
import { timedCancellable } from '@matrixai/contexts/dist/functions';
import * as utilsErrors from './errors';
import { JSONObject } from '@matrixai/rpc';

const AsyncFunction = (async () => {}).constructor;
const GeneratorFunction = function* () {}.constructor;
Expand Down
11 changes: 10 additions & 1 deletion src/vaults/fileTree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import type {
HeaderContent,
} from './types';
import path from 'path';
import { ReadableStream, TransformStream } from 'stream/web';
import { TransformStream } from 'stream/web';
import { minimatch } from 'minimatch';
import { JSONParser, TokenizerError } from '@streamparser/json';
import * as vaultsUtils from './utils';
Expand Down Expand Up @@ -532,8 +532,17 @@ function parserTransformStreamFactory(): TransformStream<
};
jsonParser.write(initialChunk);
};
let processed: boolean = false;
return new TransformStream<Uint8Array, TreeNode | ContentNode | Uint8Array>({
flush: (controller) => {
if (!processed) {
controller.error(
new validationErrors.ErrorParse('Stream ended prematurely'),
);
}
},
transform: (chunk, controller) => {
if (chunk.byteLength > 0) processed = true;
switch (phase) {
case 'START': {
workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]);
Expand Down
17 changes: 4 additions & 13 deletions tests/client/handlers/vaults.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,6 @@ import * as vaultsUtils from '@/vaults/utils';
import * as vaultsErrors from '@/vaults/errors';
import * as networkUtils from '@/network/utils';
import * as testsUtils from '../../utils';
import { fileTree } from '@/vaults';
import { ContentNode, TreeNode } from '@/vaults/types';

describe('vaultsClone', () => {
const logger = new Logger('vaultsClone test', LogLevel.WARN, [
Expand Down Expand Up @@ -1592,7 +1590,7 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => {

// List secrets with names of directories
const secrets = await rpcClient.methods.vaultsSecretsGetFileTree({
vaultNameOrId: vaultsIdEncoded,
nameOrId: vaultsIdEncoded,
pattern: '**/*',
yieldStats: false,
yieldRoot: false,
Expand All @@ -1601,21 +1599,14 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => {
yieldDirectories: true,
});

// Parse secrets
const secretDataStream = secrets.readable;
const parserTransform = fileTree.parserTransformStreamFactory();
const parsedFilesStream = secretDataStream.pipeThrough(parserTransform);

// Extract secret file paths
const parsedFiles: Array<string> = [];
for await (const file of parsedFilesStream) {
if ('path' in file) {
parsedFiles.push(file.path);
}
for await (const file of secrets) {
parsedFiles.push(file.path);
}
expect(parsedFiles).toIncludeAllMembers([
...secretList.map((secret) => path.join('secretDir', secret)),
'secretDir'
'secretDir',
]);
});
});
Expand Down
2 changes: 1 addition & 1 deletion tests/vaults/VaultOps.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import * as vaultOps from '@/vaults/VaultOps';
import * as vaultsErrors from '@/vaults/errors';
import * as vaultsUtils from '@/vaults/utils';
import * as keysUtils from '@/keys/utils';
import * as testNodesUtils from '../nodes/utils';
import * as utils from '@/utils';
import * as testNodesUtils from '../nodes/utils';

describe('VaultOps', () => {
const logger = new Logger('VaultOps', LogLevel.WARN, [new StreamHandler()]);
Expand Down
11 changes: 7 additions & 4 deletions tests/vaults/fileTree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import { ReadableStream } from 'stream/web';
import { test } from '@fast-check/jest';
import fc from 'fast-check';
import * as fileTree from '@/vaults/fileTree';
import * as vaultsTestUtils from './utils';
import * as utils from '@/utils';
import * as vaultsTestUtils from './utils';

describe('fileTree', () => {
let dataDir: string;
Expand Down Expand Up @@ -502,7 +502,8 @@ describe('fileTree', () => {
fileTreeGen,
false,
);
const serializedStream = utils.asyncGeneratorToReadableStream(serializedGen);
const serializedStream =
utils.asyncGeneratorToReadableStream(serializedGen);
const outputStream = serializedStream.pipeThrough(parserTransform);
for await (const output of outputStream) {
data.push(output);
Expand Down Expand Up @@ -549,7 +550,8 @@ describe('fileTree', () => {
fileTreeGen,
false,
);
const serializedStream = utils.asyncGeneratorToReadableStream(serializedGen);
const serializedStream =
utils.asyncGeneratorToReadableStream(serializedGen);
const outputStream = serializedStream
.pipeThrough(snipperTransform)
.pipeThrough(parserTransform);
Expand Down Expand Up @@ -667,7 +669,8 @@ describe('fileTree', () => {
fileTreeGen,
true,
);
const serializedStream = utils.asyncGeneratorToReadableStream(serializedGen);
const serializedStream =
utils.asyncGeneratorToReadableStream(serializedGen);
const outputStream = serializedStream.pipeThrough(parserTransform);
for await (const output of outputStream) {
data.push(output);
Expand Down

0 comments on commit 1fc3ba0

Please sign in to comment.