Skip to content

Commit

Permalink
chore: simplified code
Browse files Browse the repository at this point in the history
  • Loading branch information
@aryanjassal
aryanjassal committed Sep 2, 2024
1 parent 5aaae26 commit 7531ef0
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 68 deletions.
61 changes: 16 additions & 45 deletions src/vaults/fileTree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ import type {
import path from 'path';
import { ReadableStream, TransformStream } from 'stream/web';
import { minimatch } from 'minimatch';
import { JSONParser, TokenizerError } from '@streamparser/json';
import * as vaultsUtils from './utils';
import { HeaderSize, HeaderType, HeaderMagic } from './types';
import * as utils from '../utils';
import * as utilsErrors from '../utils/errors';
import * as validationErrors from '../validation/errors';

/**
Expand Down Expand Up @@ -337,28 +335,25 @@ async function* encodeContent(
* Takes an AsyncGenerator<TreeNode> and serializes it into a `ReadableStream<UInt8Array>`
* @param fs
* @param treeGen - An AsyncGenerator<TreeNode> that yields the files and directories of a file tree.
* @param yieldContents - Toggles sending the contents of files after the file tree.
*/
function serializerStreamFactory(
fs: FileSystem | FileSystemReadable,
treeGen: AsyncGenerator<TreeNode, void, void>,
): ReadableStream<Uint8Array> {
let contentsGen: AsyncGenerator<Uint8Array, void, void> | undefined;
let fileNode: TreeNode | undefined;

async function getNextFileNode(): Promise<TreeNode | undefined> {
while (true) {
const result = await treeGen.next();
if (result.done) return undefined;
if (result.value.type === 'FILE') return result.value;
// If it's not a file, keep iterating
}
}
async function getNextContentChunk(): Promise<Uint8Array | undefined> {
while (true) {
if (contentsGen == null) {
fileNode = await getNextFileNode();
if (fileNode == null) return undefined;
// Keep consuming values if the result is not a file
while (true) {
const result = await treeGen.next();
if (result.done) return undefined;
if (result.value.type === 'FILE') {
fileNode = result.value;
break;
}
}
contentsGen = encodeContent(fs, fileNode.path, fileNode.iNode);
}
const contentChunk = await contentsGen.next();
Expand All @@ -374,12 +369,8 @@ function serializerStreamFactory(
pull: async (controller) => {
try {
const contentChunk = await getNextContentChunk();
if (contentChunk == null) {
return controller.close();
}
else {
controller.enqueue(contentChunk);
}
if (contentChunk == null) return controller.close();
else controller.enqueue(contentChunk);
} catch (e) {
await cleanup(e);
return controller.error(e);
Expand Down Expand Up @@ -447,13 +438,13 @@ function parseTreeNode(data: unknown): asserts data is TreeNode {
*/
function parserTransformStreamFactory(): TransformStream<
Uint8Array,
string | ContentNode | Uint8Array
ContentNode | Uint8Array
> {
let workingBuffer: Uint8Array = new Uint8Array(0);
let contentLength: bigint | undefined = undefined;
let processedChunks: boolean = false;

return new TransformStream<Uint8Array, ContentNode | Uint8Array | string>({
return new TransformStream<Uint8Array, ContentNode | Uint8Array>({
/**
* Check if any chunks have been processed. If the stream is being flushed
* without processing any chunks, then something went wrong with the stream.
Expand All @@ -467,10 +458,7 @@ function parserTransformStreamFactory(): TransformStream<
},
transform: (chunk, controller) => {
if (chunk.byteLength > 0) processedChunks = true;
workingBuffer = vaultsUtils.uint8ArrayConcat([
workingBuffer,
chunk,
]);
workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]);
if (contentLength == null) {
const genericHeader = parseGenericHeader(workingBuffer);
if (genericHeader.data == null) return;
Expand All @@ -494,31 +482,14 @@ function parserTransformStreamFactory(): TransformStream<
if (workingBuffer.byteLength === 0) return;
if (workingBuffer.byteLength <= contentLength) {
contentLength -= BigInt(workingBuffer.byteLength);
const fileContents = new TextDecoder().decode(workingBuffer); // newcode
controller.enqueue(fileContents); // newcode
// controller.enqueue(workingBuffer);
controller.enqueue(workingBuffer);
workingBuffer = new Uint8Array(0);
if (contentLength === 0n) contentLength = undefined;
// return;
} else {
// controller.enqueue(
// workingBuffer.subarray(0, Number(contentLength)),
// );
const contentChunk = workingBuffer.subarray(0, Number(contentLength)); // new
const contentString = new TextDecoder().decode(contentChunk); // nwe
controller.enqueue(contentString); // nwe
controller.enqueue(workingBuffer.subarray(0, Number(contentLength)));
workingBuffer = workingBuffer.subarray(Number(contentLength));
contentLength = undefined;
}
// return;
// default:
// controller.error(
// new utilsErrors.ErrorUtilsUndefinedBehaviour(
// `invalid state "${phase}"`,
// ),
// );
// return;
// }
},
});
}
Expand Down
35 changes: 12 additions & 23 deletions tests/vaults/fileTree.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import type { ContentNode, FileTree, TreeNode } from '@/vaults/types';
import type { ContentNode, FileTree } from '@/vaults/types';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { ReadableStream } from 'stream/web';
import { test } from '@fast-check/jest';
import fc, { uint8Array } from 'fast-check';
import fc from 'fast-check';
import * as fileTree from '@/vaults/fileTree';
import * as vaultsTestUtils from './utils';

Expand Down Expand Up @@ -494,12 +494,11 @@ describe('fileTree', () => {
yieldParents: true,
yieldDirectories: true,
});
const data: Array<TreeNode | ContentNode | Uint8Array> = [];
const data: Array<ContentNode | Uint8Array> = [];
const parserTransform = fileTree.parserTransformStreamFactory();
const serializedStream = fileTree.serializerStreamFactory(
fs,
fileTreeGen,
false,
);
const outputStream = serializedStream.pipeThrough(parserTransform);
for await (const output of outputStream) {
Expand Down Expand Up @@ -537,15 +536,14 @@ describe('fileTree', () => {
yieldParents: true,
yieldDirectories: true,
});
const data: Array<TreeNode | ContentNode | Uint8Array> = [];
const data: Array<ContentNode | Uint8Array> = [];
const snipperTransform = vaultsTestUtils.binaryStreamToSnippedStream([
5, 7, 11, 13,
]);
const parserTransform = fileTree.parserTransformStreamFactory();
const serializedStream = fileTree.serializerStreamFactory(
fs,
fileTreeGen,
false,
);
const outputStream = serializedStream
.pipeThrough(snipperTransform)
Expand Down Expand Up @@ -588,7 +586,6 @@ describe('fileTree', () => {
yieldParents: true,
yieldDirectories: true,
}),
false,
);
const stream2 = fileTree.serializerStreamFactory(
fs,
Expand All @@ -600,7 +597,6 @@ describe('fileTree', () => {
yieldParents: true,
yieldDirectories: true,
}),
false,
);
return new ReadableStream({
start: async (controller) => {
Expand All @@ -610,7 +606,7 @@ describe('fileTree', () => {
},
});
}
const data: Array<TreeNode | ContentNode | Uint8Array> = [];
const data: Array<ContentNode | Uint8Array> = [];
const parserTransform = fileTree.parserTransformStreamFactory();
// Const serializedStream = fileTree.serializerStreamFactory(fileTreeGen);
const serializedStream = doubleWalkFactory();
Expand Down Expand Up @@ -657,12 +653,11 @@ describe('fileTree', () => {
yieldParents: false,
yieldDirectories: false,
});
const data: Array<TreeNode | ContentNode | Uint8Array> = [];
const data: Array<ContentNode | Uint8Array> = [];
const parserTransform = fileTree.parserTransformStreamFactory();
const serializedStream = fileTree.serializerStreamFactory(
fs,
fileTreeGen,
true,
);
const outputStream = serializedStream.pipeThrough(parserTransform);
for await (const output of outputStream) {
Expand Down Expand Up @@ -707,9 +702,7 @@ describe('fileTree', () => {
const parserTransform = fileTree.parserTransformStreamFactory();
const outputStream = dataStream.pipeThrough(parserTransform);
try {
for await (const _ of outputStream) {
// Only consume
}
for await (const _ of outputStream); // Consume values
} catch {
return;
}
Expand All @@ -729,17 +722,13 @@ describe('fileTree', () => {
yieldParents: false,
yieldDirectories: false,
});
let serializedStream = fileTree.serializerStreamFactory(fs, fileTreeGen);
// const data: Array<Uint8Array> = [];
// for await (const d of serializedStream) {
// data.push(d);
// }
// console.log(data.map((v) => Buffer.from(v as Uint8Array).toString()));

// serializedStream = fileTree.serializerStreamFactory(fs, fileTreeGen);
const serializedStream = fileTree.serializerStreamFactory(
fs,
fileTreeGen,
);
const parserTransform = fileTree.parserTransformStreamFactory();
const outputStream = serializedStream.pipeThrough(parserTransform);
const output: Array<string> = [];
const output: Array<ContentNode | Uint8Array> = [];
for await (const d of outputStream) {
output.push(d);
}
Expand Down

0 comments on commit 7531ef0

Please sign in to comment.