diff --git a/websocketfs/lib/metadata-file.ts b/websocketfs/lib/metadata-file.ts index 781f6c2..21f783b 100644 --- a/websocketfs/lib/metadata-file.ts +++ b/websocketfs/lib/metadata-file.ts @@ -1,8 +1,7 @@ import debug from "debug"; import { stat, readFile } from "fs/promises"; -import { decode } from "lz4"; import binarySearch from "binarysearch"; -import { symbolicToMode } from "./util"; +import { symbolicToMode, readFileLz4 } from "./util"; const log = debug("websocketfs:metadata-file"); const log_cache = debug("cache"); @@ -46,6 +45,7 @@ export class MetadataFile { try { const { mtimeMs } = await stat(this.metadataFile); if (mtimeMs <= this.lastMtimeMs) { + log("metadataFile:", this.metadataFile, "watching for changes"); // it hasn't changed so nothing to do return; } @@ -53,28 +53,31 @@ export class MetadataFile { this.lastMtimeMs = mtimeMs; let content = await readFile(this.metadataFile); if (this.metadataFile.endsWith(".lz4")) { - content = decode(content); + // We use a stream instead of blocking and using + // lz4's decode because there is a HUGE bug in + // the sync api of lz4 -- + // https://github.com/pierrec/node-lz4/issues/117 + content = await readFileLz4(this.metadataFile); } this.metadataFileContents = content.toString().split("\0\0"); this.metadataFileContents.sort(); this.state = "ready"; log( - `metadataFile: "${this.metadataFile}" is NEW -- parsed in `, + "metadataFile:", + this.metadataFile, + `CHANGED - ${this.metadataFileContents.length} files - parsed in `, Date.now() - start, "ms", ); } catch (err) { log( - "metadataFile: not reading -- ", + "metadataFile:", + this.metadataFile, + " - non-fatal issue reading - ", err.code == "ENOENT" - ? `no file '${this.metadataFile}' -- disabling metadata file cache` + ? `no file '${this.metadataFile}' -- NOT updating metadata file cache (will try again soon)` : err, ); - // expire the metadataFile cache contents. - // NOTE: this could take slightly longer than cacheTimeoutMs, depending - // on METADATA_FILE_INTERVAL_MS, but for my application I don't care. - this.state = "expired"; - this.metadataFileContents = []; } }; diff --git a/websocketfs/lib/util.ts b/websocketfs/lib/util.ts index ad92464..8fbf62b 100644 --- a/websocketfs/lib/util.ts +++ b/websocketfs/lib/util.ts @@ -2,6 +2,9 @@ Various utilities */ +import { createDecoderStream } from "lz4"; +import { createReadStream } from "fs"; +import { PassThrough } from "stream"; // See cocalc's packages/util/misc for comments function getMethods(obj: object): string[] { @@ -61,3 +64,21 @@ export function symbolicToMode(symbolic): number { return parseInt(mode, 8); } + +export async function readFileLz4(path: string): Promise { + const decoder = createDecoderStream(); + const input = createReadStream(path); + const output = new PassThrough(); + input.pipe(decoder).pipe(output); + + const chunks: Buffer[] = []; + const waitForFinish = new Promise((resolve, reject) => { + output.on("finish", resolve); + output.on("error", reject); + output.on("data", (chunk) => { + chunks.push(chunk); + }); + }); + await waitForFinish; + return Buffer.concat(chunks); +}