new-log-viewer: Add ClpIrDecoder. (#62)

new-log-viewer/package.json

@@ -19,6 +19,7 @@
   "homepage": "https://github.com/y-scope/yscope-log-viewer#readme",
   "dependencies": {
     "axios": "^1.7.2",
+    "clp-ffi-js": "^0.1.0",
     "dayjs": "^1.11.11",
     "monaco-editor": "^0.50.0",
     "react": "^18.3.1",

new-log-viewer/src/services/LogFileManager.ts

@@ -1,6 +1,7 @@
 import {
     Decoder,
     DecoderOptionsType,
+    LOG_EVENT_FILE_END_IDX,
 } from "../typings/decoders";
 import {MAX_V8_STRING_LENGTH} from "../typings/js";
 import {
@@ -13,6 +14,7 @@ import {getUint8ArrayFrom} from "../utils/http";
 import {getChunkNum} from "../utils/math";
 import {formatSizeInBytes} from "../utils/units";
 import {getBasenameFromUrlOrDefault} from "../utils/url";
+import ClpIrDecoder from "./decoders/ClpIrDecoder";
 import JsonlDecoder from "./decoders/JsonlDecoder";
 
 
@@ -48,8 +50,6 @@ const loadFile = async (fileSrc: FileSrcType)
 class LogFileManager {
     readonly #pageSize: number;
 
-    readonly #fileData: Uint8Array;
-
     readonly #fileName: string;
 
     #decoder: Decoder;
@@ -60,21 +60,27 @@ class LogFileManager {
      * Private constructor for LogFileManager. This is not intended to be invoked publicly.
      * Instead, use LogFileManager.create() to create a new instance of the class.
      *
+     * @param decoder
      * @param fileName
-     * @param fileData
      * @param pageSize Page size for setting up pagination.
-     * @param decoderOptions Initial decoder options.
      */
     constructor (
+        decoder: Decoder,
         fileName: string,
-        fileData: Uint8Array,
         pageSize: number,
-        decoderOptions: DecoderOptionsType
     ) {
         this.#fileName = fileName;
-        this.#fileData = fileData;
         this.#pageSize = pageSize;
-        this.#decoder = this.#initDecoder(decoderOptions);
+        this.#decoder = decoder;
+
+        // Build index for the entire file
+        const buildIdxResult = decoder.buildIdx(0, LOG_EVENT_FILE_END_IDX);
+        if (null !== buildIdxResult && 0 < buildIdxResult.numInvalidEvents) {
+            console.error("Invalid events found in decoder.buildIdx():", buildIdxResult);
+        }
+
+        this.#numEvents = decoder.getEstimatedNumEvents();
+        console.log(`Found ${this.#numEvents} log events.`);
     }
 
     get numEvents () {
@@ -96,7 +102,41 @@ class LogFileManager {
         decoderOptions: DecoderOptionsType
     ): Promise<LogFileManager> {
         const {fileName, fileData} = await loadFile(fileSrc);
-        return new LogFileManager(fileName, fileData, pageSize, decoderOptions);
+        const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions);
+
+        return new LogFileManager(decoder, fileName, pageSize);
+    }
+
+    /**
+     * Constructs a decoder instance based on the file extension.
+     *
+     * @param fileName
+     * @param fileData
+     * @param decoderOptions Initial decoder options.
+     * @return The constructed decoder.
+     * @throws {Error} if no decoder supports a file with the given extension.
+     */
+    static async #initDecoder (
+        fileName: string,
+        fileData: Uint8Array,
+        decoderOptions: DecoderOptionsType
+    ): Promise<Decoder> {
+        let decoder: Decoder;
+        if (fileName.endsWith(".jsonl")) {
+            decoder = new JsonlDecoder(fileData, decoderOptions);
+        } else if (fileName.endsWith(".clp.zst")) {
+            decoder = await ClpIrDecoder.create(fileData);
+        } else {
+            throw new Error(`No decoder supports ${fileName}`);
+        }
+
+        if (fileData.length > MAX_V8_STRING_LENGTH) {
+            throw new Error(`Cannot handle files larger than ${
+                formatSizeInBytes(MAX_V8_STRING_LENGTH)
+            } due to a limitation in Chromium-based browsers.`);
+        }
+
+        return decoder;
     }
 
     /**
@@ -154,33 +194,6 @@ class LogFileManager {
         };
     }
 
-    /**
-     * Constructs a decoder instance based on the file extension.
-     *
-     * @param decoderOptions Initial decoder options.
-     * @return The constructed decoder.
-     * @throws {Error} if a decoder cannot be found.
-     */
-    #initDecoder = (decoderOptions: DecoderOptionsType): Decoder => {
-        let decoder: Decoder;
-        if (this.#fileName.endsWith(".jsonl")) {
-            decoder = new JsonlDecoder(this.#fileData, decoderOptions);
-        } else {
-            throw new Error(`No decoder supports ${this.#fileName}`);
-        }
-
-        if (this.#fileData.length > MAX_V8_STRING_LENGTH) {
-            throw new Error(`Cannot handle files larger than ${
-                formatSizeInBytes(MAX_V8_STRING_LENGTH)
-            } due to a limitation in Chromium-based browsers.`);
-        }
-
-        this.#numEvents = decoder.getEstimatedNumEvents();
-        console.log(`Found ${this.#numEvents} log events.`);
-
-        return decoder;
-    };
-
     /**
      * Gets the range of log event numbers for the page containing the given cursor.
      *

new-log-viewer/src/services/decoders/ClpIrDecoder.ts

@@ -0,0 +1,51 @@
+import clpFfiJsModuleInit, {ClpIrStreamReader} from "clp-ffi-js";
+
+import {Nullable} from "../../typings/common";
+import {
+    Decoder,
+    DecodeResultType,
+    LogEventCount,
+} from "../../typings/decoders";
+
+
+class ClpIrDecoder implements Decoder {
+    #streamReader: ClpIrStreamReader;
+
+    constructor (streamReader: ClpIrStreamReader) {
+        this.#streamReader = streamReader;
+    }
+
+    /**
+     * Creates a new ClpIrDecoder instance.
+     *
+     * @param dataArray The input data array to be passed to the decoder.
+     * @return The created ClpIrDecoder instance.
+     */
+    static async create (dataArray: Uint8Array): Promise<ClpIrDecoder> {
+        const module = await clpFfiJsModuleInit();
+        const streamReader = new module.ClpIrStreamReader(dataArray);
+        return new ClpIrDecoder(streamReader);
+    }
+
+    getEstimatedNumEvents (): number {
+        return this.#streamReader.getNumEventsBuffered();
+    }
+
+    buildIdx (beginIdx: number, endIdx: number): Nullable<LogEventCount> {
+        return {
+            numInvalidEvents: 0,
+            numValidEvents: this.#streamReader.deserializeRange(beginIdx, endIdx),
+        };
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    setDecoderOptions (): boolean {
+        return true;
+    }
+
+    decode (beginIdx: number, endIdx: number): Nullable<DecodeResultType[]> {
+        return this.#streamReader.decodeRange(beginIdx, endIdx);
+    }
+}
+
+export default ClpIrDecoder;

new-log-viewer/src/typings/decoders.ts

@@ -45,7 +45,7 @@ interface Decoder {
      * When applicable, deserializes log events in the range `[beginIdx, endIdx)`.
      *
      * @param beginIdx
-     * @param endIdx
+     * @param endIdx End index. To deserialize to the end of the file, use `LOG_EVENT_FILE_END_IDX`.
      * @return Count of the successfully deserialized ("valid") log events and count of any
      * un-deserializable ("invalid") log events within the range; or null if any log event in the
      * range doesn't exist (e.g., the range exceeds the number of log events in the file).
@@ -71,7 +71,13 @@ interface Decoder {
     decode(beginIdx: number, endIdx: number): Nullable<DecodeResultType[]>;
 }
 
+/**
+ * Index for specifying the end of the file when the exact number of log events it contains is
+ *  unknown.
+ */
+const LOG_EVENT_FILE_END_IDX: number = 0;
 
+export {LOG_EVENT_FILE_END_IDX};
 export type {
     Decoder,
     DecodeResultType,