From ccee8b56b2447faa94d38929e986fa198e03c9c6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 2 Sep 2024 16:08:02 -0400 Subject: [PATCH] Support setting log level filter via decoder options. --- new-log-viewer/src/services/LogFileManager.ts | 8 +-- .../src/services/decoders/JsonlDecoder.ts | 54 ++++++++++++++----- new-log-viewer/src/typings/config.ts | 4 +- new-log-viewer/src/typings/decoders.ts | 14 +++-- new-log-viewer/src/typings/worker.ts | 6 +-- new-log-viewer/src/utils/config.ts | 4 +- 6 files changed, 61 insertions(+), 29 deletions(-) diff --git a/new-log-viewer/src/services/LogFileManager.ts b/new-log-viewer/src/services/LogFileManager.ts index ea73d7fa..3508f1e0 100644 --- a/new-log-viewer/src/services/LogFileManager.ts +++ b/new-log-viewer/src/services/LogFileManager.ts @@ -1,6 +1,6 @@ import { Decoder, - DecoderOptionsType, + DecoderOptions, LOG_EVENT_FILE_END_IDX, } from "../typings/decoders"; import {MAX_V8_STRING_LENGTH} from "../typings/js"; @@ -103,7 +103,7 @@ class LogFileManager { static async create ( fileSrc: FileSrcType, pageSize: number, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptions ): Promise { const {fileName, fileData} = await loadFile(fileSrc); const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions); @@ -123,7 +123,7 @@ class LogFileManager { static async #initDecoder ( fileName: string, fileData: Uint8Array, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptions ): Promise { let decoder: Decoder; if (fileName.endsWith(".jsonl")) { @@ -148,7 +148,7 @@ class LogFileManager { * * @param options */ - setDecoderOptions (options: DecoderOptionsType) { + setDecoderOptions (options: DecoderOptions) { this.#decoder.setDecoderOptions(options); } diff --git a/new-log-viewer/src/services/decoders/JsonlDecoder.ts b/new-log-viewer/src/services/decoders/JsonlDecoder.ts index f87d8a12..feb4722a 100644 --- a/new-log-viewer/src/services/decoders/JsonlDecoder.ts +++ b/new-log-viewer/src/services/decoders/JsonlDecoder.ts @@ -4,7 +4,7 @@ import {Nullable} from "../../typings/common"; import { Decoder, DecodeResultType, - JsonlDecoderOptionsType, + JsonlDecoderOptions, JsonLogEvent, LOG_EVENT_FILE_END_IDX, LogEventCount, @@ -30,12 +30,14 @@ class JsonlDecoder implements Decoder { #dataArray: Nullable; - #logLevelKey: string = "level"; + readonly #logLevelKey: string; - #timestampKey: string = "@timestamp"; + readonly #timestampKey: string; #logEvents: JsonLogEvent[] = []; + #filteredLogIndices: number[]; + #invalidLogEventIdxToRawLine: Map = new Map(); // @ts-expect-error #formatter is set in the constructor by `setDecoderOptions()` @@ -46,7 +48,12 @@ class JsonlDecoder implements Decoder { * @param decoderOptions * @throws {Error} if the initial decoder options are erroneous. */ - constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) { + constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) { + this.#filteredLogIndices = []; + + this.#logLevelKey = decoderOptions.logLevelKey; + this.#timestampKey = decoderOptions.timestampKey; + const isOptionSet = this.setDecoderOptions(decoderOptions); if (false === isOptionSet) { throw new Error( @@ -57,7 +64,7 @@ class JsonlDecoder implements Decoder { } getEstimatedNumEvents (): number { - return this.#logEvents.length; + return this.#filteredLogIndices.length; } buildIdx (beginIdx: number, endIdx: number): Nullable { @@ -66,6 +73,7 @@ class JsonlDecoder implements Decoder { } this.#deserialize(); + this.#filterLogs(null); const numInvalidEvents = Array.from(this.#invalidLogEventIdxToRawLine.keys()) .filter((eventIdx) => (beginIdx <= eventIdx && eventIdx < endIdx)) .length; @@ -76,17 +84,16 @@ class JsonlDecoder implements Decoder { }; } - setDecoderOptions (options: JsonlDecoderOptionsType): boolean { - // FIXME: If options changed then parse log events again. + setDecoderOptions (options: JsonlDecoderOptions): boolean { + // Note `options.timestampKey` and `options.logLevelKey` are not set by this method. this.#formatter = new LogbackFormatter(options); - this.#logLevelKey = options.logLevelKey; - this.#timestampKey = options.timestampKey; + this.#filterLogs(options.logLevelFilter); return true; } decode (beginIdx: number, endIdx: number): Nullable { - if (0 > beginIdx || this.#logEvents.length < endIdx) { + if (0 > beginIdx || this.#filteredLogIndices.length < endIdx) { return null; } @@ -94,7 +101,13 @@ class JsonlDecoder implements Decoder { // TODO We could probably optimize this to avoid checking `#invalidLogEventIdxToRawLine` on // every iteration. const results: DecodeResultType[] = []; - for (let logEventIdx = beginIdx; logEventIdx < endIdx; logEventIdx++) { + for (let filteredLogEventIdx = beginIdx; + filteredLogEventIdx < endIdx; + filteredLogEventIdx++) { + // Explicit cast since typescript thinks `#filteredLogIndices[filteredLogEventIdx]` can + // be undefined, but it shouldn't be since we performed a bounds check at the beginning + // of the method. + const logEventIdx = this.#filteredLogIndices[filteredLogEventIdx] as number; let logLevel: LOG_LEVEL; let message: string; let timestamp: number; @@ -104,8 +117,7 @@ class JsonlDecoder implements Decoder { timestamp = INVALID_TIMESTAMP_VALUE; } else { // Explicit cast since typescript thinks `#logEvents[logEventIdx]` can be undefined, - // but it shouldn't be since we performed a bounds check at the beginning of the - // method. + // but it shouldn't be since the index comes from a class-internal filter. const logEvent: JsonLogEvent = this.#logEvents[logEventIdx] as JsonLogEvent; logLevel = logEvent.level; message = this.#formatter.formatLogEvent(logEvent); @@ -172,6 +184,22 @@ class JsonlDecoder implements Decoder { this.#dataArray = null; } + #filterLogs (logLevelFilter: Nullable) { + this.#filteredLogIndices.length = 0; + if (null === logLevelFilter) { + this.#filteredLogIndices = Array.from( + {length: this.#logEvents.length}, + (_, index) => index + ); + + return; + } + this.#logEvents.forEach((logEvent, index) => { + if (logEvent.level in logLevelFilter) { + this.#filteredLogIndices.push(index); + } + }); + } /** * Parses the log level from the given log event. diff --git a/new-log-viewer/src/typings/config.ts b/new-log-viewer/src/typings/config.ts index 7f80b3c5..7e362421 100644 --- a/new-log-viewer/src/typings/config.ts +++ b/new-log-viewer/src/typings/config.ts @@ -1,4 +1,4 @@ -import {JsonlDecoderOptionsType} from "./decoders"; +import {JsonlDecoderOptions} from "./decoders"; enum THEME_NAME { @@ -24,7 +24,7 @@ enum LOCAL_STORAGE_KEY { /* eslint-enable @typescript-eslint/prefer-literal-enum-member */ type ConfigMap = { - [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType, + [CONFIG_KEY.DECODER_OPTIONS]: Partial, [CONFIG_KEY.THEME]: THEME_NAME, [CONFIG_KEY.PAGE_SIZE]: number, }; diff --git a/new-log-viewer/src/typings/decoders.ts b/new-log-viewer/src/typings/decoders.ts index a213eff6..c8679f17 100644 --- a/new-log-viewer/src/typings/decoders.ts +++ b/new-log-viewer/src/typings/decoders.ts @@ -10,6 +10,10 @@ interface LogEventCount { numInvalidEvents: number, } +interface GenericDecoderOptions { + logLevelFilter: Nullable, +} + /** * Options for the JSONL decoder. * @@ -17,13 +21,13 @@ interface LogEventCount { * @property logLevelKey The key of the kv-pair that contains the log level in every record. * @property timestampKey The key of the kv-pair that contains the timestamp in every record. */ -interface JsonlDecoderOptionsType { +interface JsonlDecoderOptions extends GenericDecoderOptions { formatString: string, logLevelKey: string, timestampKey: string, } -type DecoderOptionsType = JsonlDecoderOptionsType; +type DecoderOptions = JsonlDecoderOptions; /** * A log event parsed from a JSON log. @@ -71,7 +75,7 @@ interface Decoder { * @param options * @return Whether the options were successfully set. */ - setDecoderOptions(options: DecoderOptionsType): boolean; + setDecoderOptions(options: DecoderOptions): boolean; /** * Decodes the log events in the range `[beginIdx, endIdx)`. @@ -94,8 +98,8 @@ export {LOG_EVENT_FILE_END_IDX}; export type { Decoder, DecodeResultType, - DecoderOptionsType, - JsonlDecoderOptionsType, + DecoderOptions, + JsonlDecoderOptions, JsonLogEvent, LogEventCount, }; diff --git a/new-log-viewer/src/typings/worker.ts b/new-log-viewer/src/typings/worker.ts index 0c88beb6..dd8644b6 100644 --- a/new-log-viewer/src/typings/worker.ts +++ b/new-log-viewer/src/typings/worker.ts @@ -1,4 +1,4 @@ -import {DecoderOptionsType} from "./decoders"; +import {DecoderOptions} from "./decoders"; import {LOG_LEVEL} from "./logs"; @@ -53,11 +53,11 @@ type WorkerReqMap = { fileSrc: FileSrcType, pageSize: number, cursor: CursorType, - decoderOptions: DecoderOptionsType + decoderOptions: DecoderOptions }, [WORKER_REQ_CODE.LOAD_PAGE]: { cursor: CursorType, - decoderOptions?: DecoderOptionsType + decoderOptions?: DecoderOptions }, }; diff --git a/new-log-viewer/src/utils/config.ts b/new-log-viewer/src/utils/config.ts index 895c1950..e7cc9548 100644 --- a/new-log-viewer/src/utils/config.ts +++ b/new-log-viewer/src/utils/config.ts @@ -6,7 +6,7 @@ import { LOCAL_STORAGE_KEY, THEME_NAME, } from "../typings/config"; -import {DecoderOptionsType} from "../typings/decoders"; +import {DecoderOptions} from "../typings/decoders"; const MAX_PAGE_SIZE = 1_000_000; @@ -122,7 +122,7 @@ const getConfig = (key: T): ConfigMap[T] => { timestampKey: window.localStorage.getItem( LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY ), - } as DecoderOptionsType; + } as DecoderOptions; break; case CONFIG_KEY.THEME: { value = window.localStorage.getItem(LOCAL_STORAGE_KEY.THEME);