diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index db59260d..b119d7f0 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -33,8 +33,12 @@ import ThemeSwitchToggle from "./ThemeSwitchToggle"; const CONFIG_FORM_FIELDS = [ { - helperText: "[JSON] Log messages conversion pattern. The current syntax is similar to" + - " Logback conversion patterns but will change in a future release.", + helperText: `[JSON] Log message conversion pattern: use field placeholders to insert + values from JSON log events. The syntax is + \`{[:[:]]}\`, where \`field-name\` is + required, while \`formatter-name\` and \`formatter-options\` are optional. For example, + the following placeholder would format a timestamp field with name \`@timestamp\`: + \`{@timestamp:timestamp:YYYY-MM-DD HH\\:mm\\:ss.SSS}\`.`, initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).formatString, label: "Decoder: Format string", name: LOCAL_STORAGE_KEY.DECODER_OPTIONS_FORMAT_STRING, diff --git a/src/services/decoders/ClpIrDecoder.ts b/src/services/decoders/ClpIrDecoder.ts index 0f801664..81383259 100644 --- a/src/services/decoders/ClpIrDecoder.ts +++ b/src/services/decoders/ClpIrDecoder.ts @@ -12,7 +12,7 @@ import { import {Formatter} from "../../typings/formatters"; import {JsonObject} from "../../typings/js"; import {LogLevelFilter} from "../../typings/logs"; -import LogbackFormatter from "../formatters/LogbackFormatter"; +import YscopeFormatter from "../formatters/YscopeFormatter"; import { convertToDayjsTimestamp, isJsonObject, @@ -39,7 +39,7 @@ class ClpIrDecoder implements Decoder { this.#streamType = streamType; this.#streamReader = streamReader; this.#formatter = (streamType === CLP_IR_STREAM_TYPE.STRUCTURED) ? - new LogbackFormatter({formatString: decoderOptions.formatString}) : + new YscopeFormatter({formatString: decoderOptions.formatString}) : null; } @@ -87,7 +87,7 @@ class ClpIrDecoder implements Decoder { } setFormatterOptions (options: DecoderOptions): boolean { - this.#formatter = new LogbackFormatter({formatString: options.formatString}); + this.#formatter = new YscopeFormatter({formatString: options.formatString}); return true; } diff --git a/src/services/decoders/JsonlDecoder/index.ts b/src/services/decoders/JsonlDecoder/index.ts index 5b873ed7..5405ca00 100644 --- a/src/services/decoders/JsonlDecoder/index.ts +++ b/src/services/decoders/JsonlDecoder/index.ts @@ -16,7 +16,7 @@ import { LogEvent, LogLevelFilter, } from "../../../typings/logs"; -import LogbackFormatter from "../../formatters/LogbackFormatter"; +import YscopeFormatter from "../../formatters/YscopeFormatter"; import { convertToDayjsTimestamp, convertToLogLevelValue, @@ -53,7 +53,7 @@ class JsonlDecoder implements Decoder { this.#dataArray = dataArray; this.#logLevelKey = decoderOptions.logLevelKey; this.#timestampKey = decoderOptions.timestampKey; - this.#formatter = new LogbackFormatter({formatString: decoderOptions.formatString}); + this.#formatter = new YscopeFormatter({formatString: decoderOptions.formatString}); } getEstimatedNumEvents (): number { @@ -82,7 +82,7 @@ class JsonlDecoder implements Decoder { } setFormatterOptions (options: DecoderOptions): boolean { - this.#formatter = new LogbackFormatter({formatString: options.formatString}); + this.#formatter = new YscopeFormatter({formatString: options.formatString}); return true; } diff --git a/src/services/formatters/LogbackFormatter.ts b/src/services/formatters/LogbackFormatter.ts deleted file mode 100644 index 8caf91d3..00000000 --- a/src/services/formatters/LogbackFormatter.ts +++ /dev/null @@ -1,152 +0,0 @@ -import dayjs from "dayjs"; - -import {Nullable} from "../../typings/common"; -import { - Formatter, - FormatterOptionsType, -} from "../../typings/formatters"; -import {JsonObject} from "../../typings/js"; -import {LogEvent} from "../../typings/logs"; - - -/** - * Converts a *simple* java.time.format.DateTimeFormatter pattern to a Day.js date format string. - * - * NOTE: This method doesn't handle all possible patterns. Check the implementation to determine - * what's supported. - * - * @param pattern - * @return The corresponding Day.js date format string. - */ -const convertDateTimeFormatterPatternToDayJs = (pattern: string): string => { - pattern = pattern.replace("yyyy", "YYYY"); - pattern = pattern.replace("yy", "YY"); - pattern = pattern.replace("dd", "D"); - pattern = pattern.replace("d", "D"); - - return pattern; -}; - -/** - * A formatter that uses a Logback-like format string to format log events into a string. See - * `LogbackFormatterOptionsType` for details about the format string. - */ -class LogbackFormatter implements Formatter { - #formatString: string; - - #datePattern: Nullable = null; - - #dateFormat: string = ""; - - #keys: string[] = []; - - constructor (options: FormatterOptionsType) { - // NOTE: It's safe for these values to be empty strings. - this.#formatString = options.formatString; - - // Remove new line - this.#formatString = this.#formatString.replace("%n", ""); - - this.#parseDateFormat(); - this.#parseKeys(); - } - - /** - * Formats the given log event. - * - * @param logEvent - * @return The formatted log event. - */ - formatLogEvent (logEvent: LogEvent): string { - const {fields, timestamp} = logEvent; - const formatStringWithTimestamp: string = - this.#formatTimestamp(timestamp, this.#formatString); - - return this.#formatVariables(formatStringWithTimestamp, fields); - } - - /** - * Parses the timestamp specifier from the format string and converts the date pattern into a - * Day.js-compatible one. - */ - #parseDateFormat () { - const dateFormatMatch = this.#formatString.match(/%d\{(.+?)}/); - if (null === dateFormatMatch) { - console.warn("Unable to find date format string in #formatString:", this.#formatString); - - return; - } - - // E.g. "%d{yyyy-MM-dd HH:mm:ss.SSS}", "yyyy-MM-dd HH:mm:ss.SSS" - // Explicit cast since typescript thinks `dateFormat` can be undefined, but it can't be - // since the pattern contains a capture group. - const [pattern, dateFormat] = - <[pattern: RegExpMatchArray[0], dateFormat: string]>dateFormatMatch; - - this.#datePattern = pattern; - - this.#dateFormat = convertDateTimeFormatterPatternToDayJs(dateFormat); - } - - /** - * Parses all non-Logback specifiers (expected log event keys) from the format string. - */ - #parseKeys () { - const specifierRegex = /%([\w.]+)/g; - for (const match of this.#formatString.matchAll(specifierRegex)) { - // E.g., "%thread", "thread" - const [, key] = match; - - // Explicit cast since typescript thinks `key` can be undefined, but it can't be - // since the pattern contains a capture group. - this.#keys.push(key as string); - } - } - - /** - * Replaces the timestamp specifier in `formatString` with `timestamp`, formatted with - * `#dateFormat`. - * - * @param timestamp - * @param formatString - * @return The formatted string. - */ - #formatTimestamp (timestamp: dayjs.Dayjs, formatString: string): string { - if (null === this.#datePattern) { - return formatString; - } - - const formattedDate = timestamp.format(this.#dateFormat); - formatString = formatString.replace(this.#datePattern, formattedDate); - - return formatString; - } - - /** - * Replaces format specifiers in `formatString` with the corresponding kv-pairs from `logEvent`. - * - * @param formatString - * @param logEvent - * @return The formatted string. - */ - #formatVariables (formatString: string, logEvent: JsonObject): string { - // eslint-disable-next-line no-warning-comments - // TODO These don't handle the case where a variable value may contain a '%' itself - for (const key of this.#keys) { - if (false === (key in logEvent)) { - continue; - } - const specifier = `%${key}`; - const value = logEvent[key]; - const valueStr = "object" === typeof value ? - JSON.stringify(value) : - String(value); - - formatString = formatString.replace(specifier, valueStr); - } - - return `${formatString}\n`; - } -} - -export default LogbackFormatter; diff --git a/src/services/formatters/YscopeFormatter/FieldFormatters/RoundFormatter.ts b/src/services/formatters/YscopeFormatter/FieldFormatters/RoundFormatter.ts new file mode 100644 index 00000000..937d88a4 --- /dev/null +++ b/src/services/formatters/YscopeFormatter/FieldFormatters/RoundFormatter.ts @@ -0,0 +1,29 @@ +import {Nullable} from "../../../../typings/common"; +import {YscopeFieldFormatter} from "../../../../typings/formatters"; +import {JsonValue} from "../../../../typings/js"; +import {jsonValueToString} from "../utils"; + + +/** + * A field formatter that rounds numerical values to the nearest integer. + * For non-numerical values, the field's value is converted to a string then returned as-is. + * Options: None. + */ +class RoundFormatter implements YscopeFieldFormatter { + constructor (options: Nullable) { + if (null !== options) { + throw Error(`RoundFormatter does not support options "${options}"`); + } + } + + // eslint-disable-next-line class-methods-use-this + formatField (field: JsonValue): string { + if ("number" === typeof field) { + field = Math.round(field); + } + + return jsonValueToString(field); + } +} + +export default RoundFormatter; diff --git a/src/services/formatters/YscopeFormatter/FieldFormatters/TimestampFormatter.ts b/src/services/formatters/YscopeFormatter/FieldFormatters/TimestampFormatter.ts new file mode 100644 index 00000000..e0847121 --- /dev/null +++ b/src/services/formatters/YscopeFormatter/FieldFormatters/TimestampFormatter.ts @@ -0,0 +1,35 @@ +import {Dayjs} from "dayjs"; + +import {Nullable} from "../../../../typings/common"; +import {YscopeFieldFormatter} from "../../../../typings/formatters"; +import {JsonValue} from "../../../../typings/js"; +import {convertToDayjsTimestamp} from "../../../decoders/JsonlDecoder/utils"; + + +/** + * A formatter for timestamp values, using a specified date-time pattern. + * Options: If no pattern is provided, defaults to ISO 8601 format. + */ +class TimestampFormatter implements YscopeFieldFormatter { + #dateFormat: Nullable = null; + + constructor (options: Nullable) { + this.#dateFormat = options; + } + + formatField (field: JsonValue): string { + // eslint-disable-next-line no-warning-comments + // TODO: We already parsed the timestamp during deserialization so this is perhaps + // inefficient. However, this field formatter can be used for multiple keys, so using + // the single parsed timestamp by itself would not work. Perhaps in future we can check + // if the key is the same as timestamp key and avoid parsing again. + const timestamp: Dayjs = convertToDayjsTimestamp(field); + if (null === this.#dateFormat) { + return timestamp.format(); + } + + return timestamp.format(this.#dateFormat); + } +} + +export default TimestampFormatter; diff --git a/src/services/formatters/YscopeFormatter/index.ts b/src/services/formatters/YscopeFormatter/index.ts new file mode 100644 index 00000000..9997771f --- /dev/null +++ b/src/services/formatters/YscopeFormatter/index.ts @@ -0,0 +1,102 @@ +import {Nullable} from "../../../typings/common"; +import { + FIELD_PLACEHOLDER_REGEX, + Formatter, + FormatterOptionsType, + REPLACEMENT_CHARACTER, + YscopeFieldFormatter, + YscopeFieldPlaceholder, +} from "../../../typings/formatters"; +import {LogEvent} from "../../../typings/logs"; +import { + getFormattedField, + removeEscapeCharacters, + replaceDoubleBacklash, + splitFieldPlaceholder, + YSCOPE_FIELD_FORMATTER_MAP, +} from "./utils"; + + +/** + * A formatter that uses a YScope format string to format log events into a string. See + * `YscopeFormatterOptionsType` for details about the format string. + */ +class YscopeFormatter implements Formatter { + readonly #processedFormatString: string; + + #fieldPlaceholders: YscopeFieldPlaceholder[] = []; + + constructor (options: FormatterOptionsType) { + if (options.formatString.includes(REPLACEMENT_CHARACTER)) { + console.warn("Unicode replacement character `U+FFFD` is found in Decoder Format" + + ' String, which will appear as "\\".'); + } + + this.#processedFormatString = replaceDoubleBacklash(options.formatString); + this.#parseFieldPlaceholder(); + } + + formatLogEvent (logEvent: LogEvent): string { + const formattedLogFragments: string[] = []; + let lastIndex = 0; + + for (const fieldPlaceholder of this.#fieldPlaceholders) { + const formatStringFragment = + this.#processedFormatString.slice(lastIndex, fieldPlaceholder.range.start); + + formattedLogFragments.push(removeEscapeCharacters(formatStringFragment)); + formattedLogFragments.push(getFormattedField(logEvent, fieldPlaceholder)); + lastIndex = fieldPlaceholder.range.end; + } + + const remainder = this.#processedFormatString.slice(lastIndex); + formattedLogFragments.push(removeEscapeCharacters(remainder)); + + return `${formattedLogFragments.join("")}\n`; + } + + /** + * Parses field placeholders in format string. For each field placeholder, creates a + * corresponding `YscopeFieldFormatter` using the placeholder's field name, formatter type, + * and formatter options. Each `YscopeFieldFormatter` is then stored on the + * class-level array `#fieldPlaceholders`. + * + * @throws Error if `FIELD_PLACEHOLDER_REGEX` does not contain a capture group. + * @throws Error if a formatter type is not supported. + */ + #parseFieldPlaceholder () { + const placeholderPattern = new RegExp(FIELD_PLACEHOLDER_REGEX, "g"); + const it = this.#processedFormatString.matchAll(placeholderPattern); + for (const match of it) { + // `fullMatch` includes braces and `groupMatch` excludes them. + const [fullMatch, groupMatch]: (string | undefined) [] = match; + + if ("undefined" === typeof groupMatch) { + throw Error("Field placeholder regex is invalid and does not have a capture group"); + } + + const {fieldNameKeys, formatterName, formatterOptions} = + splitFieldPlaceholder(groupMatch); + + let fieldFormatter: Nullable = null; + if (null !== formatterName) { + const FieldFormatterConstructor = YSCOPE_FIELD_FORMATTER_MAP[formatterName]; + if ("undefined" === typeof FieldFormatterConstructor) { + throw Error(`Formatter ${formatterName} is not currently supported`); + } + fieldFormatter = new FieldFormatterConstructor(formatterOptions); + } + + this.#fieldPlaceholders.push({ + fieldNameKeys: fieldNameKeys, + fieldFormatter: fieldFormatter, + range: { + start: match.index, + end: match.index + fullMatch.length, + }, + }); + } + } +} + +export default YscopeFormatter; diff --git a/src/services/formatters/YscopeFormatter/utils.ts b/src/services/formatters/YscopeFormatter/utils.ts new file mode 100644 index 00000000..16afa501 --- /dev/null +++ b/src/services/formatters/YscopeFormatter/utils.ts @@ -0,0 +1,183 @@ +import {Nullable} from "../../../typings/common"; +import { + COLON_REGEX, + DOUBLE_BACKSLASH, + PERIOD_REGEX, + REPLACEMENT_CHARACTER, + SINGLE_BACKSLASH, + YscopeFieldFormatterMap, + YscopeFieldPlaceholder, +} from "../../../typings/formatters"; +import {JsonValue} from "../../../typings/js"; +import {LogEvent} from "../../../typings/logs"; +import {getNestedJsonValue} from "../../../utils/js"; +import RoundFormatter from "./FieldFormatters/RoundFormatter"; +import TimestampFormatter from "./FieldFormatters/TimestampFormatter"; + + +/** + * List of currently supported field formatters. + */ +const YSCOPE_FIELD_FORMATTER_MAP: YscopeFieldFormatterMap = Object.freeze({ + timestamp: TimestampFormatter, + round: RoundFormatter, +}); + + +/** + * Removes all backslashes from a string. Purpose is to remove escape character in front of brace + * and colon characters. + * + * @param str + * @return Modified string. + */ +const removeBackslash = (str: string): string => { + return str.replaceAll(SINGLE_BACKSLASH, ""); +}; + +/** + * Replaces all replacement characters in format string with a single backslash. Purpose is to + * remove, albeit indirectly through intermediate replacement character, escape character in + * front of a backslash character. + * + * @param str + * @return Modified string. + */ +const replaceReplacementCharacter = (str: string): string => { + return str.replaceAll(REPLACEMENT_CHARACTER, "\\"); +}; + +/** + * Removes escape characters from a string. + * + * @param str + * @return Modified string. + */ +const removeEscapeCharacters = (str: string): string => { + // `removeBackslash()`, which removes all backlashes, is called before + // `replaceReplacementCharacter()` to prevent removal of escaped backslashes. + return replaceReplacementCharacter(removeBackslash(str)); +}; + +/** + * Replaces all escaped backslashes in format string with replacement character. + * Replacement character is a rare character that is unlikely to be in user format string. + * Writing regex to distinguish between a single escape character ("\") and an escaped backslash + * ("\\") is challenging especially when they are in series. It is simpler to just replace + * escaped backslashes with a rare character and add them back after parsing field placeholder + * with regex is finished. + * + * @param formatString + * @return Modified format string. + */ +const replaceDoubleBacklash = (formatString: string): string => { + return formatString.replaceAll(DOUBLE_BACKSLASH, REPLACEMENT_CHARACTER); +}; + + +/** + * Converts a JSON value to its string representation. + * + * @param input + * @return + */ +const jsonValueToString = (input: JsonValue | undefined): string => { + // Behaviour is different for `undefined`. + return "object" === typeof input ? + JSON.stringify(input) : + String(input); +}; + +/** + * Gets a formatted field. Specifically, retrieves a field from a log event using a placeholder's + * `fieldNameKeys`. The field is then formatted using the placeholder's `fieldFormatter`. + * + * @param logEvent + * @param fieldPlaceholder + * @return The formatted field as a string. + */ +const getFormattedField = ( + logEvent: LogEvent, + fieldPlaceholder: YscopeFieldPlaceholder +): string => { + const nestedValue = getNestedJsonValue(logEvent.fields, fieldPlaceholder.fieldNameKeys); + if ("undefined" === typeof nestedValue) { + return ""; + } + + const formattedField = fieldPlaceholder.fieldFormatter ? + fieldPlaceholder.fieldFormatter.formatField(nestedValue) : + jsonValueToString(nestedValue); + + return formattedField; +}; + +/** + * Validates a component string. + * + * @param component + * @return The component string if valid, or `null` if the component is undefined or empty. + */ +const validateComponent = (component: string | undefined): Nullable => { + if ("undefined" === typeof component || 0 === component.length) { + return null; + } + + return component; +}; + +/** + * Splits a field placeholder string into its components: field name, formatter name, and formatter + * options. + * + * @param placeholderString + * @return - An object containing: + * - fieldNameKeys: An array of strings representing the field name split by periods. + * - formatterName: The formatter name, or `null` if not provided. + * - formatterOptions: The formatter options, or `null` if not provided. + * @throws {Error} If the field name could not be parsed. + */ +const splitFieldPlaceholder = (placeholderString: string): { + fieldNameKeys: string[], + formatterName: Nullable, + formatterOptions: Nullable, +} => { + let [ + fieldName, + formatterName, + formatterOptions, + ]: Nullable[ +] = placeholderString.split(COLON_REGEX, 3); + + fieldName = validateComponent(fieldName); + if (null === fieldName) { + throw Error("Field name could not be parsed"); + } + + // Splits field name into an array of field name keys to support nested fields. + let fieldNameKeys = fieldName.split(PERIOD_REGEX); + + fieldNameKeys = fieldNameKeys.map((key) => removeEscapeCharacters(key)); + + formatterName = validateComponent(formatterName); + if (null !== formatterName) { + formatterName = removeEscapeCharacters(formatterName); + } + + formatterOptions = validateComponent(formatterOptions); + if (null !== formatterOptions) { + formatterOptions = removeEscapeCharacters(formatterOptions); + } + + return {fieldNameKeys, formatterName, formatterOptions}; +}; + + +export { + getFormattedField, + jsonValueToString, + removeEscapeCharacters, + replaceDoubleBacklash, + splitFieldPlaceholder, + YSCOPE_FIELD_FORMATTER_MAP, +}; diff --git a/src/typings/formatters.ts b/src/typings/formatters.ts index 87a5d067..bd1dc0ce 100644 --- a/src/typings/formatters.ts +++ b/src/typings/formatters.ts @@ -1,39 +1,116 @@ +import {Nullable} from "../typings/common"; +import {JsonValue} from "./js"; import {LogEvent} from "./logs"; /** - * Options for the LogbackFormatter. + * @property formatString A Yscope format string. The format string can include field-placeholders + * to insert and format any field of a JSON log event. A field-placeholder uses the following + * syntax: + * `{[:[:]]}` + * - (required) defines the key of the field whose value should replace the + * placeholder. + * - Nested fields can be specified using periods (`.`) to denote hierarchy. E.g., the field + * `{"a:" {"b": 0}}` may be denoted by `a.b`. + * - To denote a field-name with periods escape the periods with a backslashes. + * - (optional) is the name of the formatter to apply to the value before + * inserting it into the string. + * - (optional) defines any options for the formatter denoted by formatter-name. * - * @property formatString A Logback-like format string. The format string may include specifiers - * indicating how and what kv-pair may be inserted to replace the specifier in the string. A - * specifier uses the following syntax: `%{}` - * - is the specifier's name. - * - (along with the braces) are optional and indicate how to format the kv-pair - * when inserting it. Options cannot include literal '%' or '}' characters. - * - Literal '%' characters cannot be escaped. - * - * The following specifiers are currently supported: - * - `d` - Indicates the log event's authoritative timestamp. Its options may include a - * java.time.format.DateTimeFormatter pattern indicating how to format the timestamp. NOTE: Not - * all patterns are supported; see `convertDateTimeFormatterPatternToDayJs` to determine what's - * supported. - * - `n` - Indicates a newline character. This is currently ignored by the formatter since it always - * inserts a newline as the last character. - * - `` - Any specifier besides those above indicate a key for a kv-pair, if said kv-pair - * exists in a given log event. - */ -interface LogbackFormatterOptionsType { + * All three as may contain any character, except that colons (:), right braces (}), + * and backslashes (\) must be escaped with a backslash. + */ +interface YscopeFormatterOptionsType { formatString: string, } -type FormatterOptionsType = LogbackFormatterOptionsType; +type FormatterOptionsType = YscopeFormatterOptionsType; interface Formatter { + + /** + * Formats the given log event. + * + * @param logEvent + * @return The formatted log event. + */ formatLogEvent: (logEvent: LogEvent) => string } +interface YscopeFieldFormatter { + + /** + * Formats the given field. + * + * @param logEvent + * @return The formatted field. + */ + formatField: (field: JsonValue) => string +} + +/** + * Type for list of currently supported Yscope field formatters. + */ +type YscopeFieldFormatterMap = { + [key: string]: new (options: Nullable) => YscopeFieldFormatter; +}; + +/** + * Parsed field placeholder from a Yscope format string. + */ +type YscopeFieldPlaceholder = { + fieldNameKeys: string[], + fieldFormatter: Nullable, + + // Location of field placeholder in format string including braces. + range: {start: number, end: number} +} + +/** + * Unicode replacement character `U+FFFD` to substitute escaped backslash (`\\`) in format string. + */ +const REPLACEMENT_CHARACTER = "�"; + +/** + * Used to remove single backlash in format string. + */ +const SINGLE_BACKSLASH = "\\"; + +/** + * Used to replace double backlash in format string. + */ +const DOUBLE_BACKSLASH = "\\\\"; + +// Patterns to assist parsing YScope format string. + +/** + * Pattern to split field unescaped placeholder. + */ +const COLON_REGEX = Object.freeze(/(? { + let result: JsonValue | undefined = fields; + for (const key of keys) { + if ("object" !== typeof result || null === result || Array.isArray(result)) { + // `undefined` seems natural as return value for this function since matches + // js behaviour. + // eslint-disable-next-line no-undefined + return undefined; + } + result = result[key]; + } + + return result; +}; + +export {getNestedJsonValue};