diff --git a/Cargo.lock b/Cargo.lock index 76a7077a..835af051 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -424,7 +424,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools 0.10.5", + "itertools", "num-traits", "once_cell", "oorandom", @@ -445,7 +445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.5", + "itertools", ] [[package]] @@ -719,7 +719,7 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hl" -version = "0.21.0" +version = "0.22.0" dependencies = [ "atoi", "bincode", @@ -746,7 +746,7 @@ dependencies = [ "hex", "htp", "humantime", - "itertools 0.12.0", + "itertools", "itoa", "kqueue", "notify", @@ -859,15 +859,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index 0ab5fa76..447b7c5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ categories = ["command-line-utilities"] description = "Utility for viewing json-formatted log files." keywords = ["cli", "human", "log"] name = "hl" -version = "0.21.0" +version = "0.22.0" edition = "2021" build = "build.rs" diff --git a/etc/defaults/config-ecs.yaml b/etc/defaults/config-ecs.yaml new file mode 100644 index 00000000..192f4908 --- /dev/null +++ b/etc/defaults/config-ecs.yaml @@ -0,0 +1,61 @@ +# Time format, see https://man7.org/linux/man-pages/man1/date.1.html for details. +time-format: '%b %d %T.%3N' + +# Time zone name, see column "TZ identifier" at +# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones page. +time-zone: UTC + +# Settings for fields processing. +fields: + # Configuration of the predefined set of fields. + predefined: + time: + names: ['@timestamp'] + logger: + names: [log.logger] + level: + variants: + - names: [log.level] + values: + debug: [debug, dbg, d] + info: [info, inf, i, informational] + warning: [warning, warn, wrn, w] + error: [error, err, fatal, critical, panic, e] + message: + names: [message] + caller: + names: [] + caller-file: + names: [log.origin.file.name] + caller-line: + names: [log.origin.file.line] + # List of wildcard field names to ignore. + ignore: ['_*'] + # List of exact field names to hide. + hide: + - log + +# Formatting settings. +formatting: + punctuation: + logger-name-separator: ':' + field-key-value-separator: '=' + string-opening-quote: "'" + string-closing-quote: "'" + source-location-separator: '@ ' + hidden-fields-indicator: ' ...' + level-left-separator: '|' + level-right-separator: '|' + input-number-prefix: '#' + input-number-left-separator: '' + input-number-right-separator: ' | ' + input-name-left-separator: '' + input-name-right-separator: ' | ' + input-name-clipping: '...' + input-name-common-part: '...' + +# Number of processing threads, configured automatically based on CPU count if not specified. +concurrency: ~ + +# Currently selected theme. +theme: universal diff --git a/etc/defaults/config.yaml b/etc/defaults/config.yaml index c458dcee..082f0865 100644 --- a/etc/defaults/config.yaml +++ b/etc/defaults/config.yaml @@ -41,6 +41,10 @@ fields: names: [msg, message, MESSAGE, Message] caller: names: [caller, CALLER, Caller] + caller-file: + names: [] + caller-line: + names: [] # List of wildcard field names to ignore. ignore: ['_*'] # List of exact field names to hide. diff --git a/src/formatting.rs b/src/formatting.rs index 2cbd3b04..a7d4e472 100644 --- a/src/formatting.rs +++ b/src/formatting.rs @@ -15,7 +15,7 @@ use crate::theme; use crate::IncludeExcludeKeyFilter; use datefmt::DateTimeFormatter; use fmtx::{aligned_left, centered}; -use model::Level; +use model::{Caller, Level}; use theme::{Element, StylingPush, Theme}; // --- @@ -176,14 +176,25 @@ impl RecordFormatter { // // caller // - if let Some(text) = rec.caller { + if let Some(caller) = &rec.caller { s.element(Element::Caller, |s| { s.batch(|buf| { buf.push(b' '); buf.extend_from_slice(self.cfg.punctuation.source_location_separator.as_bytes()) }); s.element(Element::CallerInner, |s| { - s.batch(|buf| buf.extend_from_slice(text.as_bytes())) + s.batch(|buf| { + match caller { + Caller::Text(text) => { + buf.extend_from_slice(text.as_bytes()); + } + Caller::FileLine(file, line) => { + buf.extend_from_slice(file.as_bytes()); + buf.push(b':'); + buf.extend_from_slice(line.as_bytes()); + } + }; + }); }); }); }; @@ -491,7 +502,7 @@ mod tests { message: Some(RawValue::from_string(r#""tm""#.into()).unwrap().as_ref()), level: Some(Level::Debug), logger: Some("tl"), - caller: Some("tc"), + caller: Some(Caller::Text("tc")), extra: heapless::Vec::from_slice(&[ ("ka", RawValue::from_string(r#"{"va":{"kb":42}}"#.into()).unwrap().as_ref()), ]).unwrap(), diff --git a/src/model.rs b/src/model.rs index 0d3ea495..f389d501 100644 --- a/src/model.rs +++ b/src/model.rs @@ -30,7 +30,7 @@ pub struct Record<'a> { pub message: Option<&'a RawValue>, pub level: Option, pub logger: Option<&'a str>, - pub caller: Option<&'a str>, + pub caller: Option>, pub(crate) extra: heapless::Vec<(&'a str, &'a RawValue), RECORD_EXTRA_CAPACITY>, pub(crate) extrax: Vec<(&'a str, &'a RawValue)>, } @@ -69,6 +69,13 @@ pub trait RecordWithSourceConstructor { // --- +pub enum Caller<'a> { + Text(&'a str), + FileLine(&'a str, &'a str), +} + +// --- + pub struct RecordWithSource<'a> { pub record: &'a Record<'a>, pub source: &'a [u8], @@ -94,9 +101,11 @@ pub trait RecordFilter { // --- -#[derive(Default)] +#[derive(Default, Debug)] pub struct ParserSettings { - fields: HashMap, + pre_parse_time: bool, + level: Vec>, + blocks: Vec, ignore: Vec, } @@ -104,145 +113,296 @@ impl ParserSettings { pub fn new<'a, I: IntoIterator>( predefined: &PredefinedFields, ignore: I, - preparse_time: bool, + pre_parse_time: bool, ) -> Self { - let mut fields = HashMap::new(); - for (i, name) in predefined.time.names.iter().enumerate() { - fields.insert(name.clone(), (FieldSettings::Time(preparse_time), i)); - } + let mut result = Self { + pre_parse_time, + level: Vec::new(), + blocks: vec![ParserSettingsBlock::default()], + ignore: ignore.into_iter().map(|x| WildMatch::new(x)).collect(), + }; + + result.init(predefined); + result + } + + fn init(&mut self, pf: &PredefinedFields) { + self.build_block(0, &pf.time.names, FieldSettings::Time, 0); + self.build_block(0, &pf.message.names, FieldSettings::Message, 0); + self.build_block(0, &pf.logger.names, FieldSettings::Logger, 0); + self.build_block(0, &pf.caller.names, FieldSettings::Caller, 0); + self.build_block(0, &pf.caller_file.names, FieldSettings::CallerFile, 0); + self.build_block(0, &pf.caller_line.names, FieldSettings::CallerLine, 0); + let mut j = 0; - for variant in &predefined.level.variants { + for variant in &pf.level.variants { let mut mapping = HashMap::new(); for (level, values) in &variant.values { for value in values { mapping.insert(value.clone(), level.clone()); } } - for (i, name) in variant.names.iter().enumerate() { - fields.insert(name.clone(), (FieldSettings::Level(mapping.clone()), j + i)); - } + let k = self.level.len(); + self.level.push(mapping.clone()); + self.build_block(0, &variant.names, FieldSettings::Level(k), j); j += variant.names.len(); } - for (i, name) in predefined.message.names.iter().enumerate() { - fields.insert(name.clone(), (FieldSettings::Message, i)); - } - for (i, name) in predefined.logger.names.iter().enumerate() { - fields.insert(name.clone(), (FieldSettings::Logger, i)); + } + + fn build_block<'a, N: IntoIterator>( + &mut self, + n: usize, + names: N, + settings: FieldSettings, + priority: usize, + ) { + for (i, name) in names.into_iter().enumerate() { + self.build_block_for_name(n, name, settings, priority + i) } - for (i, name) in predefined.caller.names.iter().enumerate() { - fields.insert(name.clone(), (FieldSettings::Caller, i)); + } + + fn build_block_for_name(&mut self, n: usize, name: &String, settings: FieldSettings, priority: usize) { + self.blocks[n].fields.insert(name.clone(), (settings, priority)); + let mut remainder = &name[..]; + while let Some(k) = remainder.rfind('.') { + let (name, nested) = name.split_at(k); + let nested = &nested[1..]; + + let nest = self.blocks[n] + .fields + .get(name) + .and_then(|f| { + if let FieldSettings::Nested(nest) = f.0 { + Some(nest) + } else { + None + } + }) + .unwrap_or_else(|| { + let nest = self.blocks.len(); + self.blocks.push(ParserSettingsBlock::default()); + self.blocks[n] + .fields + .insert(name.to_string(), (FieldSettings::Nested(nest), priority)); + nest + }); + + self.build_block_for_name(nest, &nested.into(), settings, priority); + remainder = name; } - Self { - fields, - ignore: ignore.into_iter().map(|v| WildMatch::new(v)).collect(), + } + + fn apply<'a>(&self, key: &'a str, value: &'a RawValue, to: &mut Record<'a>, pc: &mut PriorityController) { + self.blocks[0].apply(self, key, value, to, pc, true); + } + + fn apply_each<'a, 'i, I>(&self, items: I, to: &mut Record<'a>) + where + I: IntoIterator, + 'a: 'i, + { + let mut pc = PriorityController::default(); + self.apply_each_ctx(items, to, &mut pc); + } + + fn apply_each_ctx<'a, 'i, I>(&self, items: I, to: &mut Record<'a>, pc: &mut PriorityController) + where + I: IntoIterator, + 'a: 'i, + { + for (key, value) in items { + self.apply(key, value, to, pc) } } +} + +// --- - fn apply<'a>(&self, key: &'a str, value: &'a RawValue, to: &mut Record<'a>, ctx: &mut PriorityContext) { - match self.fields.get(key) { - Some((field, p)) => { +#[derive(Default, Debug)] +struct ParserSettingsBlock { + fields: HashMap, +} + +impl ParserSettingsBlock { + fn apply<'a>( + &self, + ps: &ParserSettings, + key: &'a str, + value: &'a RawValue, + to: &mut Record<'a>, + pc: &mut PriorityController, + is_root: bool, + ) { + let done = match self.fields.get(key) { + Some((field, priority)) => { let kind = field.kind(); - let priority = ctx.priority(kind); - if priority.is_none() || Some(*p) <= *priority { - field.apply(value, to); - *priority = Some(*p); - } - } - None => { - for pattern in &self.ignore { - if pattern.matches(key) { - return; - } - } - match to.extra.push((key, value)) { - Ok(_) => {} - Err(value) => to.extrax.push(value), + if let Some(kind) = kind { + pc.prioritize(kind, *priority, |pc| field.apply_ctx(ps, value, to, pc)) + } else { + field.apply_ctx(ps, value, to, pc); + false } } + None => false, }; + if done || !is_root { + return; + } + + for pattern in &ps.ignore { + if pattern.matches(key) { + return; + } + } + match to.extra.push((key, value)) { + Ok(_) => {} + Err(value) => to.extrax.push(value), + } } - fn apply_each<'a, 'i, I>(&self, items: I, to: &mut Record<'a>) - where + fn apply_each_ctx<'a, 'i, I>( + &self, + ps: &ParserSettings, + items: I, + to: &mut Record<'a>, + ctx: &mut PriorityController, + is_root: bool, + ) where I: IntoIterator, 'a: 'i, { - let mut ctx = PriorityContext { - time: None, - level: None, - logger: None, - message: None, - caller: None, - }; for (key, value) in items { - self.apply(key, value, to, &mut ctx) + self.apply(ps, key, value, to, ctx, is_root) } } } // --- -struct PriorityContext { +#[derive(Default)] +struct PriorityController { time: Option, level: Option, logger: Option, message: Option, caller: Option, + caller_file: Option, + caller_line: Option, } -impl PriorityContext { - fn priority(&mut self, kind: FieldKind) -> &mut Option { - match kind { +impl PriorityController { + fn prioritize ()>(&mut self, kind: FieldKind, priority: usize, update: F) -> bool { + let p = match kind { FieldKind::Time => &mut self.time, FieldKind::Level => &mut self.level, FieldKind::Logger => &mut self.logger, FieldKind::Message => &mut self.message, FieldKind::Caller => &mut self.caller, + FieldKind::CallerFile => &mut self.caller_file, + FieldKind::CallerLine => &mut self.caller_line, + }; + + if p.is_none() || Some(priority) <= *p { + *p = Some(priority); + update(self); + true + } else { + false } } } // --- +#[derive(Clone, Copy, Debug)] enum FieldSettings { - Time(bool), - Level(HashMap), + Time, + Level(usize), Logger, Message, Caller, + CallerFile, + CallerLine, + Nested(usize), } impl FieldSettings { - fn apply<'a>(&self, value: &'a RawValue, to: &mut Record<'a>) { - match self { - Self::Time(preparse) => { + fn apply<'a>(&self, ps: &ParserSettings, value: &'a RawValue, to: &mut Record<'a>) { + match *self { + Self::Time => { let s = value.get(); let s = if s.as_bytes()[0] == b'"' { &s[1..s.len() - 1] } else { s }; let ts = Timestamp::new(s, None); - if *preparse { + if ps.pre_parse_time { to.ts = Some(Timestamp::new(ts.raw(), Some(ts.parse()))); } else { to.ts = Some(ts); } } - Self::Level(values) => { + Self::Level(i) => { to.level = json::from_str(value.get()) .ok() - .and_then(|x: &'a str| values.get(x).cloned()); + .and_then(|x: &'a str| ps.level[i].get(x).cloned()); } Self::Logger => to.logger = json::from_str(value.get()).ok(), Self::Message => to.message = Some(value), - Self::Caller => to.caller = json::from_str(value.get()).ok(), + Self::Caller => to.caller = json::from_str(value.get()).ok().map(|x| Caller::Text(x)), + Self::CallerFile => match &mut to.caller { + None => { + to.caller = json::from_str(value.get()).ok().map(|x| Caller::FileLine(x, "")); + } + Some(Caller::FileLine(file, _)) => { + if let Some(value) = json::from_str(value.get()).ok() { + *file = value + } + } + _ => {} + }, + Self::CallerLine => match &mut to.caller { + None => { + to.caller = Some(Caller::FileLine("", value.get())); + } + Some(Caller::FileLine(_, line)) => { + if let Some(value) = json::from_str(value.get()).ok() { + *line = value + } + } + _ => {} + }, + Self::Nested(_) => {} } } - fn kind(&self) -> FieldKind { + fn apply_ctx<'a>( + &self, + ps: &ParserSettings, + value: &'a RawValue, + to: &mut Record<'a>, + ctx: &mut PriorityController, + ) { + match *self { + Self::Nested(nested) => { + let s = value.get(); + if s.len() > 0 && s.as_bytes()[0] == b'{' { + if let Ok(record) = json::from_str::(s) { + ps.blocks[nested].apply_each_ctx(ps, record.fields(), to, ctx, false); + } + } + } + _ => self.apply(ps, value, to), + } + } + + fn kind(&self) -> Option { match self { - Self::Time(_) => FieldKind::Time, - Self::Level(_) => FieldKind::Level, - Self::Logger => FieldKind::Logger, - Self::Message => FieldKind::Message, - Self::Caller => FieldKind::Caller, + Self::Time => Some(FieldKind::Time), + Self::Level(_) => Some(FieldKind::Level), + Self::Logger => Some(FieldKind::Logger), + Self::Message => Some(FieldKind::Message), + Self::Caller => Some(FieldKind::Caller), + Self::CallerFile => Some(FieldKind::CallerFile), + Self::CallerLine => Some(FieldKind::CallerLine), + Self::Nested(_) => None, } } } @@ -544,7 +704,11 @@ impl RecordFilter for FieldFilter { } } "caller" => { - if !self.match_value(record.caller, false) { + if let Some(Caller::Text(caller)) = record.caller { + if !self.match_value(Some(caller), false) { + return false; + } + } else { return false; } } diff --git a/src/settings.rs b/src/settings.rs index b01444fb..aa74fe40 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -66,12 +66,15 @@ pub struct Fields { // --- #[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub struct PredefinedFields { pub time: TimeField, pub level: LevelField, pub message: MessageField, pub logger: LoggerField, pub caller: CallerField, + pub caller_file: CallerFileField, + pub caller_line: CallerLineField, } // --- @@ -112,6 +115,16 @@ pub struct CallerField(Field); // --- +#[derive(Debug, Serialize, Deserialize, Deref)] +pub struct CallerFileField(Field); + +// --- + +#[derive(Debug, Serialize, Deserialize, Deref)] +pub struct CallerLineField(Field); + +// --- + #[derive(Debug, Serialize, Deserialize)] pub struct Field { pub names: Vec, diff --git a/src/types.rs b/src/types.rs index f1cd04e1..632e05e6 100644 --- a/src/types.rs +++ b/src/types.rs @@ -13,4 +13,6 @@ pub enum FieldKind { Logger, Message, Caller, + CallerFile, + CallerLine, }