diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0bbb86c..dc482964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: cargo build --verbose --benches - name: Run tests diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3e7ccca8..9aa7594f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -10,7 +10,7 @@ jobs: options: --security-opt seccomp=unconfined steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Generate code coverage run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 43b9bd77..5996d0dd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -87,7 +87,7 @@ jobs: asset: hl-windows.zip steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 with: diff --git a/Cargo.lock b/Cargo.lock index 7603a9ae..4588cd06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -600,7 +600,7 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encstr" -version = "0.28.2-alpha.2" +version = "0.29.0-alpha.1" [[package]] name = "enum-map" @@ -757,7 +757,7 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hl" -version = "0.28.2-alpha.2" +version = "0.29.0-alpha.1" dependencies = [ "atoi", "bincode", diff --git a/Cargo.toml b/Cargo.toml index b5406168..b48ee1d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = [".", "crate/encstr"] [workspace.package] repository = "https://github.com/pamburus/hl" authors = ["Pavel Ivanov "] -version = "0.28.2-alpha.2" +version = "0.29.0-alpha.1" edition = "2021" license = "MIT" diff --git a/README.md b/README.md index 7bc9edbc..59a44b22 100644 --- a/README.md +++ b/README.md @@ -488,6 +488,7 @@ Output Options: --no-raw Disable raw source messages output, overrides --raw option --raw-fields Output field values as is, without unescaping or prettifying -h, --hide Hide or reveal fields with the specified keys, prefix with ! to reveal, specify '!*' to reveal all + --flatten Whether to flatten objects [env: HL_FLATTEN=] [default: always] [possible values: never, always] -t, --time-format Time format, see https://man7.org/linux/man-pages/man1/date.1.html [env: HL_TIME_FORMAT=] [default: "%b %d %T.%3N"] -Z, --time-zone Time zone name, see column "TZ identifier" at https://en.wikipedia.org/wiki/List_of_tz_database_time_zones [env: HL_TIME_ZONE=] [default: UTC] -L, --local Use local time zone, overrides --time-zone option diff --git a/etc/defaults/config.yaml b/etc/defaults/config.yaml index 141e98b1..81936e4f 100644 --- a/etc/defaults/config.yaml +++ b/etc/defaults/config.yaml @@ -52,6 +52,7 @@ fields: # Formatting settings. formatting: + flatten: always punctuation: logger-name-separator: ':' field-key-value-separator: '=' diff --git a/src/app.rs b/src/app.rs index 2e1c821c..66e9b2a6 100644 --- a/src/app.rs +++ b/src/app.rs @@ -74,6 +74,7 @@ pub struct Options { pub tail: u64, pub delimiter: Delimiter, pub unix_ts_unit: Option, + pub flatten: bool, } impl Options { @@ -85,8 +86,24 @@ impl Options { (false, Some(query)) => Box::new((&self.filter).and(query)), } } + + #[cfg(test)] + fn with_theme(self, theme: Arc) -> Self { + Self { theme, ..self } + } + + #[cfg(test)] + fn with_fields(self, fields: FieldOptions) -> Self { + Self { fields, ..self } + } + + #[cfg(test)] + fn with_raw_fields(self, raw_fields: bool) -> Self { + Self { raw_fields, ..self } + } } +#[derive(Default)] pub struct FieldOptions { pub filter: Arc, pub settings: Fields, @@ -639,7 +656,8 @@ impl App { self.options.fields.filter.clone(), self.options.formatting.clone(), ) - .with_field_unescaping(!self.options.raw_fields), + .with_field_unescaping(!self.options.raw_fields) + .with_flatten(self.options.flatten), ) } } @@ -997,3 +1015,160 @@ where i += 1; } } + +// --- + +#[cfg(test)] +mod tests { + use super::*; + + use std::io::Cursor; + + use chrono_tz::UTC; + + use crate::{filtering::MatchOptions, themecfg::testing, LinuxDateFormat}; + + #[test] + fn test_common_prefix_len() { + let items = vec!["abc", "abcd", "ab", "ab"]; + assert_eq!(common_prefix_len(&items), 2); + } + + #[test] + fn test_cat_empty() { + let input = input(""); + let mut output = Vec::new(); + let app = App::new(options()); + app.run(vec![input], &mut output).unwrap(); + assert_eq!(std::str::from_utf8(&output).unwrap(), ""); + } + + #[test] + fn test_cat_one_line() { + let input = input( + r#"{"caller":"main.go:539","duration":"15d","level":"info","msg":"No time or size retention was set so using the default time retention","ts":"2023-12-07T20:07:05.949Z"}"#, + ); + let mut output = Vec::new(); + let app = App::new(options()); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "2023-12-07 20:07:05.949 |INF| No time or size retention was set so using the default time retention duration=15d @ main.go:539\n", + ); + } + + #[test] + fn test_cat_with_theme() { + let input = input( + r#"{"caller":"main.go:539","duration":"15d","level":"info","msg":"No time or size retention was set so using the default time retention","ts":"2023-12-07T20:07:05.949Z"}"#, + ); + let mut output = Vec::new(); + let app = App::new(options().with_theme(theme())); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF| \u{1b}[0;1;39mNo time or size retention was set so using the default time retention \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n", + ); + } + + #[test] + fn test_cat_no_msg() { + let input = + input(r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z"}"#); + let mut output = Vec::new(); + let app = App::new(options().with_theme(theme())); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF|\u{1b}[0m \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n", + ); + } + + #[test] + fn test_cat_msg_array() { + let input = input( + r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":["x","y"]}"#, + ); + let mut output = Vec::new(); + let app = App::new(options().with_theme(theme())); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF| \u{1b}[0;32mmsg\u{1b}[0;2m=\u{1b}[0;93m[\u{1b}[0;39mx\u{1b}[0;93m \u{1b}[0;39my\u{1b}[0;93m] \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n", + ); + } + + #[test] + fn test_cat_field_exclude() { + let input = input( + r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":"xy"}"#, + ); + let mut output = Vec::new(); + let mut ff = IncludeExcludeKeyFilter::new(MatchOptions::default()); + ff.entry("duration").exclude(); + let app = App::new(options().with_fields(FieldOptions { + filter: Arc::new(ff), + ..FieldOptions::default() + })); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "2023-12-07 20:07:05.949 |INF| xy ... @ main.go:539\n", + ); + } + + #[test] + fn test_cat_raw_fields() { + let input = input( + r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":"xy"}"#, + ); + let mut output = Vec::new(); + let mut ff = IncludeExcludeKeyFilter::new(MatchOptions::default()); + ff.entry("duration").exclude(); + let app = App::new(options().with_raw_fields(true)); + app.run(vec![input], &mut output).unwrap(); + assert_eq!( + std::str::from_utf8(&output).unwrap(), + "2023-12-07 20:07:05.949 |INF| xy duration=\"15d\" @ main.go:539\n", + ); + } + + fn input>(s: S) -> InputHolder { + InputHolder::new(InputReference::File("-".into()), Some(Box::new(Cursor::new(s.into())))) + } + + fn options() -> Options { + Options { + theme: Arc::new(Theme::none()), + time_format: LinuxDateFormat::new("%Y-%m-%d %T.%3N").compile(), + raw: false, + raw_fields: false, + allow_prefix: false, + buffer_size: NonZeroUsize::new(4096).unwrap(), + max_message_size: NonZeroUsize::new(4096 * 1024).unwrap(), + concurrency: 1, + filter: Filter::default(), + query: None, + fields: FieldOptions::default(), + formatting: Formatting::default(), + time_zone: Tz::IANA(UTC), + hide_empty_fields: false, + sort: false, + follow: false, + sync_interval: Duration::from_secs(1), + input_info: None, + input_format: None, + dump_index: false, + debug: false, + app_dirs: None, + tail: 0, + delimiter: Delimiter::default(), + unix_ts_unit: None, + flatten: false, + } + } + + fn theme() -> Arc { + Arc::new(Theme::from(testing::theme().unwrap())) + } +} diff --git a/src/cli.rs b/src/cli.rs index 6fbd4a5d..45bfb7ee 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -11,6 +11,7 @@ use crate::{ config, error::*, level::{LevelValueParser, RelaxedLevel}, + settings, }; // --- @@ -153,6 +154,21 @@ pub struct Opt { )] pub hide: Vec, + /// Whether to flatten objects. + #[arg( + long, + env = "HL_FLATTEN", + value_name = "WHEN", + value_enum, + default_value_t = config::get().formatting.flatten.as_ref().map(|x| match x{ + settings::FlattenOption::Never => FlattenOption::Never, + settings::FlattenOption::Always => FlattenOption::Always, + }).unwrap_or(FlattenOption::Always), + overrides_with = "flatten", + help_heading = heading::OUTPUT + )] + pub flatten: FlattenOption, + /// Time format, see https://man7.org/linux/man-pages/man1/date.1.html. #[arg( short, @@ -360,6 +376,12 @@ pub enum UnixTimestampUnit { Ns, } +#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlattenOption { + Never, + Always, +} + mod heading { pub const FILTERING: &str = "Filtering Options"; pub const INPUT: &str = "Input Options"; diff --git a/src/fmtx.rs b/src/fmtx.rs index 305e8dd0..c168e0d3 100644 --- a/src/fmtx.rs +++ b/src/fmtx.rs @@ -11,10 +11,12 @@ impl Push for Vec where T: Clone, { + #[inline] fn push(&mut self, value: T) { Vec::push(self, value) } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { Vec::extend_from_slice(self, values) } @@ -22,25 +24,106 @@ where // --- +#[derive(Default)] +pub struct OptimizedBuf { + pub head: heapless::Vec, + pub tail: Vec, +} + +impl OptimizedBuf +where + T: Clone, +{ + #[inline] + pub fn new() -> Self { + Self { + head: heapless::Vec::new(), + tail: Vec::new(), + } + } + + #[inline] + pub fn len(&self) -> usize { + self.head.len() + self.tail.len() + } + + #[inline] + pub fn clear(&mut self) { + self.head.clear(); + self.tail.clear(); + } + + #[inline] + pub fn truncate(&mut self, len: usize) { + if len <= self.head.len() { + self.head.truncate(len); + self.tail.clear(); + } else { + self.tail.truncate(len - self.head.len()); + } + } + + #[inline] + pub fn push(&mut self, value: T) { + if self.head.len() < N { + self.head.push(value).ok(); + } else { + self.tail.push(value); + } + } + + #[inline] + pub fn extend_from_slice(&mut self, values: &[T]) { + if self.head.len() + values.len() <= N { + self.head.extend_from_slice(values).ok(); + } else { + let n = N - self.head.len(); + self.head.extend_from_slice(&values[..n]).ok(); + self.tail.extend_from_slice(&values[n..]); + } + } +} + +impl Push for OptimizedBuf +where + T: Clone, +{ + #[inline] + fn push(&mut self, value: T) { + OptimizedBuf::push(self, value) + } + + #[inline] + fn extend_from_slice(&mut self, values: &[T]) { + OptimizedBuf::extend_from_slice(self, values) + } +} + +// --- + pub struct Counter { value: usize, } impl Counter { + #[inline] pub fn new() -> Self { Self { value: 0 } } + #[inline] pub fn result(&self) -> usize { self.value } } impl Push for Counter { + #[inline] fn push(&mut self, _: T) { self.value += 1 } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { self.value += values.len() } @@ -65,6 +148,7 @@ pub struct Padding { } impl Padding { + #[inline] pub fn new(pad: T, width: usize) -> Self { Self { pad, width } } @@ -79,6 +163,7 @@ pub struct Adjustment { } impl Adjustment { + #[inline] pub fn new(alignment: Alignment, padding: Padding) -> Self { Self { alignment, padding } } @@ -101,6 +186,7 @@ where T: Clone, O: Push, { + #[inline] fn new(out: &'a mut O, adjustment: Option>) -> Self { if let Some(adjustment) = adjustment { match adjustment.alignment { @@ -114,6 +200,7 @@ where } } + #[inline] fn push(&mut self, value: T) { match self { Self::Disabled(ref mut aligner) => aligner.push(value), @@ -122,6 +209,7 @@ where } } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { match self { Self::Disabled(ref mut aligner) => aligner.extend_from_slice(values), @@ -136,10 +224,12 @@ where T: Clone, B: Push, { + #[inline] fn push(&mut self, value: T) { Aligner::push(self, value) } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { Aligner::extend_from_slice(self, values) } @@ -152,6 +242,7 @@ pub struct DisabledAligner<'a, O> { } impl<'a, O> DisabledAligner<'a, O> { + #[inline] pub fn new(out: &'a mut O) -> Self { Self { out } } @@ -162,10 +253,12 @@ where T: Clone, O: Push, { + #[inline] fn push(&mut self, value: T) { self.out.push(value) } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { self.out.extend_from_slice(values) } @@ -188,10 +281,12 @@ where T: Clone, O: Push, { + #[inline] pub fn new(out: &'a mut O, padding: Padding) -> Self { Self { out, padding, cur: 0 } } + #[inline] pub fn push(&mut self, value: T) { if self.cur < self.padding.width { self.out.push(value); @@ -199,6 +294,7 @@ where } } + #[inline] pub fn extend_from_slice(&mut self, values: &[T]) { if self.cur < self.padding.width { let n = min(self.padding.width - self.cur, values.len()); @@ -213,10 +309,12 @@ where T: Clone, O: Push, { + #[inline] fn push(&mut self, value: T) { UnbufferedAligner::push(self, value) } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { UnbufferedAligner::extend_from_slice(self, values) } @@ -227,6 +325,7 @@ where T: Clone, O: Push, { + #[inline] fn drop(&mut self) { for _ in self.cur..self.padding.width { self.out.push(self.padding.pad.clone()); @@ -252,6 +351,7 @@ where T: Clone, O: Push, { + #[inline] fn new(out: &'a mut O, padding: Padding, alignment: Alignment) -> Self { Self { out, @@ -265,6 +365,7 @@ where } } + #[inline] pub fn push(&mut self, value: T) { match self.buf { AlignerBuffer::Static(ref mut buf) => { @@ -280,6 +381,7 @@ where } } + #[inline] pub fn extend_from_slice(&mut self, values: &[T]) { match self.buf { AlignerBuffer::Static(ref mut buf) => { @@ -299,10 +401,12 @@ where T: Clone, O: Push, { + #[inline] fn push(&mut self, value: T) { BufferedAligner::push(self, value) } + #[inline] fn extend_from_slice(&mut self, values: &[T]) { BufferedAligner::extend_from_slice(self, values) } @@ -313,6 +417,7 @@ where T: Clone, O: Push, { + #[inline] fn drop(&mut self) { let buf = match &self.buf { AlignerBuffer::Static(buf) => &buf[..], @@ -342,6 +447,7 @@ enum AlignerBuffer { // --- +#[inline] pub fn aligned<'a, T, O, F>(out: &'a mut O, adjustment: Option>, f: F) where T: Clone, @@ -351,6 +457,7 @@ where f(Aligner::new(out, adjustment)); } +#[inline] pub fn aligned_left<'a, T, O, F>(out: &'a mut O, width: usize, pad: T, f: F) where T: Clone, @@ -360,6 +467,7 @@ where f(UnbufferedAligner::new(out, Padding::new(pad, width))); } +#[inline] pub fn centered<'a, T, O, F>(out: &'a mut O, width: usize, pad: T, f: F) where T: Clone, @@ -368,3 +476,79 @@ where { f(BufferedAligner::new(out, Padding::new(pad, width), Alignment::Center)); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_optimized_buf_push() { + let mut buf = OptimizedBuf::::new(); + assert_eq!(buf.len(), 0); + buf.push(1); + assert_eq!(buf.len(), 1); + buf.push(2); + assert_eq!(buf.len(), 2); + buf.push(3); + assert_eq!(buf.len(), 3); + buf.push(4); + assert_eq!(buf.len(), 4); + buf.push(5); + assert_eq!(buf.len(), 5); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[5]); + } + + #[test] + fn test_optimized_buf_extend() { + let mut buf = OptimizedBuf::::new(); + assert_eq!(buf.len(), 0); + buf.extend_from_slice(&[]); + assert_eq!(buf.len(), 0); + buf.extend_from_slice(&[1]); + assert_eq!(buf.len(), 1); + buf.extend_from_slice(&[2, 3]); + assert_eq!(buf.len(), 3); + buf.extend_from_slice(&[4, 5, 6]); + assert_eq!(buf.len(), 6); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[5, 6]); + } + + #[test] + fn test_optimized_buf_truncate() { + let mut buf = OptimizedBuf::::new(); + assert_eq!(buf.len(), 0); + buf.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7]); + assert_eq!(buf.len(), 7); + buf.truncate(8); + assert_eq!(buf.len(), 7); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[5, 6, 7]); + buf.truncate(7); + assert_eq!(buf.len(), 7); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[5, 6, 7]); + buf.truncate(6); + assert_eq!(buf.len(), 6); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[5, 6]); + buf.truncate(4); + assert_eq!(buf.len(), 4); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.len(), 0); + buf.truncate(4); + buf.extend_from_slice(&[8, 9]); + assert_eq!(buf.len(), 6); + assert_eq!(buf.head.as_slice(), &[1, 2, 3, 4]); + assert_eq!(buf.tail.as_slice(), &[8, 9]); + buf.truncate(3); + assert_eq!(buf.len(), 3); + assert_eq!(buf.head.as_slice(), &[1, 2, 3]); + assert_eq!(buf.tail.len(), 0); + buf.truncate(0); + assert_eq!(buf.len(), 0); + assert_eq!(buf.head.len(), 0); + assert_eq!(buf.tail.len(), 0); + } +} diff --git a/src/formatting.rs b/src/formatting.rs index 2a5ab049..204bf9a7 100644 --- a/src/formatting.rs +++ b/src/formatting.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::{ datefmt::DateTimeFormatter, filtering::IncludeExcludeSetting, - fmtx::{aligned_left, centered, Push}, + fmtx::{aligned_left, centered, OptimizedBuf, Push}, model::{self, Caller, Level, RawValue}, settings::Formatting, theme::{Element, StylingPush, Theme}, @@ -57,6 +57,7 @@ pub struct RecordFormatter { ts_formatter: DateTimeFormatter, ts_width: usize, hide_empty_fields: bool, + flatten: bool, fields: Arc, cfg: Formatting, } @@ -76,6 +77,7 @@ impl RecordFormatter { ts_formatter, ts_width, hide_empty_fields, + flatten: false, fields, cfg, } @@ -86,7 +88,14 @@ impl RecordFormatter { self } + pub fn with_flatten(mut self, value: bool) -> Self { + self.flatten = value; + self + } + pub fn format_record(&self, buf: &mut Buf, rec: &model::Record) { + let mut fs = FormattingState::new(self.flatten && self.unescape_fields); + self.theme.apply(buf, &rec.level, |s| { // // time @@ -150,9 +159,8 @@ impl RecordFormatter { // // message text // - if let Some(text) = &rec.message { - s.batch(|buf| buf.push(b' ')); - s.element(Element::Message, |s| self.format_message(s, *text)); + if let Some(value) = &rec.message { + self.format_message(s, &mut fs, *value); } else { s.reset(); } @@ -162,7 +170,7 @@ impl RecordFormatter { let mut some_fields_hidden = false; for (k, v) in rec.fields() { if !self.hide_empty_fields || !v.is_empty() { - some_fields_hidden |= !self.format_field(s, k, *v, Some(&self.fields)); + some_fields_hidden |= !self.format_field(s, k, *v, &mut fs, Some(&self.fields)); } } if some_fields_hidden { @@ -200,104 +208,38 @@ impl RecordFormatter { }); } + #[inline] fn format_field<'a, S: StylingPush>( &self, s: &mut S, key: &str, value: RawValue<'a>, + fs: &mut FormattingState, filter: Option<&IncludeExcludeKeyFilter>, ) -> bool { let mut fv = FieldFormatter::new(self); - fv.format(s, key, value, filter, IncludeExcludeSetting::Unspecified) - } - - fn format_value<'a, S: StylingPush>(&self, s: &mut S, value: RawValue<'a>) { - let mut fv = FieldFormatter::new(self); - fv.format_value(s, value, None, IncludeExcludeSetting::Unspecified); + fv.format(s, key, value, fs, filter, IncludeExcludeSetting::Unspecified) } - fn format_message<'a, S: StylingPush>(&self, s: &mut S, value: RawValue<'a>) { + #[inline] + fn format_message<'a, S: StylingPush>(&self, s: &mut S, fs: &mut FormattingState, value: RawValue<'a>) { match value { RawValue::String(value) => { - s.element(Element::Message, |s| { - s.batch(|buf| buf.with_auto_trim(|buf| MessageFormatAuto::new(value).format(buf).unwrap())) - }); - } - RawValue::Number(value) => { - s.element(Element::Number, |s| s.batch(|buf| buf.extend(value.as_bytes()))); - } - RawValue::Boolean(true) => { - s.element(Element::Boolean, |s| s.batch(|buf| buf.extend(b"true"))); - } - RawValue::Boolean(false) => { - s.element(Element::Boolean, |s| s.batch(|buf| buf.extend(b"false"))); - } - RawValue::Null => { - s.element(Element::Boolean, |s| s.batch(|buf| buf.extend(b"null"))); - } - RawValue::Object(value) => { - s.element(Element::Object, |s| { - let item = value.parse().unwrap(); - s.batch(|buf| buf.push(b'{')); - let mut has_some = false; - for (k, v) in item.fields.iter() { - has_some |= self.format_field(s, k, *v, None) - } - s.batch(|buf| { - if has_some { - buf.push(b' '); - } - buf.push(b'}'); - }); - }); - } - RawValue::Array(value) => { - let item = value.parse::<256>().unwrap(); - let is_byte_string = item - .iter() - .map(|&v| v.is_byte_code()) - .position(|x| x == false) - .is_none(); - if is_byte_string { - s.batch(|buf| buf.extend(b"b'")); + if !value.is_empty() { + s.space(); s.element(Element::Message, |s| { - for item in item.iter() { - let b = item.parse_byte_code(); - if b >= 32 { - s.batch(|buf| buf.push(b)); - } else { - s.element(Element::String, |s| { - s.batch(|buf| { - buf.push(b'\\'); - buf.push(HEXDIGIT[(b >> 4) as usize]); - buf.push(HEXDIGIT[(b & 0xF) as usize]); - }) - }); - } - } - }); - s.batch(|buf| buf.push(b'\'')); - } else { - s.element(Element::Array, |s| { - s.batch(|buf| buf.push(b'[')); - let mut first = true; - for v in item.iter() { - if !first { - s.batch(|buf| buf.extend(self.cfg.punctuation.array_separator.as_bytes())); - } else { - first = false; - } - self.format_value(s, *v); - } - s.batch(|buf| buf.push(b']')) + s.batch(|buf| buf.with_auto_trim(|buf| MessageFormatAuto::new(value).format(buf).unwrap())) }); } + false } + _ => self.format_field(s, "msg", value, fs, Some(self.fields.as_ref())), }; } } impl RecordWithSourceFormatter for RecordFormatter { + #[inline] fn format_record(&self, buf: &mut Buf, rec: model::RecordWithSource) { RecordFormatter::format_record(self, buf, rec.record) } @@ -305,6 +247,65 @@ impl RecordWithSourceFormatter for RecordFormatter { // --- +struct FormattingState { + key_prefix: KeyPrefix, + flatten: bool, +} + +impl FormattingState { + #[inline] + fn new(flatten: bool) -> Self { + Self { + key_prefix: KeyPrefix::default(), + flatten, + } + } +} + +// --- + +#[derive(Default)] +struct KeyPrefix { + value: OptimizedBuf, +} + +impl KeyPrefix { + #[inline] + fn len(&self) -> usize { + self.value.len() + } + + #[inline] + fn format>(&self, buf: &mut B) { + buf.extend_from_slice(&self.value.head); + buf.extend_from_slice(&self.value.tail); + } + + #[inline] + fn push(&mut self, key: &str) -> usize { + let len = self.len(); + if len != 0 { + self.value.push(b'.'); + } + key.key_prettify(&mut self.value); + self.len() - len + } + + #[inline] + fn pop(&mut self, n: usize) { + if n != 0 { + let len = self.len(); + if n >= len { + self.value.clear(); + } else { + self.value.truncate(len - n); + } + } + } +} + +// --- + struct FieldFormatter<'a> { rf: &'a RecordFormatter, } @@ -319,6 +320,7 @@ impl<'a> FieldFormatter<'a> { s: &mut S, key: &str, value: RawValue<'a>, + fs: &mut FormattingState, filter: Option<&IncludeExcludeKeyFilter>, setting: IncludeExcludeSetting, ) -> bool { @@ -335,20 +337,15 @@ impl<'a> FieldFormatter<'a> { if setting == IncludeExcludeSetting::Exclude && leaf { return false; } - s.space(); - s.element(Element::Key, |s| { - s.batch(|buf| key.key_prettify(buf)); - }); - s.element(Element::Field, |s| { - s.batch(|buf| buf.extend_from_slice(self.rf.cfg.punctuation.field_key_value_separator.as_bytes())); - }); + let ffv = self.begin(s, key, value, fs); if self.rf.unescape_fields { - self.format_value(s, value, filter, setting); + self.format_value(s, value, fs, filter, setting); } else { s.element(Element::String, |s| { s.batch(|buf| buf.extend(value.raw_str().as_bytes())) }); } + self.end(fs, ffv); true } @@ -356,6 +353,7 @@ impl<'a> FieldFormatter<'a> { &mut self, s: &mut S, value: RawValue<'a>, + fs: &mut FormattingState, filter: Option<&IncludeExcludeKeyFilter>, setting: IncludeExcludeSetting, ) { @@ -384,22 +382,26 @@ impl<'a> FieldFormatter<'a> { RawValue::Object(value) => { let item = value.parse().unwrap(); s.element(Element::Object, |s| { - s.batch(|buf| buf.push(b'{')); + if !fs.flatten { + s.batch(|buf| buf.push(b'{')); + } let mut some_fields_hidden = false; for (k, v) in item.fields.iter() { - some_fields_hidden |= !self.format(s, k, *v, filter, setting); + some_fields_hidden |= !self.format(s, k, *v, fs, filter, setting); } if some_fields_hidden { s.element(Element::Ellipsis, |s| { s.batch(|buf| buf.extend(self.rf.cfg.punctuation.hidden_fields_indicator.as_bytes())) }); } - s.batch(|buf| { - if item.fields.len() != 0 { - buf.push(b' '); - } - buf.push(b'}'); - }); + if !fs.flatten { + s.batch(|buf| { + if item.fields.len() != 0 { + buf.push(b' '); + } + buf.push(b'}'); + }); + } }); } RawValue::Array(value) => { @@ -413,13 +415,59 @@ impl<'a> FieldFormatter<'a> { } else { first = false; } - self.format_value(s, *v, None, IncludeExcludeSetting::Unspecified); + self.format_value(s, *v, fs, None, IncludeExcludeSetting::Unspecified); } s.batch(|buf| buf.push(b']')); }); } }; } + + #[inline(always)] + fn begin>( + &mut self, + s: &mut S, + key: &str, + value: RawValue<'a>, + fs: &mut FormattingState, + ) -> FormattedFieldVariant { + if fs.flatten && matches!(value, RawValue::Object(_)) { + return FormattedFieldVariant::Flattened(fs.key_prefix.push(key)); + } + + let variant = FormattedFieldVariant::Normal { flatten: fs.flatten }; + + s.space(); + s.element(Element::Key, |s| { + s.batch(|buf| { + if fs.flatten { + fs.flatten = false; + if fs.key_prefix.len() != 0 { + fs.key_prefix.format(buf); + buf.push(b'.'); + } + } + key.key_prettify(buf); + }); + }); + s.element(Element::Field, |s| { + s.batch(|buf| buf.extend(self.rf.cfg.punctuation.field_key_value_separator.as_bytes())); + }); + + variant + } + + #[inline] + fn end(&mut self, fs: &mut FormattingState, v: FormattedFieldVariant) { + match v { + FormattedFieldVariant::Normal { flatten } => { + fs.flatten = flatten; + } + FormattedFieldVariant::Flattened(n) => { + fs.key_prefix.pop(n); + } + } + } } // --- @@ -446,12 +494,12 @@ impl WithAutoTrim for Vec { // --- -pub trait KeyPrettify { +trait KeyPrettify { fn key_prettify>(&self, buf: &mut B); } impl KeyPrettify for str { - #[inline(always)] + #[inline] fn key_prettify>(&self, buf: &mut B) { let bytes = self.as_bytes(); let mut i = 0; @@ -466,9 +514,12 @@ impl KeyPrettify for str { // --- -const HEXDIGIT: [u8; 16] = [ - b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f', -]; +enum FormattedFieldVariant { + Normal { flatten: bool }, + Flattened(usize), +} + +// --- pub mod string { // workspace imports @@ -787,7 +838,6 @@ mod tests { use super::*; use crate::{ datefmt::LinuxDateFormat, - error::Error, model::{RawObject, Record, RecordFields}, settings::Punctuation, theme::Theme, @@ -799,9 +849,31 @@ mod tests { use encstr::EncodedString; use serde_json as json; - fn format(rec: &Record) -> Result { - let formatter = RecordFormatter::new( - Arc::new(Theme::from(testing::theme()?)), + trait FormatToVec { + fn format_to_vec(&self, rec: &Record) -> Vec; + } + + trait FormatToString { + fn format_to_string(&self, rec: &Record) -> String; + } + + impl FormatToVec for RecordFormatter { + fn format_to_vec(&self, rec: &Record) -> Vec { + let mut buf = Vec::new(); + self.format_record(&mut buf, rec); + buf + } + } + + impl FormatToString for RecordFormatter { + fn format_to_string(&self, rec: &Record) -> String { + String::from_utf8(self.format_to_vec(rec)).unwrap() + } + } + + fn formatter() -> RecordFormatter { + RecordFormatter::new( + Arc::new(Theme::from(testing::theme().unwrap())), DateTimeFormatter::new( LinuxDateFormat::new("%y-%m-%d %T.%3N").compile(), Tz::FixedOffset(Utc.fix()), @@ -810,11 +882,13 @@ mod tests { Arc::new(IncludeExcludeKeyFilter::default()), Formatting { punctuation: Punctuation::test_default(), + flatten: None, }, - ); - let mut buf = Vec::new(); - formatter.format_record(&mut buf, rec); - Ok(String::from_utf8(buf)?) + ) + } + + fn format(rec: &Record) -> String { + formatter().format_to_string(rec) } fn json_raw_value(s: &str) -> Box { @@ -823,22 +897,28 @@ mod tests { #[test] fn test_nested_objects() { + let ka = json_raw_value(r#"{"va":{"kb":42,"kc":43}}"#); + let rec = Record { + ts: Some(Timestamp::new("2000-01-02T03:04:05.123Z")), + message: Some(RawValue::String(EncodedString::json(r#""tm""#))), + level: Some(Level::Debug), + logger: Some("tl"), + caller: Some(Caller::Text("tc")), + fields: RecordFields { + head: heapless::Vec::from_slice(&[("k_a", RawValue::from(RawObject::Json(&ka)))]).unwrap(), + tail: Vec::default(), + }, + predefined: heapless::Vec::default(), + }; + + assert_eq!( + &format(&rec), + "\u{1b}[0;2;3m00-01-02 03:04:05.123 \u{1b}[0;36m|\u{1b}[0;95mDBG\u{1b}[0;36m|\u{1b}[0;2;3m \u{1b}[0;2;4mtl: \u{1b}[0;1;39mtm \u{1b}[0;32mk-a\u{1b}[0;2m=\u{1b}[0;33m{ \u{1b}[0;32mva\u{1b}[0;2m=\u{1b}[0;33m{ \u{1b}[0;32mkb\u{1b}[0;2m=\u{1b}[0;94m42 \u{1b}[0;32mkc\u{1b}[0;2m=\u{1b}[0;94m43\u{1b}[0;33m } }\u{1b}[0;2;3m @ tc\u{1b}[0m", + ); + assert_eq!( - format(&Record { - ts: Some(Timestamp::new("2000-01-02T03:04:05.123Z")), - message: Some(RawValue::String(EncodedString::json(r#""tm""#))), - level: Some(Level::Debug), - logger: Some("tl"), - caller: Some(Caller::Text("tc")), - fields: RecordFields{ - head: heapless::Vec::from_slice(&[ - ("ka", RawValue::from(RawObject::Json(&json_raw_value(r#"{"va":{"kb":42}}"#)))), - ]).unwrap(), - tail: Vec::default(), - }, - predefined: heapless::Vec::default(), - }).unwrap(), - String::from("\u{1b}[0;2;3m00-01-02 03:04:05.123 \u{1b}[0;36m|\u{1b}[0;95mDBG\u{1b}[0;36m|\u{1b}[0;2;3m \u{1b}[0;2;4mtl:\u{1b}[0;2;3m \u{1b}[0;1;39mtm \u{1b}[0;32mka\u{1b}[0;2m:\u{1b}[0;33m{ \u{1b}[0;32mva\u{1b}[0;2m:\u{1b}[0;33m{ \u{1b}[0;32mkb\u{1b}[0;2m:\u{1b}[0;94m42\u{1b}[0;33m } }\u{1b}[0;2;3m @ tc\u{1b}[0m"), + &formatter().with_flatten(true).format_to_string(&rec), + "\u{1b}[0;2;3m00-01-02 03:04:05.123 \u{1b}[0;36m|\u{1b}[0;95mDBG\u{1b}[0;36m|\u{1b}[0;2;3m \u{1b}[0;2;4mtl: \u{1b}[0;1;39mtm \u{1b}[0;32mk-a.va.kb\u{1b}[0;2m=\u{1b}[0;94m42 \u{1b}[0;32mk-a.va.kc\u{1b}[0;2m=\u{1b}[0;94m43\u{1b}[0;2;3m @ tc\u{1b}[0m", ); } } diff --git a/src/main.rs b/src/main.rs index 2e3df636..fe687465 100644 --- a/src/main.rs +++ b/src/main.rs @@ -227,6 +227,7 @@ fn run() -> Result<()> { cli::UnixTimestampUnit::Us => Some(app::UnixTimestampUnit::Microseconds), cli::UnixTimestampUnit::Ns => Some(app::UnixTimestampUnit::Nanoseconds), }, + flatten: opt.flatten != cli::FlattenOption::Never, }); // Configure input. diff --git a/src/model.rs b/src/model.rs index a0d484b5..371f6578 100644 --- a/src/model.rs +++ b/src/model.rs @@ -206,7 +206,7 @@ impl<'a> RawObject<'a> { } } - #[inline(always)] + #[inline] pub fn parse(&self) -> Result> { match self { Self::Json(value) => json::from_str::(value.get()).map_err(Error::JsonParseError), diff --git a/src/settings.rs b/src/settings.rs index c799d2b2..e2ca4902 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -57,7 +57,7 @@ impl Default for Settings { // --- -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct Fields { pub predefined: PredefinedFields, pub ignore: Vec, @@ -86,7 +86,7 @@ pub struct TimeField(pub Field); impl Default for TimeField { fn default() -> Self { Self(Field { - names: vec!["time".into()], + names: vec!["time".into(), "ts".into()], }) } } @@ -104,7 +104,7 @@ impl Default for LevelField { variants: vec![LevelFieldVariant { names: vec!["level".into()], values: Level::iter() - .map(|level| (level, vec![level.as_ref().into()])) + .map(|level| (level, vec![level.as_ref().to_lowercase().into()])) .collect(), level: None, }], @@ -200,6 +200,16 @@ pub struct Field { #[serde(rename_all = "kebab-case")] pub struct Formatting { pub punctuation: Punctuation, + pub flatten: Option, +} + +// --- + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FlattenOption { + Never, + Always, } // --- @@ -229,7 +239,7 @@ impl Default for Punctuation { fn default() -> Self { Self { logger_name_separator: ":".into(), - field_key_value_separator: ":".into(), + field_key_value_separator: "=".into(), string_opening_quote: "'".into(), string_closing_quote: "'".into(), source_location_separator: "@ ".into(), @@ -253,7 +263,7 @@ impl Punctuation { pub fn test_default() -> Self { Self { logger_name_separator: ":".into(), - field_key_value_separator: ":".into(), + field_key_value_separator: "=".into(), string_opening_quote: "'".into(), string_closing_quote: "'".into(), source_location_separator: "@ ".into(), diff --git a/src/theme.rs b/src/theme.rs index 016c8435..f44fdfcf 100644 --- a/src/theme.rs +++ b/src/theme.rs @@ -224,7 +224,7 @@ impl<'a, B: Push> Styler<'a, B> { } impl<'a, B: Push> StylingPush for Styler<'a, B> { - #[inline(always)] + #[inline] fn element R>(&mut self, element: Element, f: F) -> R { let style = self.current; self.set(element); @@ -232,11 +232,13 @@ impl<'a, B: Push> StylingPush for Styler<'a, B> { self.set_style(style); result } - #[inline(always)] + + #[inline] fn space(&mut self) { self.buf.push(b' '); } - #[inline(always)] + + #[inline] fn batch(&mut self, f: F) { self.sync(); f(self.buf)