new: fields and message expansion feature

Cargo.toml

@@ -4,7 +4,7 @@ members = [".", "crate/encstr"]
 [workspace.package]
 repository = "https://github.com/pamburus/hl"
 authors = ["Pavel Ivanov <[email protected]>"]
-version = "0.30.0"
+version = "0.31.0-alpha.1"
 edition = "2021"
 license = "MIT"
 
@@ -53,13 +53,15 @@ enumset-ext = { path = "./crate/enumset-ext" }
 env_logger = "0"
 flate2 = "1"
 heapless = "0"
+heapopt = { path = "./crate/heapopt" }
 hex = "0"
 htp = { git = "https://github.com/pamburus/htp.git" }
 humantime = "2"
 itertools = "0.13"
 itoa = { version = "1", default-features = false }
 known-folders = "1"
 log = "0"
+mline = { path = "./crate/mline" }
 nonzero_ext = "0"
 notify = { version = "7", features = ["macos_kqueue"] }
 nu-ansi-term = "0"

README.md

@@ -575,6 +575,7 @@ Output Options:
       --no-local              Disable local time zone, overrides --local option
   -e, --hide-empty-fields     Hide empty fields, applies for null, string, object and array fields only [env: HL_HIDE_EMPTY_FIELDS=]
   -E, --show-empty-fields     Show empty fields, overrides --hide-empty-fields option [env: HL_SHOW_EMPTY_FIELDS=]
+  -x, --expansion <MODE>      Whether to expand fields and messages [env: HL_EXPANSION=] [default: medium] [possible values: never, inline, low, medium, high, always]
       --input-info <VARIANT>  Show input number and/or input filename before each message [default: auto] [possible values: auto, none, full, compact, minimal]
   -o, --output <FILE>         Output file
 

benches/parse-and-format.rs

@@ -9,9 +9,10 @@ use criterion::{criterion_group, criterion_main, Criterion};
 // local imports
 use hl::{
     app::{RecordIgnorer, SegmentProcess, SegmentProcessorOptions},
-    settings,
+    formatting::Expansion,
+    settings::ExpansionMode,
     timezone::Tz,
-    DateTimeFormatter, Filter, IncludeExcludeKeyFilter, LinuxDateFormat, Parser, ParserSettings, RecordFormatter,
+    DateTimeFormatter, Filter, LinuxDateFormat, Parser, ParserSettings, RecordFormatter, RecordFormatterSettings,
     SegmentProcessor, Settings, Theme,
 };
 
@@ -20,29 +21,33 @@ use hl::{
 fn benchmark(c: &mut Criterion) {
     let mut c = c.benchmark_group("parse-and-format");
     for (name, record) in [("kibana-record-01", KIBANA_RECORD_01)] {
-        for theme in ["universal", "classic"] {
-            c.bench_function(format!("{}/{}", name, theme), |b| {
-                let settings = Settings::default();
-                let parser = Parser::new(ParserSettings::new(&settings.fields.predefined, empty(), false, None));
-                let formatter = RecordFormatter::new(
-                    Arc::new(Theme::embedded(theme).unwrap()),
-                    DateTimeFormatter::new(
-                        LinuxDateFormat::new("%b %d %T.%3N").compile(),
-                        Tz::FixedOffset(Utc.fix()),
-                    ),
-                    false,
-                    Arc::new(IncludeExcludeKeyFilter::default()),
-                    settings::Formatting::default(),
-                );
-                let filter = Filter::default();
-                let mut processor =
-                    SegmentProcessor::new(&parser, &formatter, &filter, SegmentProcessorOptions::default());
-                let mut buf = Vec::new();
-                b.iter(|| {
-                    processor.process(record, &mut buf, "", None, &mut RecordIgnorer {});
-                    buf.clear();
+        for expansion in [ExpansionMode::Inline, ExpansionMode::Medium] {
+            for theme in ["universal", "classic"] {
+                c.bench_function(format!("{}/expansion-{}/theme-{}", name, expansion, theme), |b| {
+                    let settings = Settings::default();
+                    let parser = Parser::new(ParserSettings::new(&settings.fields.predefined, empty(), false, None));
+                    let formatter = RecordFormatter::new(RecordFormatterSettings {
+                        theme: Arc::new(Theme::embedded(theme).unwrap()),
+                        ts_formatter: DateTimeFormatter::new(
+                            LinuxDateFormat::new("%b %d %T.%3N").compile(),
+                            Tz::FixedOffset(Utc.fix()),
+                        ),
+                        expansion: Expansion {
+                            mode: expansion,
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    });
+                    let filter = Filter::default();
+                    let mut processor =
+                        SegmentProcessor::new(&parser, &formatter, &filter, SegmentProcessorOptions::default());
+                    let mut buf = Vec::new();
+                    b.iter(|| {
+                        processor.process(record, &mut buf, "", None, &mut RecordIgnorer {});
+                        buf.clear();
+                    });
                 });
-            });
+            }
         }
     }
 }

benches/ts-format.rs

@@ -1,16 +1,11 @@
-use std::io::Write;
+use std::{hint::black_box, io::Write};
 
 use chrono::{format::strftime::StrftimeItems, Datelike, FixedOffset, Timelike};
 use criterion::{criterion_group, criterion_main, Criterion};
 
 use hl::datefmt::{DateTimeFormatter, LinuxDateFormat};
 use hl::{timestamp::Timestamp, timezone::Tz};
 
-#[inline(never)]
-fn ignore<T>(t: T) {
-    drop(t)
-}
-
 fn benchmark(c: &mut Criterion) {
     let mut c = c.benchmark_group("ts-format");
     let tsr = Timestamp::new("2020-06-27T00:48:30.466249792+00:00");
@@ -19,15 +14,15 @@ fn benchmark(c: &mut Criterion) {
     let tz = |secs| Tz::FixedOffset(FixedOffset::east_opt(secs).unwrap());
     c.bench_function("chrono conversion to naive local", |b| {
         b.iter(|| {
-            ignore(ts.naive_local());
+            black_box(ts.naive_local());
         });
     });
     c.bench_function("datefmt format utc [%y-%m-%d %T.%N]", |b| {
         let mut buf = Vec::<u8>::with_capacity(4096);
         let format = LinuxDateFormat::new("%y-%m-%d %T.%N").compile();
         let formatter = DateTimeFormatter::new(format, tz(0));
         b.iter(|| {
-            formatter.format(&mut buf, ts);
+            black_box(formatter.format(&mut buf, ts));
             buf.clear();
         });
     });
@@ -36,7 +31,7 @@ fn benchmark(c: &mut Criterion) {
         let format = LinuxDateFormat::new("%y-%m-%d %T.%N").compile();
         let formatter = DateTimeFormatter::new(format, tz(3 * 3600));
         b.iter(|| {
-            formatter.format(&mut buf, ts);
+            black_box(formatter.format(&mut buf, ts));
             buf.clear();
         });
     });
@@ -45,7 +40,7 @@ fn benchmark(c: &mut Criterion) {
         let format = LinuxDateFormat::new("%b %d %T.%N").compile();
         let formatter = DateTimeFormatter::new(format, tz(0));
         b.iter(|| {
-            formatter.format(&mut buf, ts);
+            black_box(formatter.format(&mut buf, ts));
             buf.clear();
         });
     });
@@ -54,7 +49,7 @@ fn benchmark(c: &mut Criterion) {
         let format = LinuxDateFormat::new("%b %d %T.%N").compile();
         let formatter = DateTimeFormatter::new(format, tz(3 * 3600));
         b.iter(|| {
-            formatter.format(&mut buf, ts);
+            black_box(formatter.format(&mut buf, ts));
             buf.clear();
         });
     });
@@ -64,7 +59,7 @@ fn benchmark(c: &mut Criterion) {
         let formatter = DateTimeFormatter::new(format, tz(0));
         let tsr = tsr.as_rfc3339().unwrap();
         b.iter(|| {
-            formatter.reformat_rfc3339(&mut buf, tsr.clone());
+            black_box(formatter.reformat_rfc3339(&mut buf, tsr.clone()));
             buf.clear();
         });
     });
@@ -74,7 +69,7 @@ fn benchmark(c: &mut Criterion) {
         let formatter = DateTimeFormatter::new(format, tz(0));
         let tsr = tsr.as_rfc3339().unwrap();
         b.iter(|| {
-            formatter.reformat_rfc3339(&mut buf, tsr.clone());
+            black_box(formatter.reformat_rfc3339(&mut buf, tsr.clone()));
             buf.clear();
         });
     });
@@ -84,20 +79,20 @@ fn benchmark(c: &mut Criterion) {
         let formatter = DateTimeFormatter::new(format, tz(0));
         let tsr = tsr.as_rfc3339().unwrap();
         b.iter(|| {
-            formatter.reformat_rfc3339(&mut buf, tsr.clone());
+            black_box(formatter.reformat_rfc3339(&mut buf, tsr.clone()));
             buf.clear();
         });
     });
     c.bench_function("calling chrono date-time methods", |b| {
         b.iter(|| {
             assert!(
-                tsn.year() as i64
-                    + tsn.month() as i64
-                    + tsn.day() as i64
-                    + tsn.hour() as i64
-                    + tsn.minute() as i64
-                    + tsn.second() as i64
-                    + tsn.nanosecond() as i64
+                black_box(tsn.year()) as i64
+                    + black_box(tsn.month()) as i64
+                    + black_box(tsn.day()) as i64
+                    + black_box(tsn.hour()) as i64
+                    + black_box(tsn.minute()) as i64
+                    + black_box(tsn.second()) as i64
+                    + black_box(tsn.nanosecond()) as i64
                     != 0
             );
         });
@@ -106,7 +101,7 @@ fn benchmark(c: &mut Criterion) {
     c.bench_function("chrono format", |b| {
         let mut buf = Vec::<u8>::with_capacity(4096);
         b.iter(|| {
-            assert!(write!(&mut buf, "{}", ts.format_with_items(items.clone())).is_ok());
+            assert!(black_box(write!(&mut buf, "{}", ts.format_with_items(items.clone()))).is_ok());
             buf.clear();
         });
     });

build/ci/coverage.sh

@@ -49,7 +49,7 @@ function test() {
     ${MAIN_EXECUTABLE:?} --config - --man-page > /dev/null
     ${MAIN_EXECUTABLE:?} --config - --list-themes > /dev/null
     ${MAIN_EXECUTABLE:?} --config - sample/prometheus.log -P > /dev/null
-    echo "" | ${MAIN_EXECUTABLE:?} --config - --concurrency 4 > /dev/null
+    echo "" | ${MAIN_EXECUTABLE:?} --config - --config=etc/defaults/config.yaml --concurrency 4 > /dev/null
 } 
 
 function merge() {

crate/encstr/src/error.rs

@@ -24,3 +24,5 @@ impl fmt::Display for Error {
         }
     }
 }
+
+impl std::error::Error for Error {}

crate/heapopt/.gitignore

@@ -0,0 +1 @@
+/target

crate/heapopt/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+description = "Optimized collections for minimal heap usage"
+name = "heapopt"
+version = "0.1.0"
+workspace = "../.."
+repository.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+
+[dependencies]
+heapless = "0"