Merge pull request #9 from pamburus/feature/config

new: added configuration file support

Cargo.toml

@@ -4,7 +4,7 @@ categories = ["command-line-utilities"]
 description = "Utility for viewing json-formatted log files."
 keywords = ["cli", "human", "log"]
 name = "hl"
-version = "0.8.15"
+version = "0.9.0"
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -18,15 +18,18 @@ bytefmt = "0"
 chrono = { version = "0", features = ["serde"] }
 chrono-tz = "0"
 closure = "0"
+config = "0"
 crossbeam-channel = "0"
 crossbeam-queue = "0"
 crossbeam-utils = "0"
+derive_deref = "1"
 error-chain = "0"
 flate2 = "1"
 heapless = "0"
 atty = "0"
 itertools = "0"
 num_cpus = "1"
+platform-dirs = "0"
 serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["raw_value"] }
 shellwords = "1"

etc/default-config.yaml

@@ -0,0 +1,23 @@
+fields:
+  time:
+    names: [ts, TS, time, TIME, Time, _SOURCE_REALTIME_TIMESTAMP, __REALTIME_TIMESTAMP]
+  logger:
+    names: [logger, LOGGER, Logger]
+  level:
+    variants:
+      - names: [level, LEVEL, Level]
+        values:
+          debug: [debug]
+          info: [info, information]
+          warning: [warning, warn]
+          error: [error, err, fatal, critical, panic]
+      - names: [PRIORITY]
+        values:
+          debug: [7]
+          info: [6]
+          warning: [5, 4]
+          error: [3, 2, 1]
+  message:
+    names: [msg, message, MESSAGE, Message]
+  caller:
+    names: [caller, CALLER, Caller]

src/app.rs

@@ -12,12 +12,15 @@ use serde_json as json;
 use crate::datefmt::{DateTimeFormat, DateTimeFormatter};
 use crate::error::*;
 use crate::formatting::RecordFormatter;
-use crate::model::{Filter, Record};
+use crate::model::{Filter, Parser, ParserSettings, RawRecord};
 use crate::scanning::{BufFactory, Scanner, Segment, SegmentBuf, SegmentBufFactory};
+use crate::settings::Settings;
 use crate::theme::Theme;
 use crate::IncludeExcludeKeyFilter;
 
+// TODO: merge Options to Settings and replace Options with Settings.
 pub struct Options {
+    pub settings: Settings,
     pub theme: Arc<Theme>,
     pub time_format: DateTimeFormat,
     pub raw_fields: bool,
@@ -47,6 +50,8 @@ impl App {
         let n = self.options.concurrency;
         let sfi = Arc::new(SegmentBufFactory::new(self.options.buffer_size));
         let bfo = BufFactory::new(self.options.buffer_size);
+        let settings = ParserSettings::from(&self.options.settings);
+        let parser = Parser::new(&settings);
         thread::scope(|scope| -> Result<()> {
             // prepare receive/transmit channels for input data
             let (txi, rxi): (Vec<_>, Vec<_>) = (0..n).map(|_| channel::bounded(1)).unzip();
@@ -69,7 +74,7 @@ impl App {
             }));
             // spawn processing threads
             for (rxi, txo) in izip!(rxi, txo) {
-                scope.spawn(closure!(ref bfo, ref sfi, |_| {
+                scope.spawn(closure!(ref bfo, ref parser, ref sfi, |_| {
                     let mut formatter = RecordFormatter::new(
                         self.options.theme.clone(),
                         DateTimeFormatter::new(
@@ -84,7 +89,7 @@ impl App {
                         match segment {
                             Segment::Complete(segment) => {
                                 let mut buf = bfo.new_buf();
-                                self.process_segement(&segment, &mut formatter, &mut buf);
+                                self.process_segement(&parser, &segment, &mut formatter, &mut buf);
                                 sfi.recycle(segment);
                                 if let Err(_) = txo.send(buf) {
                                     break;
@@ -128,6 +133,7 @@ impl App {
 
     fn process_segement(
         &self,
+        parser: &Parser,
         segment: &SegmentBuf,
         formatter: &mut RecordFormatter,
         buf: &mut Vec<u8>,
@@ -137,8 +143,9 @@ impl App {
             if data.len() == 0 {
                 continue;
             }
-            let mut stream = json::Deserializer::from_slice(data).into_iter::<Record>();
+            let mut stream = json::Deserializer::from_slice(data).into_iter::<RawRecord>();
             while let Some(Ok(record)) = stream.next() {
+                let record = parser.parse(record);
                 if record.matches(&self.options.filter) {
                     formatter.format_record(buf, &record);
                 }

src/error.rs

@@ -4,6 +4,7 @@ use std::io;
 use std::num::{ParseIntError, TryFromIntError};
 
 // third-party imports
+use config::ConfigError;
 use thiserror::Error;
 
 /// Error is an error which may occur in the application.
@@ -15,6 +16,8 @@ pub enum Error {
     ParseIntError(#[from] ParseIntError),
     #[error(transparent)]
     TryFromIntError(#[from] TryFromIntError),
+    #[error("failed to load configuration: {0}")]
+    Config(#[from] ConfigError),
     #[error(transparent)]
     Boxed(#[from] Box<dyn std::error::Error + std::marker::Send>),
     #[error("file {filename:?} not found")]
@@ -24,6 +27,11 @@ pub enum Error {
         value: String,
         valid_values: Vec<String>,
     },
+    #[error("invalid field kind {value:?}, use any of {valid_values:?}")]
+    InvalidFieldKind {
+        value: String,
+        valid_values: Vec<String>,
+    },
     #[error(
         "invalid size {0:?}, use {:?} or {:?} format for IEC units or {:?} format for SI units",
         "64K",

src/lib.rs

@@ -5,6 +5,7 @@ pub mod error;
 pub mod fmtx;
 pub mod input;
 pub mod output;
+pub mod settings;
 pub mod theme;
 pub mod timestamp;
 pub mod types;

src/main.rs

@@ -10,6 +10,7 @@ use std::time::Duration;
 use ansi_term::Colour;
 use chrono::{FixedOffset, Local, TimeZone};
 use chrono_tz::{Tz, UTC};
+use platform_dirs::AppDirs;
 use structopt::clap::arg_enum;
 use structopt::StructOpt;
 
@@ -18,6 +19,7 @@ use hl::datefmt::LinuxDateFormat;
 use hl::error::*;
 use hl::input::{open, ConcatReader, Input, InputStream};
 use hl::output::{OutputStream, Pager};
+use hl::settings::Settings;
 use hl::signal::SignalHandler;
 use hl::theme::Theme;
 use hl::Level;
@@ -166,6 +168,8 @@ fn parse_non_zero_size(s: &str) -> Result<usize> {
 }
 
 fn run() -> Result<()> {
+    let app_dirs = AppDirs::new(Some("hl"), true).unwrap();
+    let settings = Settings::load(&app_dirs)?;
     let opt = Opt::from_args();
     let stdout_is_atty = || atty::is(atty::Stream::Stdout);
 
@@ -220,6 +224,7 @@ fn run() -> Result<()> {
 
     // Create app.
     let app = hl::App::new(hl::Options {
+        settings: settings,
         theme: Arc::new(theme),
         raw_fields: opt.raw_fields,
         time_format: LinuxDateFormat::new(&opt.time_format).compile(),