feat: added support for multiple configuration files

Cargo.toml

@@ -4,7 +4,7 @@ members = [".", "crate/encstr"]
 [workspace.package]
 repository = "https://github.com/pamburus/hl"
 authors = ["Pavel Ivanov <[email protected]>"]
-version = "0.30.0-alpha.4"
+version = "0.30.0-alpha.5"
 edition = "2021"
 license = "MIT"
 
@@ -58,6 +58,7 @@ htp = { git = "https://github.com/pamburus/htp.git" }
 humantime = "2"
 itertools = "0.13"
 itoa = { version = "1", default-features = false }
+known-folders = "1"
 log = "0"
 nonzero_ext = "0"
 notify = { version = "7", features = ["macos_kqueue"] }

Makefile

@@ -55,7 +55,7 @@ install-man-pages: ~/share/man/man1/hl.1
 	@echo $$(tput setaf 3)NOTE:$$(tput sgr0) ensure $$(tput setaf 2)~/share/man$$(tput sgr0) is added to $$(tput setaf 2)MANPATH$$(tput sgr0) environment variable
 
 ~/share/man/man1/hl.1: contrib-build | ~/share/man/man1
-	@HL_CONFIG= cargo run --release --locked -- --man-page >"$@"
+	cargo run --release --locked -- --config - --man-page >"$@"
 
 ~/share/man/man1:
 	@mkdir -p "$@"
@@ -79,7 +79,7 @@ bench: contrib-build
 ## Show usage of the binary
 .PHONY: usage
 usage: build
-	@env -i HL_CONFIG= ./target/debug/hl --help
+	@./target/debug/hl --config - --help
 
 ## Clean build artifacts
 .PHONY: clean

README.md

@@ -391,15 +391,27 @@ See other [screenshots](https://github.com/pamburus/hl-extra/tree/90be58af2fb91d
 
 ### Configuration files
 
-* Configuration file is automatically loaded if found in a predefined platform-specific location.
+* Configuration files are automatically loaded if found in predefined platform-specific locations.
 
-    | OS      | Location                                                  |
-    | ------- | --------------------------------------------------------- |
-    | macOS   | ~/.config/hl/config.{yaml,toml,json}                      |
-    | Linux   | ~/.config/hl/config.{yaml,toml,json}                      |
-    | Windows | %USERPROFILE%\AppData\Roaming\hl\config.{yaml,toml,json}  |
+    | OS      | System-Wide Location                     | User Profile Location                                    |
+    | ------- | ---------------------------------------- |  ------------------------------------------------------- |
+    | macOS   | /etc/hl/config.{yaml,toml,json}          | ~/.config/hl/config.{yaml,toml,json}                     |
+    | Linux   | /etc/hl/config.{yaml,toml,json}          | ~/.config/hl/config.{yaml,toml,json}                     |
+    | Windows | %PROGRAMDATA%\hl\config.{yaml,toml,json} | %USERPROFILE%\AppData\Roaming\hl\config.{yaml,toml,json} |
 
-* The path to the configuration file can be overridden using the HL_CONFIG environment variable.
+* The path to the configuration file can be overridden using the `HL_CONFIG` environment variable or the `--config` command-line option.
+
+    The order in which the configuration files are searched and loaded is as follows:
+    1. **The system-wide location.**
+    2. **The user profile location.**
+    3. **The location specified by the `HL_CONFIG` environment variable** (unless the `--config` option is used).
+    4. **The locations specified by the `--config` option** (can be specified multiple times).
+
+    If a configuration file is found in multiple locations, the file in each subsequent location overrides only the parameters it contains.
+
+    If `HL_CONFIG` or `--config` specifies `-` or an empty string, all default locations and any locations specified by previous `--config` options are discarded. The search for the configuration file locations starts over.
+
+    To disable loading of configuration files and use the built-in defaults, `--config -` can be used.
 
 * All parameters in the configuration file are optional and can be omitted. In this case, default values are used.
 
@@ -581,7 +593,6 @@ Advanced Options:
       --man-page                    Print man page and exit
       --list-themes                 Print available themes and exit
       --dump-index                  Print debug index metadata (in --sort mode) and exit
-      --debug                       Print debug error messages that can help with troubleshooting
 ```
 
 ## Performance

build/ci/coverage.sh

@@ -41,15 +41,15 @@ function clean() {
 function test() {
     cargo test --tests --workspace
     cargo build
-    ${MAIN_EXECUTABLE:?} --config= > /dev/null
-    ${MAIN_EXECUTABLE:?} --config= --help > /dev/null
-    ${MAIN_EXECUTABLE:?} --config=etc/defaults/config-k8s.yaml > /dev/null
-    ${MAIN_EXECUTABLE:?} --config=etc/defaults/config-ecs.yaml > /dev/null
-    ${MAIN_EXECUTABLE:?} --config= --shell-completions bash > /dev/null
-    ${MAIN_EXECUTABLE:?} --config= --man-page > /dev/null
-    ${MAIN_EXECUTABLE:?} --config= --list-themes > /dev/null
-    ${MAIN_EXECUTABLE:?} --config= sample/prometheus.log -P > /dev/null
-    echo "" | ${MAIN_EXECUTABLE:?} --config= --concurrency 4 > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --help > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --config=etc/defaults/config-k8s.yaml > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --config=etc/defaults/config-ecs.yaml > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --shell-completions bash > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --man-page > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - --list-themes > /dev/null
+    ${MAIN_EXECUTABLE:?} --config - sample/prometheus.log -P > /dev/null
+    echo "" | ${MAIN_EXECUTABLE:?} --config - --concurrency 4 > /dev/null
 } 
 
 function merge() {

contrib/bin/screenshot.sh

@@ -9,12 +9,13 @@ SAMPLE=${2:?}
 THEME=${3:?}
 TITLE="hl ${SAMPLE:?}"
 
-HL_CONFIG= "${ALACRITTY:?}" \
+"${ALACRITTY:?}" \
     --config-file "${HL_SRC:?}"/contrib/etc/alacritty/${MODE:?}.toml \
     -T "${TITLE:?}" \
     --hold \
     -e \
         "${HL_SRC:?}"/target/debug/hl \
+        --config - \
         --theme ${THEME:?} \
         -P \
         "${HL_SRC:?}"/sample/${SAMPLE:?} &

src/app.rs

@@ -70,7 +70,6 @@ pub struct Options {
     pub input_info: Option<InputInfo>,
     pub input_format: Option<InputFormat>,
     pub dump_index: bool,
-    pub debug: bool,
     pub app_dirs: Option<AppDirs>,
     pub tail: u64,
     pub delimiter: Delimiter,
@@ -396,8 +395,8 @@ impl App {
                                     if let Some(ts) = &record.ts {
                                         if let Some(unix_ts) = ts.unix_utc() {
                                             items.push((unix_ts.into(), location));
-                                        } else if self.options.debug {
-                                            eprintln!(
+                                        } else {
+                                            log::warn!(
                                                 "skipped a message because its timestamp could not be parsed: {:#?}",
                                                 ts.raw()
                                             )
@@ -1450,7 +1449,6 @@ mod tests {
             input_info: None,
             input_format: None,
             dump_index: false,
-            debug: false,
             app_dirs: None,
             tail: 0,
             delimiter: Delimiter::default(),

src/appdirs.rs

@@ -4,13 +4,19 @@ use std::path::PathBuf;
 pub struct AppDirs {
     pub cache_dir: PathBuf,
     pub config_dir: PathBuf,
+    pub system_config_dirs: Vec<PathBuf>,
 }
 
 impl AppDirs {
     pub fn new(name: &str) -> Option<Self> {
         let cache_dir = sys::cache_dir()?.join(name);
         let config_dir = sys::config_dir()?.join(name);
-        Some(Self { cache_dir, config_dir })
+        let system_config_dirs = sys::system_config_dirs().into_iter().map(|d| d.join(name)).collect();
+        Some(Self {
+            cache_dir,
+            config_dir,
+            system_config_dirs,
+        })
     }
 }
 
@@ -19,16 +25,20 @@ mod sys {
     use super::*;
     use std::env;
 
+    pub(crate) fn cache_dir() -> Option<PathBuf> {
+        env::var_os("XDG_CACHE_HOME")
+            .and_then(dirs_sys::is_absolute_path)
+            .or_else(|| dirs::home_dir().map(|h| h.join(".cache")))
+    }
+
     pub(crate) fn config_dir() -> Option<PathBuf> {
         env::var_os("XDG_CONFIG_HOME")
             .and_then(dirs_sys::is_absolute_path)
             .or_else(|| dirs::home_dir().map(|h| h.join(".config")))
     }
 
-    pub(crate) fn cache_dir() -> Option<PathBuf> {
-        env::var_os("XDG_CACHE_HOME")
-            .and_then(dirs_sys::is_absolute_path)
-            .or_else(|| dirs::home_dir().map(|h| h.join(".cache")))
+    pub(crate) fn system_config_dirs() -> Vec<PathBuf> {
+        vec![PathBuf::from("/etc")]
     }
 }
 
@@ -43,4 +53,16 @@ mod sys {
     pub(crate) fn cache_dir() -> Option<PathBuf> {
         dirs::cache_dir()
     }
+
+    #[cfg(not(target_os = "windows"))]
+    pub(crate) fn system_config_dirs() -> Vec<PathBuf> {
+        vec![PathBuf::from("/etc")]
+    }
+
+    #[cfg(target_os = "windows")]
+    pub(crate) fn system_config_dirs() -> Vec<PathBuf> {
+        use known_folders::{get_known_folder_path, KnownFolder};
+
+        get_known_folder_path(KnownFolder::ProgramData).into_iter().collect()
+    }
 }

src/cli.rs

@@ -19,14 +19,8 @@ use crate::{
 #[derive(Args)]
 pub struct BootstrapArgs {
     /// Configuration file path.
-    #[arg(
-        long,
-        overrides_with = "config",
-        value_name = "FILE",
-        env = "HL_CONFIG",
-        num_args = 1
-    )]
-    pub config: Option<String>,
+    #[arg(long, value_name = "FILE", env = "HL_CONFIG", num_args = 1)]
+    pub config: Vec<String>,
 }
 
 /// JSON and logfmt log converter to human readable representation.
@@ -133,8 +127,8 @@ pub struct Opt {
         short,
         long,
         env = "HL_LEVEL",
-        overrides_with="level",
-        ignore_case=true,
+        overrides_with = "level",
+        ignore_case = true,
         value_parser = LevelValueParser,
         value_enum,
         help_heading = heading::FILTERING
@@ -164,12 +158,12 @@ pub struct Opt {
     /// Filter messages by field values
     /// [k=v, k~=v, k~~=v, 'k!=v', 'k!~=v', 'k!~~=v']
     /// where ~ does substring match and ~~ does regular expression match.
-    #[arg(short, long, number_of_values = 1, help_heading = heading::FILTERING)]
+    #[arg(short, long, num_args = 1, help_heading = heading::FILTERING)]
     pub filter: Vec<String>,
 
     /// Filter using query, accepts expressions from --filter
     /// and supports '(', ')', 'and', 'or', 'not', 'in', 'contain', 'like', '<', '>', '<=', '>=', etc.
-    #[arg(short, long, number_of_values = 1, help_heading = heading::FILTERING)]
+    #[arg(short, long, num_args = 1, help_heading = heading::FILTERING)]
     pub query: Vec<String>,
 
     /// Color output control.
@@ -220,7 +214,7 @@ pub struct Opt {
     #[arg(
         long,
         short = 'h',
-        number_of_values = 1,
+        num_args = 1,
         value_name = "KEY",
         help_heading = heading::OUTPUT
     )]
@@ -405,10 +399,6 @@ pub struct Opt {
     #[arg(long, requires = "sort", help_heading = heading::ADVANCED)]
     pub dump_index: bool,
 
-    /// Print debug error messages that can help with troubleshooting.
-    #[arg(long, help_heading = heading::ADVANCED)]
-    pub debug: bool,
-
     /// Print help.
     #[arg(long, default_value_t = false, action = ArgAction::SetTrue)]
     pub help: bool,