From 19ea89e68dfb46603271993d9a6f451926569631 Mon Sep 17 00:00:00 2001 From: Pavel Ivanov Date: Thu, 19 Dec 2024 00:37:47 +0100 Subject: [PATCH] feat: added support for multiple configuration files --- Makefile | 4 +- README.md | 27 +++++++--- build/ci/coverage.sh | 18 +++---- contrib/bin/screenshot.sh | 3 +- src/app.rs | 6 +-- src/appdirs.rs | 32 +++++++++-- src/cli.rs | 24 +++------ src/config.rs | 108 +++++++++++++++++++++++++++----------- src/main.rs | 30 +++++++---- src/settings.rs | 71 +++++++++++++++++-------- src/themecfg.rs | 1 + 11 files changed, 216 insertions(+), 108 deletions(-) diff --git a/Makefile b/Makefile index aee74767..f3b8fca2 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ install-man-pages: ~/share/man/man1/hl.1 @echo $$(tput setaf 3)NOTE:$$(tput sgr0) ensure $$(tput setaf 2)~/share/man$$(tput sgr0) is added to $$(tput setaf 2)MANPATH$$(tput sgr0) environment variable ~/share/man/man1/hl.1: contrib-build | ~/share/man/man1 - @HL_CONFIG= cargo run --release --locked -- --man-page >"$@" + cargo run --release --locked -- --config - --man-page >"$@" ~/share/man/man1: @mkdir -p "$@" @@ -79,7 +79,7 @@ bench: contrib-build ## Show usage of the binary .PHONY: usage usage: build - @env -i HL_CONFIG= ./target/debug/hl --help + @./target/debug/hl --config - --help ## Clean build artifacts .PHONY: clean diff --git a/README.md b/README.md index dcea815d..d5357ed1 100644 --- a/README.md +++ b/README.md @@ -391,15 +391,27 @@ See other [screenshots](https://github.com/pamburus/hl-extra/tree/90be58af2fb91d ### Configuration files -* Configuration file is automatically loaded if found in a predefined platform-specific location. +* Configuration files are automatically loaded if found in predefined platform-specific locations. - | OS | Location | - | ------- | --------------------------------------------------------- | - | macOS | ~/.config/hl/config.{yaml,toml,json} | - | Linux | ~/.config/hl/config.{yaml,toml,json} | - | Windows | %USERPROFILE%\AppData\Roaming\hl\config.{yaml,toml,json} | + | OS | System-Wide Location | User Profile Location | + | ------- | ---------------------------------------- | ------------------------------------------------------- | + | macOS | /etc/hl/config.{yaml,toml,json} | ~/.config/hl/config.{yaml,toml,json} | + | Linux | /etc/hl/config.{yaml,toml,json} | ~/.config/hl/config.{yaml,toml,json} | + | Windows | %PROGRAMDATA%\hl\config.{yaml,toml,json} | %USERPROFILE%\AppData\Roaming\hl\config.{yaml,toml,json} | -* The path to the configuration file can be overridden using the HL_CONFIG environment variable. +* The path to the configuration file can be overridden using the `HL_CONFIG` environment variable or the `--config` command-line option. + + The order in which the configuration files are searched and loaded is as follows: + 1. **The system-wide location.** + 2. **The user profile location.** + 3. **The location specified by the `HL_CONFIG` environment variable** (unless the `--config` option is used). + 4. **The locations specified by the `--config` option** (can be specified multiple times). + + If a configuration file is found in multiple locations, the file in each subsequent location overrides only the parameters it contains. + + If `HL_CONFIG` or `--config` specifies `-` or an empty string, all default locations and any locations specified by previous `--config` options are discarded. The search for the configuration file locations starts over. + + To disable loading of configuration files and use the built-in defaults, `--config -` can be used. * All parameters in the configuration file are optional and can be omitted. In this case, default values are used. @@ -581,7 +593,6 @@ Advanced Options: --man-page Print man page and exit --list-themes Print available themes and exit --dump-index Print debug index metadata (in --sort mode) and exit - --debug Print debug error messages that can help with troubleshooting ``` ## Performance diff --git a/build/ci/coverage.sh b/build/ci/coverage.sh index c9392605..91e8f026 100755 --- a/build/ci/coverage.sh +++ b/build/ci/coverage.sh @@ -41,15 +41,15 @@ function clean() { function test() { cargo test --tests --workspace cargo build - ${MAIN_EXECUTABLE:?} --config= > /dev/null - ${MAIN_EXECUTABLE:?} --config= --help > /dev/null - ${MAIN_EXECUTABLE:?} --config=etc/defaults/config-k8s.yaml > /dev/null - ${MAIN_EXECUTABLE:?} --config=etc/defaults/config-ecs.yaml > /dev/null - ${MAIN_EXECUTABLE:?} --config= --shell-completions bash > /dev/null - ${MAIN_EXECUTABLE:?} --config= --man-page > /dev/null - ${MAIN_EXECUTABLE:?} --config= --list-themes > /dev/null - ${MAIN_EXECUTABLE:?} --config= sample/prometheus.log -P > /dev/null - echo "" | ${MAIN_EXECUTABLE:?} --config= --concurrency 4 > /dev/null + ${MAIN_EXECUTABLE:?} --config - > /dev/null + ${MAIN_EXECUTABLE:?} --config - --help > /dev/null + ${MAIN_EXECUTABLE:?} --config - --config=etc/defaults/config-k8s.yaml > /dev/null + ${MAIN_EXECUTABLE:?} --config - --config=etc/defaults/config-ecs.yaml > /dev/null + ${MAIN_EXECUTABLE:?} --config - --shell-completions bash > /dev/null + ${MAIN_EXECUTABLE:?} --config - --man-page > /dev/null + ${MAIN_EXECUTABLE:?} --config - --list-themes > /dev/null + ${MAIN_EXECUTABLE:?} --config - sample/prometheus.log -P > /dev/null + echo "" | ${MAIN_EXECUTABLE:?} --config - --concurrency 4 > /dev/null } function merge() { diff --git a/contrib/bin/screenshot.sh b/contrib/bin/screenshot.sh index cc4e98f4..c946e49a 100755 --- a/contrib/bin/screenshot.sh +++ b/contrib/bin/screenshot.sh @@ -9,12 +9,13 @@ SAMPLE=${2:?} THEME=${3:?} TITLE="hl ${SAMPLE:?}" -HL_CONFIG= "${ALACRITTY:?}" \ +"${ALACRITTY:?}" \ --config-file "${HL_SRC:?}"/contrib/etc/alacritty/${MODE:?}.toml \ -T "${TITLE:?}" \ --hold \ -e \ "${HL_SRC:?}"/target/debug/hl \ + --config - \ --theme ${THEME:?} \ -P \ "${HL_SRC:?}"/sample/${SAMPLE:?} & diff --git a/src/app.rs b/src/app.rs index 7ed72829..aa93dfe4 100644 --- a/src/app.rs +++ b/src/app.rs @@ -70,7 +70,6 @@ pub struct Options { pub input_info: Option, pub input_format: Option, pub dump_index: bool, - pub debug: bool, pub app_dirs: Option, pub tail: u64, pub delimiter: Delimiter, @@ -396,8 +395,8 @@ impl App { if let Some(ts) = &record.ts { if let Some(unix_ts) = ts.unix_utc() { items.push((unix_ts.into(), location)); - } else if self.options.debug { - eprintln!( + } else { + log::debug!( "skipped a message because its timestamp could not be parsed: {:#?}", ts.raw() ) @@ -1450,7 +1449,6 @@ mod tests { input_info: None, input_format: None, dump_index: false, - debug: false, app_dirs: None, tail: 0, delimiter: Delimiter::default(), diff --git a/src/appdirs.rs b/src/appdirs.rs index eb704126..505cdf7f 100644 --- a/src/appdirs.rs +++ b/src/appdirs.rs @@ -4,13 +4,19 @@ use std::path::PathBuf; pub struct AppDirs { pub cache_dir: PathBuf, pub config_dir: PathBuf, + pub system_config_dirs: Vec, } impl AppDirs { pub fn new(name: &str) -> Option { let cache_dir = sys::cache_dir()?.join(name); let config_dir = sys::config_dir()?.join(name); - Some(Self { cache_dir, config_dir }) + let system_config_dirs = sys::system_config_dirs().into_iter().map(|d| d.join(name)).collect(); + Some(Self { + cache_dir, + config_dir, + system_config_dirs, + }) } } @@ -19,16 +25,20 @@ mod sys { use super::*; use std::env; + pub(crate) fn cache_dir() -> Option { + env::var_os("XDG_CACHE_HOME") + .and_then(dirs_sys::is_absolute_path) + .or_else(|| dirs::home_dir().map(|h| h.join(".cache"))) + } + pub(crate) fn config_dir() -> Option { env::var_os("XDG_CONFIG_HOME") .and_then(dirs_sys::is_absolute_path) .or_else(|| dirs::home_dir().map(|h| h.join(".config"))) } - pub(crate) fn cache_dir() -> Option { - env::var_os("XDG_CACHE_HOME") - .and_then(dirs_sys::is_absolute_path) - .or_else(|| dirs::home_dir().map(|h| h.join(".cache"))) + pub(crate) fn system_config_dirs() -> Vec { + vec![PathBuf::from("/etc")] } } @@ -43,4 +53,16 @@ mod sys { pub(crate) fn cache_dir() -> Option { dirs::cache_dir() } + + #[cfg(not(target_os = "windows"))] + pub(crate) fn system_config_dirs() -> Vec { + vec![PathBuf::from("/etc")] + } + + #[cfg(target_os = "windows")] + pub(crate) fn system_config_dirs() -> Vec { + use known_folders::{get_known_folder_path, KnownFolder}; + + get_known_folder_path(KnownFolder::ProgramData).to_vec() + } } diff --git a/src/cli.rs b/src/cli.rs index 8310dbbd..14521502 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -19,14 +19,8 @@ use crate::{ #[derive(Args)] pub struct BootstrapArgs { /// Configuration file path. - #[arg( - long, - overrides_with = "config", - value_name = "FILE", - env = "HL_CONFIG", - num_args = 1 - )] - pub config: Option, + #[arg(long, value_name = "FILE", env = "HL_CONFIG", num_args = 1)] + pub config: Vec, } /// JSON and logfmt log converter to human readable representation. @@ -133,8 +127,8 @@ pub struct Opt { short, long, env = "HL_LEVEL", - overrides_with="level", - ignore_case=true, + overrides_with = "level", + ignore_case = true, value_parser = LevelValueParser, value_enum, help_heading = heading::FILTERING @@ -164,12 +158,12 @@ pub struct Opt { /// Filter messages by field values /// [k=v, k~=v, k~~=v, 'k!=v', 'k!~=v', 'k!~~=v'] /// where ~ does substring match and ~~ does regular expression match. - #[arg(short, long, number_of_values = 1, help_heading = heading::FILTERING)] + #[arg(short, long, num_args = 1, help_heading = heading::FILTERING)] pub filter: Vec, /// Filter using query, accepts expressions from --filter /// and supports '(', ')', 'and', 'or', 'not', 'in', 'contain', 'like', '<', '>', '<=', '>=', etc. - #[arg(short, long, number_of_values = 1, help_heading = heading::FILTERING)] + #[arg(short, long, num_args = 1, help_heading = heading::FILTERING)] pub query: Vec, /// Color output control. @@ -220,7 +214,7 @@ pub struct Opt { #[arg( long, short = 'h', - number_of_values = 1, + num_args = 1, value_name = "KEY", help_heading = heading::OUTPUT )] @@ -405,10 +399,6 @@ pub struct Opt { #[arg(long, requires = "sort", help_heading = heading::ADVANCED)] pub dump_index: bool, - /// Print debug error messages that can help with troubleshooting. - #[arg(long, help_heading = heading::ADVANCED)] - pub debug: bool, - /// Print help. #[arg(long, default_value_t = false, action = ArgAction::SetTrue)] pub help: bool, diff --git a/src/config.rs b/src/config.rs index 90d534a2..19e861a5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,8 @@ // std imports -use std::sync::Mutex; +use std::{ + path::{Path, PathBuf}, + sync::Mutex, +}; // third-party imports use once_cell::sync::Lazy; @@ -8,7 +11,7 @@ use once_cell::sync::Lazy; use crate::{ appdirs::AppDirs, error::Result, - settings::{Settings, SourceFile}, + settings::{Settings, Source, SourceFile}, }; // --- @@ -20,28 +23,79 @@ pub fn default() -> &'static Settings { Default::default() } -/// Load settings from the given file or the default configuration file per platform. -pub fn load(path: Option<&str>) -> Result { - let mut default = None; - let (filename, required) = path.map(|p| (p, true)).unwrap_or_else(|| { - ( - if let Some(dirs) = app_dirs() { - default = Some(dirs.config_dir.join("config").to_string_lossy().to_string()); - default.as_deref().unwrap() - } else { - "" - }, - false, - ) - }); - - if filename.is_empty() { - return Ok(Default::default()); +/// Load settings from the given file. +pub fn at(paths: I) -> Loader +where + I: IntoIterator, + P: AsRef, +{ + Loader::new(paths.into_iter().map(|path| path.as_ref().into()).collect()) +} + +/// Load settings from the default configuration file per platform. +pub fn load() -> Result { + Loader::new(Vec::new()).load() +} + +// --- + +pub struct Loader { + paths: Vec, + no_default: bool, + dirs: Option, +} + +impl Loader { + fn new(paths: Vec) -> Self { + Self { + paths, + no_default: false, + dirs: app_dirs(), + } + } + + pub fn no_default(mut self, val: bool) -> Self { + self.no_default = val; + self + } + + pub fn load(self) -> Result { + if self.no_default { + Settings::load(self.custom()) + } else { + Settings::load(self.system().chain(self.user()).chain(self.custom())) + } } - Settings::load(SourceFile::new(filename).required(required).into()) + fn system(&self) -> impl Iterator { + self.dirs + .as_ref() + .map(|dirs| dirs.system_config_dirs.clone()) + .unwrap_or_default() + .into_iter() + .map(|dir| SourceFile::new(&Self::config(&dir)).required(false).into()) + } + + fn user(&self) -> impl Iterator { + self.dirs + .as_ref() + .map(|dirs| SourceFile::new(&Self::config(&dirs.config_dir)).required(false).into()) + .into_iter() + } + + fn custom<'a>(&'a self) -> impl Iterator + 'a { + self.paths + .iter() + .map(|path| SourceFile::new(path).required(true).into()) + } + + fn config(dir: &Path) -> PathBuf { + dir.join("config") + } } +// --- + /// Get the application platform-specific directories. pub fn app_dirs() -> Option { AppDirs::new(APP_NAME) @@ -74,22 +128,16 @@ mod tests { use maplit::hashmap; - use crate::{level::Level, settings::Settings}; + use crate::level::Level; #[test] fn test_default() { assert_eq!(default().theme, "universal"); } - #[test] - fn test_load_empty_filename() { - let settings = super::load(Some("")).unwrap(); - assert_eq!(settings, Settings::default()); - } - #[test] fn test_load_k8s() { - let settings = super::load(Some("etc/defaults/config-k8s.yaml")).unwrap(); + let settings = super::at(["etc/defaults/config-k8s.yaml"]).load().unwrap(); assert_eq!(settings.fields.predefined.time.0.names, &["ts"]); assert_eq!(settings.fields.predefined.message.0.names, &["msg"]); assert_eq!(settings.fields.predefined.level.variants.len(), 2); @@ -97,7 +145,7 @@ mod tests { #[test] fn test_issue_288() { - let settings = super::load(Some("src/testing/assets/configs/issue-288.yaml")).unwrap(); + let settings = super::at(["src/testing/assets/configs/issue-288.yaml"]).load().unwrap(); assert_eq!(settings.fields.predefined.level.variants.len(), 1); let variant = &settings.fields.predefined.level.variants[0]; assert_eq!(variant.names, vec!["level".to_owned()]); @@ -115,6 +163,6 @@ mod tests { #[test] fn test_load_auto() { - super::load(None).unwrap(); + super::load().unwrap(); } } diff --git a/src/main.rs b/src/main.rs index 008edaa4..d1a132a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use std::{ // third-party imports use chrono::Utc; use clap::{CommandFactory, Parser}; -use env_logger::fmt::TimestampPrecision; +use env_logger::{self as logger, fmt::TimestampPrecision}; use itertools::Itertools; // local imports @@ -29,10 +29,28 @@ use hl::{ Delimiter, {IncludeExcludeKeyFilter, KeyMatchOptions}, }; +const HL_DEBUG_LOG: &str = "HL_DEBUG_LOG"; + // --- fn run() -> Result<()> { - let settings = config::load(cli::BootstrapOpt::parse().args.config.as_deref())?; + let opt = cli::BootstrapOpt::parse().args; + let (offset, no_default_configs) = opt + .config + .iter() + .rposition(|x| x == "" || x == "-") + .map(|x| (x + 1, true)) + .unwrap_or_default(); + let configs = &opt.config[offset..]; + + if std::env::var(HL_DEBUG_LOG).is_ok() { + logger::Builder::from_env(HL_DEBUG_LOG) + .format_timestamp(Some(TimestampPrecision::Micros)) + .init(); + log::debug!("logging initialized"); + } + + let settings = config::at(configs).no_default(no_default_configs).load()?; config::global::initialize(settings.clone()); let opt = cli::Opt::parse_from(wild::args()); @@ -40,13 +58,6 @@ fn run() -> Result<()> { return cli::Opt::command().print_help().map_err(Error::Io); } - if opt.debug { - env_logger::builder() - .format_timestamp(Some(TimestampPrecision::Micros)) - .init(); - log::debug!("logging initialized"); - } - if let Some(shell) = opt.shell_completions { let mut cmd = cli::Opt::command(); let name = cmd.get_name().to_string(); @@ -231,7 +242,6 @@ fn run() -> Result<()> { cli::InputFormat::Logfmt => Some(app::InputFormat::Logfmt), }, dump_index: opt.dump_index, - debug: opt.debug, app_dirs: Some(app_dirs), tail: opt.tail, delimiter, diff --git a/src/settings.rs b/src/settings.rs index 26cd9558..8459d0fa 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1,6 +1,9 @@ // std imports -use std::collections::{BTreeMap, HashMap}; -use std::include_str; +use std::{ + collections::{BTreeMap, HashMap}, + include_str, + path::{Path, PathBuf}, +}; // third-party imports use chrono_tz::Tz; @@ -17,7 +20,7 @@ use crate::level::Level; // --- static DEFAULT_SETTINGS_RAW: &str = include_str!("../etc/defaults/config.yaml"); -static DEFAULT_SETTINGS: Lazy = Lazy::new(|| Settings::load(Source::Str("", FileFormat::Yaml)).unwrap()); +static DEFAULT_SETTINGS: Lazy = Lazy::new(|| Settings::load([Source::string("", FileFormat::Yaml)]).unwrap()); // --- @@ -33,14 +36,26 @@ pub struct Settings { } impl Settings { - pub fn load(source: Source) -> Result { - let builder = Config::builder().add_source(File::from_str(DEFAULT_SETTINGS_RAW, FileFormat::Yaml)); - let builder = match source { - Source::File(SourceFile { filename, required }) => { - builder.add_source(File::with_name(filename).required(required)) - } - Source::Str(value, format) => builder.add_source(File::from_str(value, format)), - }; + pub fn load(sources: I) -> Result + where + I: IntoIterator, + { + let mut builder = Config::builder().add_source(File::from_str(DEFAULT_SETTINGS_RAW, FileFormat::Yaml)); + + for source in sources { + builder = match source { + Source::File(SourceFile { filename, required }) => { + log::debug!( + "added configuration file {} search path: {}", + if required { "required" } else { "optional" }, + filename.display(), + ); + builder.add_source(File::from(filename.as_path()).required(required)) + } + Source::String(value, format) => builder.add_source(File::from_str(&value, format)), + }; + } + Ok(builder.build()?.try_deserialize()?) } } @@ -59,28 +74,40 @@ impl Default for &'static Settings { // --- -pub enum Source<'a> { - File(SourceFile<'a>), - Str(&'a str, FileFormat), +pub enum Source { + File(SourceFile), + String(String, FileFormat), +} + +impl Source { + pub fn string(value: S, format: FileFormat) -> Self + where + S: Into, + { + Self::String(value.into(), format) + } } -impl<'a> From> for Source<'a> { - fn from(file: SourceFile<'a>) -> Self { +impl From for Source { + fn from(file: SourceFile) -> Self { Self::File(file) } } // --- -pub struct SourceFile<'a> { - filename: &'a str, +pub struct SourceFile { + filename: PathBuf, required: bool, } -impl<'a> SourceFile<'a> { - pub fn new(filename: &'a str) -> Self { +impl SourceFile { + pub fn new

(filename: P) -> Self + where + P: AsRef, + { Self { - filename, + filename: filename.as_ref().into(), required: true, } } @@ -371,7 +398,7 @@ mod tests { #[test] fn test_load_settings_k8s() { - let settings = Settings::load(SourceFile::new("etc/defaults/config-k8s.yaml").into()).unwrap(); + let settings = Settings::load([SourceFile::new("etc/defaults/config-k8s.yaml").into()]).unwrap(); assert_eq!( settings.fields.predefined.time, TimeField(Field { diff --git a/src/themecfg.rs b/src/themecfg.rs index bd4527c7..89f5f89b 100644 --- a/src/themecfg.rs +++ b/src/themecfg.rs @@ -543,6 +543,7 @@ mod tests { let app_dirs = AppDirs { config_dir: PathBuf::from("src/testing/assets"), cache_dir: Default::default(), + system_config_dirs: Default::default(), }; assert_ne!(Theme::load(&app_dirs, "test").unwrap().elements.len(), 0); assert_ne!(Theme::load(&app_dirs, "universal").unwrap().elements.len(), 0);