wip: integrate certmanager to rpxy-bin along with existing old rustls

junkurihara · May 28, 2024 · e25c6fa · e25c6fa
1 parent e5bfc2c
commit e25c6fa
Show file tree

Hide file tree

Showing 8 changed files with 170 additions and 61 deletions.
diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml
@@ -26,6 +26,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [
   "sticky-cookie",
 ] }
 
+mimalloc = { version = "*", default-features = false }
 anyhow = "1.0.86"
 rustc-hash = "1.1.0"
 serde = { version = "1.0.202", default-features = false, features = ["derive"] }
@@ -39,7 +40,7 @@ tokio = { version = "1.37.0", default-features = false, features = [
 ] }
 async-trait = "0.1.80"
 rustls-pemfile = "1.0.4"
-mimalloc = { version = "*", default-features = false }
+
 
 # config
 clap = { version = "4.5.4", features = ["std", "cargo", "wrap_help"] }
@@ -50,5 +51,10 @@ hot_reload = "0.1.5"
 tracing = { version = "0.1.40" }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 
+################################
+# cert management
+rpxy-certs = { path = "../rpxy-certs/", default-features = false, features = [
+  "http3",
+] }
 
 [dev-dependencies]
diff --git a/rpxy-bin/src/config/mod.rs b/rpxy-bin/src/config/mod.rs
@@ -4,6 +4,6 @@ mod toml;
 
 pub use {
   self::toml::ConfigToml,
-  parse::{build_settings, parse_opts},
+  parse::{build_cert_manager, build_settings, parse_opts},
   service::ConfigTomlReloader,
 };
diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs
@@ -4,7 +4,10 @@ use crate::{
   error::{anyhow, ensure},
 };
 use clap::{Arg, ArgAction};
+use hot_reload::{ReloaderReceiver, ReloaderService};
+use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase};
 use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig};
+use rustc_hash::FxHashMap as HashMap;
 
 /// Parsed options
 pub struct Opts {
@@ -37,20 +40,13 @@ pub fn parse_opts() -> Result<Opts, anyhow::Error> {
   let config_file_path = matches.get_one::<String>("config_file").unwrap().to_owned();
   let watch = matches.get_one::<bool>("watch").unwrap().to_owned();
 
-  Ok(Opts {
-    config_file_path,
-    watch,
-  })
+  Ok(Opts { config_file_path, watch })
 }
 
-pub fn build_settings(
-  config: &ConfigToml,
-) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
-  ///////////////////////////////////
+pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
   // build proxy config
   let proxy_config: ProxyConfig = config.try_into()?;
 
-  ///////////////////////////////////
   // backend_apps
   let apps = config.apps.clone().ok_or(anyhow!("Missing application spec"))?;
 
@@ -95,3 +91,32 @@ pub fn build_settings(
 
   Ok((proxy_config, app_config_list))
 }
+
+/* ----------------------- */
+/// Build cert map
+pub async fn build_cert_manager(
+  config: &ConfigToml,
+) -> Result<
+  (
+    ReloaderService<CryptoReloader, ServerCryptoBase>,
+    ReloaderReceiver<ServerCryptoBase>,
+  ),
+  anyhow::Error,
+> {
+  let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?;
+  let mut crypto_source_map = HashMap::default();
+  for app in apps.0.values() {
+    if let Some(tls) = app.tls.as_ref() {
+      ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some());
+      let server_name = app.server_name.as_ref().ok_or(anyhow!("No server name"))?;
+      let crypto_file_source = CryptoFileSourceBuilder::default()
+        .tls_cert_path(tls.tls_cert_path.as_ref().unwrap())
+        .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap())
+        .client_ca_cert_path(tls.client_ca_cert_path.as_deref())
+        .build()?;
+      crypto_source_map.insert(server_name.to_owned(), crypto_file_source);
+    }
+  }
+  let res = build_cert_reloader(&crypto_source_map, None).await?;
+  Ok(res)
+}
diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs
@@ -8,8 +8,9 @@ mod error;
 mod log;
 
 use crate::{
-  config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
+  config::{build_cert_manager, build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
   constants::CONFIG_WATCH_DELAY_SECS,
+  error::*,
   log::*,
 };
 use hot_reload::{ReloaderReceiver, ReloaderService};
@@ -36,13 +37,10 @@ fn main() {
         std::process::exit(1);
       }
     } else {
-      let (config_service, config_rx) = ReloaderService::<ConfigTomlReloader, ConfigToml>::new(
-        &parsed_opts.config_file_path,
-        CONFIG_WATCH_DELAY_SECS,
-        false,
-      )
-      .await
-      .unwrap();
+      let (config_service, config_rx) =
+        ReloaderService::<ConfigTomlReloader, ConfigToml>::new(&parsed_opts.config_file_path, CONFIG_WATCH_DELAY_SECS, false)
+          .await
+          .unwrap();
 
       tokio::select! {
         Err(e) = config_service.start() => {
@@ -53,6 +51,9 @@ fn main() {
           error!("rpxy service existed: {e}");
           std::process::exit(1);
         }
+        else => {
+          std::process::exit(0);
+        }
       }
     }
   });
@@ -63,23 +64,22 @@ async fn rpxy_service_without_watcher(
   runtime_handle: tokio::runtime::Handle,
 ) -> Result<(), anyhow::Error> {
   info!("Start rpxy service");
-  let config_toml = match ConfigToml::new(config_file_path) {
-    Ok(v) => v,
-    Err(e) => {
-      error!("Invalid toml file: {e}");
-      std::process::exit(1);
+  let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?;
+  let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
+  let (cert_service, cert_rx) = build_cert_manager(&config_toml)
+    .await
+    .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
+
+  tokio::select! {
+    rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => {
+      error!("rpxy entrypoint exited");
+      rpxy_res.map_err(|e| anyhow!(e))
     }
-  };
-  let (proxy_conf, app_conf) = match build_settings(&config_toml) {
-    Ok(v) => v,
-    Err(e) => {
-      error!("Invalid configuration: {e}");
-      return Err(anyhow::anyhow!(e));
+    cert_res = cert_service.start() => {
+      error!("cert reloader service exited");
+      cert_res.map_err(|e| anyhow!(e))
     }
-  };
-  entrypoint(&proxy_conf, &app_conf, &runtime_handle, None)
-    .await
-    .map_err(|e| anyhow::anyhow!(e))
+  }
 }
 
 async fn rpxy_service_with_watcher(
@@ -89,31 +89,31 @@ async fn rpxy_service_with_watcher(
   info!("Start rpxy service with dynamic config reloader");
   // Initial loading
   config_rx.changed().await?;
-  let config_toml = config_rx.borrow().clone().unwrap();
-  let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) {
-    Ok(v) => v,
-    Err(e) => {
-      error!("Invalid configuration: {e}");
-      return Err(anyhow::anyhow!(e));
-    }
-  };
+  let config_toml = config_rx
+    .borrow()
+    .clone()
+    .ok_or(anyhow!("Something wrong in config reloader receiver"))?;
+  let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
+
+  let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml)
+    .await
+    .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
 
   // Notifier for proxy service termination
   let term_notify = std::sync::Arc::new(tokio::sync::Notify::new());
 
   // Continuous monitoring
   loop {
     tokio::select! {
-      _ = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
+      rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
         error!("rpxy entrypoint exited");
-        break;
+        return rpxy_res.map_err(|e| anyhow!(e));
       }
       _ = config_rx.changed() => {
-        if config_rx.borrow().is_none() {
+        let Some(config_toml) = config_rx.borrow().clone() else {
           error!("Something wrong in config reloader receiver");
-          break;
-        }
-        let config_toml = config_rx.borrow().clone().unwrap();
+          return Err(anyhow!("Something wrong in config reloader receiver"));
+        };
         match build_settings(&config_toml) {
           Ok((p, a)) => {
             (proxy_conf, app_conf) = (p, a)
@@ -123,13 +123,27 @@ async fn rpxy_service_with_watcher(
             continue;
           }
         };
+        match build_cert_manager(&config_toml).await {
+          Ok((c, r)) => {
+            (cert_service, cert_rx) = (c, r)
+          },
+          Err(e) => {
+            error!("Invalid cert configuration. Configuration does not updated: {e}");
+            continue;
+          }
+        };
+
         info!("Configuration updated. Terminate all spawned proxy services and force to re-bind TCP/UDP sockets");
         term_notify.notify_waiters();
         // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
       }
+      cert_res = cert_service.start() => {
+        error!("cert reloader service exited");
+        return cert_res.map_err(|e| anyhow!(e));
+      }
       else => break
     }
   }
 
-  Err(anyhow::anyhow!("rpxy or continuous monitoring service exited"))
+  Ok(())
 }
diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml
@@ -22,6 +22,7 @@ thiserror = { version = "1.0.61" }
 hot_reload = { version = "0.1.5" }
 async-trait = { version = "0.1.80" }
 rustls = { version = "0.23.8", default-features = false, features = [
+  "std",
   "aws_lc_rs",
 ] }
 rustls-pemfile = { version = "2.1.2" }
@@ -33,9 +34,6 @@ x509-parser = { version = "0.16.0" }
 
 [dev-dependencies]
 tokio = { version = "1.37.0", default-features = false, features = [
-  # "net",
   "rt-multi-thread",
-  # "time",
-  # "sync",
   "macros",
 ] }
diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs
@@ -18,4 +18,10 @@ pub enum RpxyCertError {
   /// Error when converting server name bytes to string
   #[error("Failed to convert server name bytes to string: {0}")]
   ServerNameBytesToString(#[from] std::string::FromUtf8Error),
+  /// Rustls error
+  #[error("Rustls error: {0}")]
+  RustlsError(#[from] rustls::Error),
+  /// Rustls CryptoProvider error
+  #[error("Rustls No default CryptoProvider error")]
+  NoDefaultCryptoProvider,
 }
diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs
@@ -9,18 +9,17 @@ mod log {
   pub(super) use tracing::{debug, error, info, warn};
 }
 
-use crate::{
-  error::*,
-  reloader_service::{CryptoReloader, DynCryptoSource},
-};
+use crate::{error::*, log::*, reloader_service::DynCryptoSource};
 use hot_reload::{ReloaderReceiver, ReloaderService};
 use rustc_hash::FxHashMap as HashMap;
+use rustls::crypto::{aws_lc_rs, CryptoProvider};
 use std::sync::Arc;
 
 /* ------------------------------------------------ */
 pub use crate::{
   certs::SingleServerCertsKeys,
   crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource},
+  reloader_service::CryptoReloader,
   server_crypto::{ServerCrypto, ServerCryptoBase},
 };
 
@@ -44,6 +43,10 @@ pub async fn build_cert_reloader<T>(
 where
   T: CryptoSource<Error = RpxyCertError> + Send + Sync + Clone + 'static,
 {
+  info!("Building certificate reloader service");
+  // Install aws_lc_rs as default crypto provider for rustls
+  let _ = CryptoProvider::install_default(aws_lc_rs::default_provider());
+
   let source = crypto_source_map
     .iter()
     .map(|(k, v)| {