logger: add klog functionality for rust servers (#335)

Adds a new logger which has command logger specific functionality.

Adds command logging to pelikan_segcache_rs and 
pelikan_pingserver_rs

Changes to the memcache and ping wire protocols to support
providing the necessary context for command logging.

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "src/rust/config",
     "src/rust/core/server",
     "src/rust/entrystore",
+    "src/rust/logger",
     "src/rust/metrics",
     "src/rust/macros",
     "src/rust/protocol",

config/pingserver.toml

@@ -26,7 +26,27 @@ nevent = 1024
 [buf]
 
 [debug]
+# choose from: error, warn, info, debug, trace
 log_level = "info"
+# optionally, log to the file below instead of standard out
+# log_file = "pingserver.log"
+# backup file name for use with log rotation
+log_backup = "pingserver.log.old"
+# trigger log rotation when the file grows beyond this size (in bytes). Set this
+# option to '0' to disable log rotation.
+log_max_size = 1073741824
+
+[klog]
+# optionally, log commands to the file below
+# file = "pingserver.cmd"
+# backup file name for use with log rotation
+backup = "pingserver.cmd.old"
+# trigger log rotation when the file grows beyond this size (in bytes). Set this
+# option to '0' to disable log rotation.
+max_size = 1073741824
+# specify the sampling ratio, 1 in N commands will be logged. Setting to '0'
+# will disable command logging.
+sample = 100
 
 # NOTE: not currently implemented
 [sockio]

config/segcache.toml

@@ -48,9 +48,34 @@ time_type = "Memcache"
 [debug]
 # choose from: error, warn, info, debug, trace
 log_level = "info"
+# optionally, log to the file below instead of standard out
+# log_file = "segcache.log"
+# backup file name for use with log rotation
+log_backup = "segcache.log.old"
+# trigger log rotation when the file grows beyond this size (in bytes). Set this
+# option to '0' to disable log rotation.
+log_max_size = 1073741824
+
+[klog]
+# optionally, log commands to the file below
+# file = "segcache.cmd"
+# backup file name for use with log rotation
+backup = "segcache.cmd.old"
+# trigger log rotation when the file grows beyond this size (in bytes). Set this
+# option to '0' to disable log rotation.
+max_size = 1073741824
+# specify the sampling ratio, 1 in N commands will be logged. Setting to '0'
+# will disable command logging.
+sample = 100
 
 [sockio]
 
 [tcp]
 
 [tls]
+# certificate chain used to validate client certificate
+# certificate_chain = "client.chain"
+# server certificate
+# certificate = "server.crt"
+# server private key
+# private_key = "server.key"

src/rust/config/src/debug.rs

@@ -2,37 +2,59 @@
 // Licensed under the Apache License, Version 2.0
 // http://www.apache.org/licenses/LICENSE-2.0
 
+use crate::units::*;
 use log::Level;
 use serde::{Deserialize, Serialize};
 
 // constants to define default values
-const DEBUG_LOG_LEVEL: Level = Level::Info;
-const DEBUG_LOG_FILE: Option<String> = None;
-const DEBUG_LOG_NBUF: usize = 0;
+const LOG_LEVEL: Level = Level::Info;
+const LOG_FILE: Option<String> = None;
+const LOG_BACKUP: Option<String> = None;
+const LOG_MAX_SIZE: u64 = GB as u64;
+const LOG_QUEUE_DEPTH: usize = 4096;
+const LOG_SINGLE_MESSAGE_SIZE: usize = KB;
 
 // helper functions
 fn log_level() -> Level {
-    DEBUG_LOG_LEVEL
+    LOG_LEVEL
 }
 
 fn log_file() -> Option<String> {
-    DEBUG_LOG_FILE
+    LOG_FILE
 }
 
-fn log_nbuf() -> usize {
-    DEBUG_LOG_NBUF
+fn log_backup() -> Option<String> {
+    LOG_BACKUP
+}
+
+fn log_max_size() -> u64 {
+    LOG_MAX_SIZE
+}
+
+fn log_queue_depth() -> usize {
+    LOG_QUEUE_DEPTH
+}
+
+fn log_single_message_size() -> usize {
+    LOG_SINGLE_MESSAGE_SIZE
 }
 
 // struct definitions
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct DebugConfig {
     #[serde(with = "LevelDef")]
     #[serde(default = "log_level")]
     log_level: Level,
     #[serde(default = "log_file")]
     log_file: Option<String>,
-    #[serde(default = "log_nbuf")]
-    log_nbuf: usize,
+    #[serde(default = "log_backup")]
+    log_backup: Option<String>,
+    #[serde(default = "log_max_size")]
+    log_max_size: u64,
+    #[serde(default = "log_queue_depth")]
+    log_queue_depth: usize,
+    #[serde(default = "log_single_message_size")]
+    log_single_message_size: usize,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -57,8 +79,23 @@ impl DebugConfig {
         self.log_file.clone()
     }
 
-    pub fn log_nbuf(&self) -> usize {
-        self.log_nbuf
+    pub fn log_backup(&self) -> Option<String> {
+        match &self.log_backup {
+            Some(path) => Some(path.clone()),
+            None => self.log_file.as_ref().map(|path| format!("{}.old", path)),
+        }
+    }
+
+    pub fn log_max_size(&self) -> u64 {
+        self.log_max_size
+    }
+
+    pub fn log_queue_depth(&self) -> usize {
+        self.log_queue_depth
+    }
+
+    pub fn log_single_message_size(&self) -> usize {
+        self.log_single_message_size
     }
 }
 
@@ -68,7 +105,10 @@ impl Default for DebugConfig {
         Self {
             log_level: log_level(),
             log_file: log_file(),
-            log_nbuf: log_nbuf(),
+            log_backup: log_backup(),
+            log_max_size: log_max_size(),
+            log_queue_depth: log_queue_depth(),
+            log_single_message_size: log_single_message_size(),
         }
     }
 }

src/rust/config/src/klog.rs

@@ -0,0 +1,137 @@
+// Copyright 2021 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0
+// http://www.apache.org/licenses/LICENSE-2.0
+
+use crate::units::*;
+use serde::{Deserialize, Serialize};
+
+////////////////////////////////////////////////////////////////////////////////
+// constants to define default values
+////////////////////////////////////////////////////////////////////////////////
+
+// log to the file path
+const FILE: Option<String> = None;
+
+// log will rotate to the given backup path
+const BACKUP: Option<String> = None;
+
+// flush interval in milliseconds
+const INTERVAL: usize = 100;
+
+// max log size before rotate in bytes
+const MAX_SIZE: u64 = GB as u64;
+
+// logger queue depth
+const QUEUE_DEPTH: usize = 4096;
+
+// log 1 in every N commands
+const SAMPLE: usize = 100;
+
+// single message buffer size in bytes
+const SINGLE_MESSAGE_SIZE: usize = KB;
+
+////////////////////////////////////////////////////////////////////////////////
+// helper functions
+////////////////////////////////////////////////////////////////////////////////
+
+fn file() -> Option<String> {
+    FILE
+}
+
+fn backup() -> Option<String> {
+    BACKUP
+}
+
+fn interval() -> usize {
+    INTERVAL
+}
+
+fn max_size() -> u64 {
+    MAX_SIZE
+}
+
+fn queue_depth() -> usize {
+    QUEUE_DEPTH
+}
+
+fn sample() -> usize {
+    SAMPLE
+}
+
+fn single_message_size() -> usize {
+    SINGLE_MESSAGE_SIZE
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// struct definitions
+////////////////////////////////////////////////////////////////////////////////
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct KlogConfig {
+    #[serde(default = "backup")]
+    backup: Option<String>,
+    #[serde(default = "file")]
+    file: Option<String>,
+    #[serde(default = "interval")]
+    interval: usize,
+    #[serde(default = "max_size")]
+    max_size: u64,
+    #[serde(default = "queue_depth")]
+    queue_depth: usize,
+    #[serde(default = "sample")]
+    sample: usize,
+    #[serde(default = "single_message_size")]
+    single_message_size: usize,
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// implementation
+////////////////////////////////////////////////////////////////////////////////
+
+impl KlogConfig {
+    pub fn file(&self) -> Option<String> {
+        self.file.clone()
+    }
+
+    pub fn backup(&self) -> Option<String> {
+        match &self.backup {
+            Some(path) => Some(path.clone()),
+            None => self.file.as_ref().map(|path| format!("{}.old", path)),
+        }
+    }
+
+    pub fn interval(&self) -> usize {
+        self.interval
+    }
+
+    pub fn max_size(&self) -> u64 {
+        self.max_size
+    }
+
+    pub fn queue_depth(&self) -> usize {
+        self.queue_depth
+    }
+
+    pub fn sample(&self) -> usize {
+        self.sample
+    }
+
+    pub fn single_message_size(&self) -> usize {
+        self.single_message_size
+    }
+}
+
+// trait implementations
+impl Default for KlogConfig {
+    fn default() -> Self {
+        Self {
+            file: file(),
+            backup: backup(),
+            interval: interval(),
+            max_size: max_size(),
+            queue_depth: queue_depth(),
+            sample: sample(),
+            single_message_size: single_message_size(),
+        }
+    }
+}