Merge branch 'main' into small-refactor-settlement

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable"]

Cargo.toml

@@ -47,7 +47,7 @@ tempfile = "3.10.1"
 time = { version = "0.3.36", features = ["macros"] }
 thiserror = "1.0.61"
 toml = "0.8.14"
-tokio = "1.38.0"
+tokio = { version = "1.38.0", features = ["tracing"] }
 tokio-stream = { version = "0.1.15", features = ["sync"] }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"

README.md

@@ -117,6 +117,27 @@ ANVIL_IP_ADDR=0.0.0.0 anvil \
   --timestamp 1577836800
 ```
 
+### Profiling
+
+The most important binaries support [tokio-console](https://github.com/tokio-rs/console) to allow you a could look inside the tokio runtime.
+
+Simply enable the feature by passing `--enable-tokio-console true` when running a binary and then in another shell, run
+
+```
+cargo install --locked tokio-console
+tokio-console
+```
+
+
+### Changing Log Filters
+
+It's possible to change the tracing log filter while the process is running. This can be useful to debug an error that requires more verbose logs but which might no longer appear after restarting the system.
+
+Each process opens a UNIX socket at `/tmp/log_filter_override_<program_name>_<pid>.sock`. To change the log filter connect to it with `nc -U <path>` and enter a new log filter.
+You can also reset the log filter to the filter the program was initially started with by entering `reset`.
+
+See [here](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives) for documentation on the supported log filter format.
+
 ## Running the Services Locally
 
 ### Prerequisites

crates/autopilot/src/run_loop.rs

@@ -20,6 +20,7 @@ use {
     ::observe::metrics,
     anyhow::Result,
     database::order_events::OrderEventLabel,
+    itertools::Itertools,
     model::solver_competition::{
         CompetitionAuction,
         Order,
@@ -130,6 +131,14 @@ impl RunLoop {
         };
         let competition_simulation_block = self.eth.current_block().borrow().number;
 
+        let considered_orders = solutions
+            .iter()
+            .flat_map(|solution| solution.solution.order_ids().copied())
+            .unique()
+            .collect();
+        self.persistence
+            .store_order_events(considered_orders, OrderEventLabel::Considered);
+
         // TODO: Keep going with other solutions until some deadline.
         if let Some(Participant { driver, solution }) = solutions.last() {
             tracing::info!(driver = %driver.name, solution = %solution.id(), "winner");
@@ -146,10 +155,6 @@ impl RunLoop {
                 }
             };
 
-            let order_uids = solution.order_ids().copied().collect();
-            self.persistence
-                .store_order_events(order_uids, OrderEventLabel::Considered);
-
             let winner = solution.solver().into();
             let winning_score = solution.score().get().0;
             let reference_score = solutions

crates/observe/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2021"
 license = "MIT OR Apache-2.0"
 
 [dependencies]
-atty = "0.2.14"
+atty = "0.2"
+console-subscriber = "0.3.0"
 futures = { workspace = true }
 once_cell = { workspace = true }
 pin-project-lite = "0.2.14"

crates/observe/build.rs

@@ -0,0 +1,4 @@
+fn main() {
+    // Make build system aware of custom config flags to avoid clippy warnings
+    println!("cargo::rustc-check-cfg=cfg(tokio_unstable)");
+}

crates/observe/src/lib.rs

@@ -6,3 +6,6 @@ pub mod metrics;
 pub mod panic_hook;
 pub mod request_id;
 pub mod tracing;
+
+#[cfg(unix)]
+mod tracing_reload_handler;