From 15359ac027de2a2f97461fd867fdd9d1d8f10f15 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 3 Dec 2024 17:31:08 +0100 Subject: [PATCH 01/72] E2E IPv6 steal test --- tests/src/traffic/steal.rs | 34 +++++++++++++++++++++++++++++++++- tests/src/utils.rs | 12 +++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 518aa0bc13e..188c951711c 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -25,7 +25,6 @@ mod steal_tests { }; #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] - #[cfg(target_os = "linux")] #[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[timeout(Duration::from_secs(240))] @@ -63,6 +62,39 @@ mod steal_tests { application.assert(&process).await; } + #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] + #[rstest] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[timeout(Duration::from_secs(240))] + async fn steal_http_ipv6_traffic( + #[future] service: KubeService, + #[future] kube_client: Client, + ) { + let application = Application::PythonFastApiHTTPIPv6; + let service = service.await; + let kube_client = kube_client.await; + let url = get_service_url(kube_client.clone(), &service).await; + let mut flags = vec!["--steal"]; + + if cfg!(feature = "ephemeral") { + flags.extend(["-e"].into_iter()); + } + + let mut process = application + .run(&service.target, Some(&service.namespace), Some(flags), None) + .await; + + process + .wait_for_line(Duration::from_secs(40), "daemon subscribed") + .await; + send_requests(&url, true, Default::default()).await; + tokio::time::timeout(Duration::from_secs(40), process.wait()) + .await + .unwrap(); + + application.assert(&process).await; + } + #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] #[cfg(target_os = "linux")] #[rstest] diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 538a4d753e6..3717f8c371f 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -90,6 +90,7 @@ fn format_time() -> String { pub enum Application { PythonFlaskHTTP, PythonFastApiHTTP, + PythonFastApiHTTPIPv6, NodeHTTP, NodeHTTP2, Go21HTTP, @@ -399,6 +400,15 @@ impl Application { "app_fastapi:app", ] } + Application::PythonFastApiHTTPIPv6 => { + vec![ + "uvicorn", + "--port=80", + "--host=::", + "--app-dir=./python-e2e/", + "app_fastapi:app", + ] + } Application::PythonCloseSocket => { vec!["python3", "-u", "python-e2e/close_socket.py"] } @@ -447,7 +457,7 @@ impl Application { } pub async fn assert(&self, process: &TestProcess) { - if let Application::PythonFastApiHTTP = self { + if matches!(self, Self::PythonFastApiHTTP | Self::PythonFastApiHTTPIPv6) { process.assert_log_level(true, "ERROR").await; process.assert_log_level(false, "ERROR").await; process.assert_log_level(true, "CRITICAL").await; From af5db7c3a6651ea454163df6d0e4e97cb3fd3b57 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 4 Dec 2024 14:54:43 +0100 Subject: [PATCH 02/72] e2e ipv6 service --- tests/src/traffic/steal.rs | 10 +++++----- tests/src/utils.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 188c951711c..99b30c4d724 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -19,9 +19,9 @@ mod steal_tests { }; use crate::utils::{ - config_dir, get_service_host_and_port, get_service_url, http2_service, kube_client, - send_request, send_requests, service, tcp_echo_service, websocket_service, Application, - KubeService, + config_dir, get_service_host_and_port, get_service_url, http2_service, ipv6_service, + kube_client, send_request, send_requests, service, tcp_echo_service, websocket_service, + Application, KubeService, }; #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] @@ -67,11 +67,11 @@ mod steal_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[timeout(Duration::from_secs(240))] async fn steal_http_ipv6_traffic( - #[future] service: KubeService, + #[future] ipv6_service: KubeService, #[future] kube_client: Client, ) { let application = Application::PythonFastApiHTTPIPv6; - let service = service.await; + let service = ipv6_service.await; let kube_client = kube_client.await; let url = get_service_url(kube_client.clone(), &service).await; let mut flags = vec!["--steal"]; diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 3717f8c371f..e4e8dca87f8 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -1587,6 +1587,37 @@ pub async fn random_namespace_self_deleting_service(#[future] kube_client: Clien .await } +/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted +/// when the returned service is dropped, unless it is dropped during panic. +/// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. +/// * `randomize_name` - whether a random suffix should be added to the end of the resource names +#[fixture] +pub async fn ipv6_service( + #[default("default")] namespace: &str, + #[default("NodePort")] service_type: &str, + // #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, + #[default("docker.io/t4lz/mirrord-pytest:1204")] image: &str, + #[default("http-echo")] service_name: &str, + #[default(true)] randomize_name: bool, + #[future] kube_client: Client, +) -> KubeService { + service_with_env( + namespace, + service_type, + image, + service_name, + randomize_name, + kube_client.await, + serde_json::json!([ + { + "name": "HOST", + "value": "::" + } + ]), + ) + .await +} + pub fn resolve_node_host() -> String { if (cfg!(target_os = "linux") && !wsl::is_wsl()) || std::env::var("USE_MINIKUBE").is_ok() { let output = std::process::Command::new("minikube") From d0c384b4519635c9041d41a55aa88c1704924c7f Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 6 Dec 2024 23:11:34 +0100 Subject: [PATCH 03/72] Local E2E IPv6 testing --- CONTRIBUTING.md | 222 ++++++++++++++++++++++++++++----------------- tests/src/lib.rs | 1 + tests/src/utils.rs | 39 ++++++-- 3 files changed, 171 insertions(+), 91 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 818a39e3a89..33e0f0ce262 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,8 @@ # Contributing Before submitting pull request features, please discuss them with us first by opening an issue or a discussion. -We welcome new/junior/starting developers. Feel free to join to our [Discord channel](https://discord.gg/metalbear) for help and guidance. +We welcome new/junior/starting developers. Feel free to join to our [Discord channel](https://discord.gg/metalbear) for +help and guidance. If you would like to start working on an issue, please comment on the issue on GitHub, so that we can assign you to that issue. @@ -11,10 +12,10 @@ Make sure to take a look at the project's [style guide](STYLE.md). # Contents - [Contents](#contents) - - [Getting Started](#getting-started) - - [Debugging mirrord](#debugging-mirrord) - - [New Hook Guidelines](#new-hook-guidelines) - - [Compiling on MacOs](#compliling-on-macos) + - [Getting Started](#getting-started) + - [Debugging mirrord](#debugging-mirrord) + - [New Hook Guidelines](#new-hook-guidelines) + - [Compiling on MacOs](#compliling-on-macos) # Getting Started @@ -31,7 +32,8 @@ The following guide details the steps to setup a local development environment f ### Setup a Kubernetes cluster -For E2E tests and testing mirrord manually you will need a working Kubernetes cluster. A minimal cluster can be easily setup locally using either of the following: +For E2E tests and testing mirrord manually you will need a working Kubernetes cluster. A minimal cluster can be easily +setup locally using either of the following: - [Minikube](https://minikube.sigs.k8s.io/) - [Docker Desktop](https://www.docker.com/products/docker-desktop/) @@ -51,11 +53,13 @@ minikube start --driver=docker ### Prepare a cluster - Build mirrord-agent Docker Image. - -Make sure you're [logged in to GHCR](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). +Build mirrord-agent Docker Image. + +Make sure +you're [logged in to GHCR](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Then run: + ```bash docker buildx build -t test . --file mirrord/agent/Dockerfile ``` @@ -87,22 +91,25 @@ kubectl config use-context minikube ## E2E Tests -The E2E tests create Kubernetes resources in the cluster that kubectl is configured to use and then run sample apps -with the mirrord CLI. The mirrord CLI spawns an agent for the target on the cluster, and runs the test app, with the -layer injected into it. Some test apps need to be compiled before they can be used in the tests +The E2E tests create Kubernetes resources in the cluster that kubectl is configured to use and then run sample apps +with the mirrord CLI. The mirrord CLI spawns an agent for the target on the cluster, and runs the test app, with the +layer injected into it. Some test apps need to be compiled before they can be used in the tests ([this should be automated in the future](https://github.com/metalbear-co/mirrord/issues/982)). The basic command to run the E2E tests is: + ```bash cargo test --package tests ``` However, when running on macOS a universal binary has to be created first: + ```bash scripts/build_fat_mac.sh ``` And then in order to use that binary in the tests, run the tests like this: + ```bash MIRRORD_TESTS_USE_BINARY=../target/universal-apple-darwin/debug/mirrord cargo test -p tests ``` @@ -111,11 +118,32 @@ If new tests are added, decorate them with `cfg_attr` attribute macro to define For example, a test which only tests sanity of the ephemeral container feature should be decorated with `#[cfg_attr(not(feature = "ephemeral"), ignore)]` -On Linux, running tests may exhaust a large amount of RAM and crash the machine. To prevent this, limit the number of concurrent jobs by running the command with e.g. `-j 4` +On Linux, running tests may exhaust a large amount of RAM and crash the machine. To prevent this, limit the number of +concurrent jobs by running the command with e.g. `-j 4` + +### IPv6 + +Some tests create a single-stack IPv6 service. They can only be run on clusters with IPv6 enabled. +In order to test IPv6 on a local cluster on macOS, you can use Kind: + +1. `brew install kind` +2. ```shell + cat >kind-config.yaml <sample/node/app.mjs ```js -import { Buffer } from "node:buffer"; -import { createServer } from "net"; -import { open, readFile } from "fs/promises"; +import {Buffer} from "node:buffer"; +import {createServer} from "net"; +import {open, readFile} from "fs/promises"; async function debug_file_ops() { - try { - const readOnlyFile = await open("/var/log/dpkg.log", "r"); - console.log(">>>>> open readOnlyFile ", readOnlyFile); + try { + const readOnlyFile = await open("/var/log/dpkg.log", "r"); + console.log(">>>>> open readOnlyFile ", readOnlyFile); - let buffer = Buffer.alloc(128); - let bufferResult = await readOnlyFile.read(buffer); - console.log(">>>>> read readOnlyFile returned with ", bufferResult); + let buffer = Buffer.alloc(128); + let bufferResult = await readOnlyFile.read(buffer); + console.log(">>>>> read readOnlyFile returned with ", bufferResult); - const sampleFile = await open("/tmp/node_sample.txt", "w+"); - console.log(">>>>> open file ", sampleFile); + const sampleFile = await open("/tmp/node_sample.txt", "w+"); + console.log(">>>>> open file ", sampleFile); - const written = await sampleFile.write("mirrord sample node"); - console.log(">>>>> written ", written, " bytes to file ", sampleFile); + const written = await sampleFile.write("mirrord sample node"); + console.log(">>>>> written ", written, " bytes to file ", sampleFile); - let sampleBuffer = Buffer.alloc(32); - let sampleBufferResult = await sampleFile.read(buffer); - console.log(">>>>> read ", sampleBufferResult, " bytes from ", sampleFile); + let sampleBuffer = Buffer.alloc(32); + let sampleBufferResult = await sampleFile.read(buffer); + console.log(">>>>> read ", sampleBufferResult, " bytes from ", sampleFile); - readOnlyFile.close(); - sampleFile.close(); - } catch (fail) { - console.error("!!! Failed file operation with ", fail); - } + readOnlyFile.close(); + sampleFile.close(); + } catch (fail) { + console.error("!!! Failed file operation with ", fail); + } } // debug_file_ops(); @@ -304,32 +338,34 @@ async function debug_file_ops() { const server = createServer(); server.on("connection", handleConnection); server.listen( - { - host: "localhost", - port: 80, - }, - function () { - console.log("server listening to %j", server.address()); - } + { + host: "localhost", + port: 80, + }, + function () { + console.log("server listening to %j", server.address()); + } ); function handleConnection(conn) { - var remoteAddress = conn.remoteAddress + ":" + conn.remotePort; - console.log("new client connection from %s", remoteAddress); - conn.on("data", onConnData); - conn.once("close", onConnClose); - conn.on("error", onConnError); - - function onConnData(d) { - console.log("connection data from %s: %j", remoteAddress, d.toString()); - conn.write(d); - } - function onConnClose() { - console.log("connection from %s closed", remoteAddress); - } - function onConnError(err) { - console.log("Connection %s error: %s", remoteAddress, err.message); - } + var remoteAddress = conn.remoteAddress + ":" + conn.remotePort; + console.log("new client connection from %s", remoteAddress); + conn.on("data", onConnData); + conn.once("close", onConnClose); + conn.on("error", onConnError); + + function onConnData(d) { + console.log("connection data from %s: %j", remoteAddress, d.toString()); + conn.write(d); + } + + function onConnClose() { + console.log("connection from %s closed", remoteAddress); + } + + function onConnError(err) { + console.log("Connection %s error: %s", remoteAddress, err.message); + } } ``` @@ -339,8 +375,8 @@ function handleConnection(conn) { ```bash RUST_LOG=debug target/debug/mirrord exec -i test -l debug -c --target pod/py-serv-deployment-ff89b5974-x9tjx node sample/node/app.mjs ``` -> **Note:** You need to change the pod name here to the name of the pod created on your system. +> **Note:** You need to change the pod name here to the name of the pod created on your system. ``` . @@ -403,9 +439,11 @@ OK - GET: Request completed ## mirrord console -Debugging mirrord can get hard since we're running from another app flow, so the fact we're debugging might affect the program and make it unusable/buggy (due to sharing stdout with scripts/other applications). +Debugging mirrord can get hard since we're running from another app flow, so the fact we're debugging might affect the +program and make it unusable/buggy (due to sharing stdout with scripts/other applications). -The recommended way to do it is to use `mirrord-console`. It is a small application that receives log information from different mirrord instances and prints it, controlled via `RUST_LOG` environment variable. +The recommended way to do it is to use `mirrord-console`. It is a small application that receives log information from +different mirrord instances and prints it, controlled via `RUST_LOG` environment variable. To use mirrord console, run it: `cargo run --bin mirrord-console --features binary` @@ -423,24 +461,29 @@ To debug it with a debugger: ```Rust tokio::time::sleep(Duration::from_secs(20)).await; ``` - to [somewhere](https://github.com/metalbear-co/mirrord/blob/fa2af7f1e77a9254fb0908be40b0dae5da53d298/mirrord/cli/src/internal_proxy.rs#L145) in the start of the intproxy code. + to [somewhere](https://github.com/metalbear-co/mirrord/blob/fa2af7f1e77a9254fb0908be40b0dae5da53d298/mirrord/cli/src/internal_proxy.rs#L145) + in the start of the intproxy code. 2. Set breakpoints in vscode in the relevant lines of the intproxy code. 3. Build mirrord. 4. Run mirrord. -5. Attach a debugger in vscode to the inproxy process. On macOS you can do that with `Cmd` + `Shift` + `P` -> `LLDB: Attach to Process...` -> type `intproxy` and choose the `mirrord intproxy` process. The sleep you added at the start of the intproxy is time for you to attach the debugger. +5. Attach a debugger in vscode to the inproxy process. On macOS you can do that with `Cmd` + `Shift` + `P` -> + `LLDB: Attach to Process...` -> type `intproxy` and choose the `mirrord intproxy` process. The sleep you added at the + start of the intproxy is time for you to attach the debugger. ## Retrieving Agent Logs -By default, the agent's pod will complete and disappear shortly after the agent exits. In order to be able to retrieve +By default, the agent's pod will complete and disappear shortly after the agent exits. In order to be able to retrieve the agent's logs after it crashes, set the agent's pod's TTL to a comfortable number of seconds. This configuration can be specified either as a command line argument (`--agent-ttl`), environment variable (`MIRRORD_AGENT_TTL`), or in a configuration file: + ```toml [agent] ttl = 30 ``` Then, when running with some reasonable TTL, you can retrieve the agent log like this: + ```bash kubectl logs -l app=mirrord --tail=-1 | less -R ``` @@ -448,9 +491,11 @@ kubectl logs -l app=mirrord --tail=-1 | less -R This will retrieve the logs from all running mirrord agents, so it is only useful when just one agent pod exists. If there are currently multiple agent pods running on your cluster, you would have to run + ```bash kubectl get pods ``` + and find the name of the agent pod you're interested in, then run ```bash @@ -461,36 +506,48 @@ where you would replace `` with the name of the pod. # New Hook Guidelines -Adding a feature to mirrord that introduces a new hook (file system, network) can be tricky and there are a lot of edge cases we might need to cover. +Adding a feature to mirrord that introduces a new hook (file system, network) can be tricky and there are a lot of edge +cases we might need to cover. In order to have a more structured approach, here’s the flow you should follow when working on such a feature. -1. Start with the use case. Write an example use case of the feature, for example “App needs to read credentials from a file”. -2. Write a minimal app that implements the use case - so in the case of our example, an app that reads credentials from a file. Start with either Node or Python as those are most common. -3. Figure out what functions need to be hooked in order for the behavior to be run through the mirrord-agent instead of locally. We suggest using `strace`. +1. Start with the use case. Write an example use case of the feature, for example “App needs to read credentials from a + file”. +2. Write a minimal app that implements the use case - so in the case of our example, an app that reads credentials from + a file. Start with either Node or Python as those are most common. +3. Figure out what functions need to be hooked in order for the behavior to be run through the mirrord-agent instead of + locally. We suggest using `strace`. 4. Write a doc on how you would hook and handle the cases, for example: 1. To implement use case “App needs to read credentials from a file*” 2. I will hook `open` and `read` handling calls only with flags O_RDONLY. - 3. Once `open` is called, I’ll send a blocking request to the agent to open the remote file, returning the return code of the operation. - 4. Create an fd using `memfd`. The result will be returned to the local app, and if successful we’ll save that fd into a HashMap that matches between local fd and remote fd/identifier. - 5. When `read` is called, I will check if the fd being read was previously opened by us, and if it is we’ll send a blocking `read` request to the agent. The result will be sent back to the caller. + 3. Once `open` is called, I’ll send a blocking request to the agent to open the remote file, returning the return + code of the operation. + 4. Create an fd using `memfd`. The result will be returned to the local app, and if successful we’ll save that fd + into a HashMap that matches between local fd and remote fd/identifier. + 5. When `read` is called, I will check if the fd being read was previously opened by us, and if it is we’ll send a + blocking `read` request to the agent. The result will be sent back to the caller. 6. And so on. 5. This doc should go later on to our mirrord docs for advanced developers so people can understand how stuff works 6. After approval of the implementation, you can start writing code, and add relevant e2e tests. # Compliling on MacOS -The `mirrord-agent` crate makes use of the `#[cfg(target_os = "linux")]` attribute to allow the whole repo to compile on MacOS when you run `cargo build`. +The `mirrord-agent` crate makes use of the `#[cfg(target_os = "linux")]` attribute to allow the whole repo to compile on +MacOS when you run `cargo build`. To enable `mirrord-agent` code analysis with rust-analyzer: + 1. Install additional targets + ```sh rustup target add x86_64-unknown-linux-gnu rustup target add aarch64-apple-darwin rustup target add x86_64-apple-darwin rustup target add aarch64-unknown-linux-gnu ``` + 2. Add additional targets to your local `.cargo/config.toml` block: + ```toml [build] target = [ @@ -502,14 +559,15 @@ target = [ ``` If you're using rust-analyzer VSCode extension, put this block in `.vscode/settings.json` as well: + ```json { - "rust-analyzer.check.targets": [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu" - ] + "rust-analyzer.check.targets": [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu" + ] } ``` diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 98d801f5d23..6e9663bc138 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,4 +1,5 @@ #![feature(stmt_expr_attributes)] +#![feature(ip)] #![warn(clippy::indexing_slicing)] #[cfg(feature = "cli")] diff --git a/tests/src/utils.rs b/tests/src/utils.rs index e4e8dca87f8..538a8978e83 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -4,7 +4,7 @@ use std::{ collections::HashMap, fmt::Debug, - net::Ipv4Addr, + net::IpAddr, ops::Not, path::PathBuf, process::{ExitStatus, Stdio}, @@ -827,6 +827,17 @@ fn deployment_from_json(name: &str, image: &str, env: Value) -> Deployment { .expect("Failed creating `deployment` from json spec!") } +/// Change the `ipFamilies` and `ipFamilyPolicy` fields to make the service IPv6-only. +/// +/// # Panics +/// +/// Will panic if the given service does not have a spec. +fn set_ipv6_only(service: &mut Service) { + let spec = service.spec.as_mut().unwrap(); + spec.ip_families = Some(vec!["IPv6".to_string()]); + spec.ip_family_policy = Some("SingleStack".to_string()); +} + fn service_from_json(name: &str, service_type: &str) -> Service { serde_json::from_value(json!({ "apiVersion": "v1", @@ -1075,6 +1086,7 @@ pub async fn service( randomize_name, kube_client.await, default_env(), + false, ) .await } @@ -1103,6 +1115,7 @@ pub async fn service_with_env( randomize_name, kube_client, env, + false, ) .await } @@ -1126,6 +1139,7 @@ async fn internal_service( randomize_name: bool, kube_client: Client, env: Value, + ipv6_only: bool, ) -> KubeService { let delete_after_fail = std::env::var_os(PRESERVE_FAILED_ENV_NAME).is_none(); @@ -1192,7 +1206,10 @@ async fn internal_service( watch_resource_exists(&deployment_api, &name).await; // `Service` - let service = service_from_json(&name, service_type); + let mut service = service_from_json(&name, service_type); + if ipv6_only { + set_ipv6_only(&mut service); + } let service_guard = ResourceGuard::create( service_api.clone(), name.clone(), @@ -1601,7 +1618,7 @@ pub async fn ipv6_service( #[default(true)] randomize_name: bool, #[future] kube_client: Client, ) -> KubeService { - service_with_env( + internal_service( namespace, service_type, image, @@ -1614,6 +1631,7 @@ pub async fn ipv6_service( "value": "::" } ]), + true, ) .await } @@ -1648,12 +1666,15 @@ async fn get_pod_or_node_host(kube_client: Client, name: &str, namespace: &str) .next() .and_then(|pod| pod.status) .and_then(|status| status.host_ip) - .and_then(|ip| { - ip.parse::() - .unwrap() - .is_private() - .not() - .then_some(ip) + .filter(|ip| { + // use this IP only if it's a public one. + match ip.parse::().unwrap() { + IpAddr::V4(ip4) => ip4.is_private(), + IpAddr::V6(ip6) => { + ip6.is_unicast_link_local() || ip6.is_unique_local() || ip6.is_loopback() + } + } + .not() }) .unwrap_or_else(resolve_node_host) } From 48094bdc1f0b2d901a900d11e5c761e1f85fdbd9 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 9 Dec 2024 15:32:45 +0100 Subject: [PATCH 04/72] No ephemeral, need to delete or uncomment later --- tests/src/traffic/steal.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 99b30c4d724..d8687b59ee1 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -76,9 +76,9 @@ mod steal_tests { let url = get_service_url(kube_client.clone(), &service).await; let mut flags = vec!["--steal"]; - if cfg!(feature = "ephemeral") { - flags.extend(["-e"].into_iter()); - } + // if cfg!(feature = "ephemeral") { + // flags.extend(["-e"].into_iter()); + // } let mut process = application .run(&service.target, Some(&service.namespace), Some(flags), None) From 6cec6b2d7df9bedf472ac0bfd42e70b1d4519d09 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 10 Dec 2024 12:52:19 +0100 Subject: [PATCH 05/72] For local testing. DROP --- tests/src/utils.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 538a8978e83..3287e55020e 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -592,7 +592,10 @@ pub async fn run_exec( // docker build -t test . -f mirrord/agent/Dockerfile // minikube load image test:latest let mut base_env = HashMap::new(); - base_env.insert("MIRRORD_AGENT_IMAGE", "test"); + // TODO: revert + // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); + base_env.insert("MIRRORD_AGENT_IMAGE", "docker.io/t4lz/mirrord-agent:latest"); + base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From f2343f2a43a5091e3b1ff515cc7cd4bfc009e53c Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 10 Dec 2024 15:40:28 +0100 Subject: [PATCH 06/72] add ipv6 flag --- mirrord/config/src/config/from_env.rs | 4 ++ .../config/src/feature/network/incoming.rs | 43 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/mirrord/config/src/config/from_env.rs b/mirrord/config/src/config/from_env.rs index 9770456721a..0f87ef59034 100644 --- a/mirrord/config/src/config/from_env.rs +++ b/mirrord/config/src/config/from_env.rs @@ -20,6 +20,10 @@ where { type Value = T; + /// Returns: + /// - `None` if there is no env var with that name. + /// - `Some(Err(ConfigError::InvalidValue{...}))` if the value of the env var cannot be parsed. + /// - `Some(Ok(...))` if the env var exists and was parsed successfully. fn source_value(self, _context: &mut ConfigContext) -> Option> { std::env::var(self.0).ok().map(|var| { var.parse::() diff --git a/mirrord/config/src/feature/network/incoming.rs b/mirrord/config/src/feature/network/incoming.rs index d56199e003e..a0906c5a5f1 100644 --- a/mirrord/config/src/feature/network/incoming.rs +++ b/mirrord/config/src/feature/network/incoming.rs @@ -18,6 +18,8 @@ pub mod http_filter; use http_filter::*; +const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; + /// ## incoming (network) /// /// Controls the incoming TCP traffic feature. @@ -58,8 +60,9 @@ use http_filter::*; /// }, /// "port_mapping": [[ 7777, 8888 ]], /// "ignore_localhost": false, -/// "ignore_ports": [9999, 10000] -/// "listen_ports": [[80, 8111]] +/// "ignore_ports": [9999, 10000], +/// "listen_ports": [[80, 8111]], +/// "ipv6": false /// } /// } /// } @@ -96,12 +99,14 @@ impl MirrordConfig for IncomingFileConfig { .unwrap_or_default(), http_filter: HttpFilterFileConfig::default().generate_config(context)?, on_concurrent_steal: FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") - .layer(|layer| { - Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer) - }) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(), + ipv6: FromEnv::new(IPV6_ENV_VAR) + .source_value(context) + .transpose()? // error on invalid env var value + .unwrap_or_default(), // if not set either by env or file - set to false. ..Default::default() }, IncomingFileConfig::Advanced(advanced) => IncomingConfig { @@ -129,13 +134,16 @@ impl MirrordConfig for IncomingFileConfig { .unwrap_or_default(), on_concurrent_steal: FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") .or(advanced.on_concurrent_steal) - .layer(|layer| { - Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer) - }) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(), ports: advanced.ports.map(|ports| ports.into_iter().collect()), + ipv6: FromEnv::new(IPV6_ENV_VAR) + .source_value(context) + .transpose()? // error on invalid env var value + .or(advanced.ipv6) // only use file if env var not set. + .unwrap_or_default(), // if not set either by env or file - set to false. }, }; @@ -148,8 +156,13 @@ impl MirrordToggleableConfig for IncomingFileConfig { .source_value(context) .unwrap_or_else(|| Ok(IncomingMode::Off))?; + let ipv6 = FromEnv::new(IPV6_ENV_VAR) + .source_value(context) + .transpose()? + .unwrap_or_default(); + let on_concurrent_steal = FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") - .layer(|layer| Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer)) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(); @@ -158,6 +171,7 @@ impl MirrordToggleableConfig for IncomingFileConfig { mode, on_concurrent_steal, http_filter: HttpFilterFileConfig::disabled_config(context)?, + ipv6, ..Default::default() }) } @@ -308,6 +322,11 @@ pub struct IncomingAdvancedFileConfig { /// /// Mutually exclusive with [`ignore_ports`](###ignore_ports). pub ports: Option>, + + /// ### ipv6 + /// + /// Enable ipv6 support. Turn on if your applications listens to incoming traffic over IPv6. + pub ipv6: Option, } fn serialize_bi_map(map: &BiMap, serializer: S) -> Result @@ -451,6 +470,11 @@ pub struct IncomingConfig { /// Mutually exclusive with /// [`feature.network.incoming.ignore_ports`](#feature-network-ignore_ports). pub ports: Option>, + + /// #### feature.network.incoming.ipv6 {#feature-network-incoming-ipv6} + /// + /// Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. + pub ipv6: bool, } impl IncomingConfig { @@ -615,5 +639,6 @@ impl CollectAnalytics for &IncomingConfig { analytics.add("ignore_localhost", self.ignore_localhost); analytics.add("ignore_ports_count", self.ignore_ports.len()); analytics.add("http", &self.http_filter); + analytics.add("ipv6", self.ipv6); } } From 7442ecfc9216862a6b3e8352a182fa8937211754 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 10 Dec 2024 18:54:05 +0100 Subject: [PATCH 07/72] allow IPv6 in socket if enabled in config --- mirrord/layer/src/socket/ops.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirrord/layer/src/socket/ops.rs b/mirrord/layer/src/socket/ops.rs index 543a512629c..4ff4062f9b0 100644 --- a/mirrord/layer/src/socket/ops.rs +++ b/mirrord/layer/src/socket/ops.rs @@ -4,6 +4,7 @@ use std::{ collections::HashMap, io, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}, + ops::Not, os::{ fd::{BorrowedFd, FromRawFd, IntoRawFd}, unix::io::RawFd, @@ -130,7 +131,9 @@ pub(super) fn socket(domain: c_int, type_: c_int, protocol: c_int) -> Detour Date: Tue, 10 Dec 2024 18:54:23 +0100 Subject: [PATCH 08/72] enable ipv6 in test config --- tests/src/traffic/steal.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index d8687b59ee1..76a8cda6bb8 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -81,7 +81,12 @@ mod steal_tests { // } let mut process = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.target, + Some(&service.namespace), + Some(flags), + Some(vec![("MIRRORD_INCOMING_ENABLE_IPV6", "true")]), + ) .await; process From 8bd4e233bf69e90e897d8fefd05f6098b6bf91f3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 10 Dec 2024 18:58:40 +0100 Subject: [PATCH 09/72] don't change CONTRIBUTING.md formatting --- CONTRIBUTING.md | 205 ++++++++++++++++++++---------------------------- 1 file changed, 83 insertions(+), 122 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33e0f0ce262..fbf717d7069 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,7 @@ # Contributing Before submitting pull request features, please discuss them with us first by opening an issue or a discussion. -We welcome new/junior/starting developers. Feel free to join to our [Discord channel](https://discord.gg/metalbear) for -help and guidance. +We welcome new/junior/starting developers. Feel free to join to our [Discord channel](https://discord.gg/metalbear) for help and guidance. If you would like to start working on an issue, please comment on the issue on GitHub, so that we can assign you to that issue. @@ -12,10 +11,10 @@ Make sure to take a look at the project's [style guide](STYLE.md). # Contents - [Contents](#contents) - - [Getting Started](#getting-started) - - [Debugging mirrord](#debugging-mirrord) - - [New Hook Guidelines](#new-hook-guidelines) - - [Compiling on MacOs](#compliling-on-macos) + - [Getting Started](#getting-started) + - [Debugging mirrord](#debugging-mirrord) + - [New Hook Guidelines](#new-hook-guidelines) + - [Compiling on MacOs](#compliling-on-macos) # Getting Started @@ -32,8 +31,7 @@ The following guide details the steps to setup a local development environment f ### Setup a Kubernetes cluster -For E2E tests and testing mirrord manually you will need a working Kubernetes cluster. A minimal cluster can be easily -setup locally using either of the following: +For E2E tests and testing mirrord manually you will need a working Kubernetes cluster. A minimal cluster can be easily setup locally using either of the following: - [Minikube](https://minikube.sigs.k8s.io/) - [Docker Desktop](https://www.docker.com/products/docker-desktop/) @@ -53,13 +51,11 @@ minikube start --driver=docker ### Prepare a cluster -Build mirrord-agent Docker Image. - -Make sure -you're [logged in to GHCR](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). + Build mirrord-agent Docker Image. + +Make sure you're [logged in to GHCR](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Then run: - ```bash docker buildx build -t test . --file mirrord/agent/Dockerfile ``` @@ -91,25 +87,22 @@ kubectl config use-context minikube ## E2E Tests -The E2E tests create Kubernetes resources in the cluster that kubectl is configured to use and then run sample apps -with the mirrord CLI. The mirrord CLI spawns an agent for the target on the cluster, and runs the test app, with the -layer injected into it. Some test apps need to be compiled before they can be used in the tests +The E2E tests create Kubernetes resources in the cluster that kubectl is configured to use and then run sample apps +with the mirrord CLI. The mirrord CLI spawns an agent for the target on the cluster, and runs the test app, with the +layer injected into it. Some test apps need to be compiled before they can be used in the tests ([this should be automated in the future](https://github.com/metalbear-co/mirrord/issues/982)). The basic command to run the E2E tests is: - ```bash cargo test --package tests ``` However, when running on macOS a universal binary has to be created first: - ```bash scripts/build_fat_mac.sh ``` And then in order to use that binary in the tests, run the tests like this: - ```bash MIRRORD_TESTS_USE_BINARY=../target/universal-apple-darwin/debug/mirrord cargo test -p tests ``` @@ -118,8 +111,7 @@ If new tests are added, decorate them with `cfg_attr` attribute macro to define For example, a test which only tests sanity of the ephemeral container feature should be decorated with `#[cfg_attr(not(feature = "ephemeral"), ignore)]` -On Linux, running tests may exhaust a large amount of RAM and crash the machine. To prevent this, limit the number of -concurrent jobs by running the command with e.g. `-j 4` +On Linux, running tests may exhaust a large amount of RAM and crash the machine. To prevent this, limit the number of concurrent jobs by running the command with e.g. `-j 4` ### IPv6 @@ -139,11 +131,10 @@ In order to test IPv6 on a local cluster on macOS, you can use Kind: 3. `kind create cluster --config kind-config.yaml` 4. When you run `kubectl get svc -o wide --all-namespaces` you should see IPv6 addresses. + ### Cleanup -The Kubernetes resources created by the E2E tests are automatically deleted when the test exits. However, you can -preserve resources from failed tests for debugging. To do this, set the `MIRRORD_E2E_PRESERVE_FAILED` variable to any -value. +The Kubernetes resources created by the E2E tests are automatically deleted when the test exits. However, you can preserve resources from failed tests for debugging. To do this, set the `MIRRORD_E2E_PRESERVE_FAILED` variable to any value. ```bash MIRRORD_E2E_PRESERVE_FAILED=y cargo test --package tests @@ -158,11 +149,11 @@ kubectl delete namespaces,deployments,services -l mirrord-e2e-test-resource=true ## Integration Tests The layer's integration tests test the hooks and their logic without actually using a Kubernetes cluster and spawning -an agent. The integration tests usually execute a test app and load the dynamic library of the layer into them. The -tests set the layer to connect to a TCP/IP address instead of spawning a new agent. The tests then have to simulate the +an agent. The integration tests usually execute a test app and load the dynamic library of the layer into them. The +tests set the layer to connect to a TCP/IP address instead of spawning a new agent. The tests then have to simulate the agent - they accept the layer's connection, receive the layers messages and answer them as the agent would. -Since they do not need to create Kubernetes resources and spawn agents, the integration tests complete much faster than +Since they do not need to create Kubernetes resources and spawn agents, the integration tests complete much faster than the E2E tests, especially on GitHub Actions. Therefore, whenever possible we create integration tests, and only resort to E2E tests when necessary. @@ -173,19 +164,16 @@ Some test apps need to be compiled before they can be used in the tests ([this should be automated in the future](https://github.com/metalbear-co/mirrord/issues/982)). The basic command to run the integration tests is: - ```bash cargo test --package mirrord-layer ``` However, when running on macOS a dylib has to be created first: - ```bash scripts/build_fat_mac.sh ``` And then in order to use that dylib in the tests, run the tests like this: - ```bash MIRRORD_TEST_USE_EXISTING_LIB=../../target/universal-apple-darwin/debug/libmirrord_layer.dylib cargo test -p mirrord-layer ``` @@ -278,17 +266,14 @@ py-serv-deployment-ff89b5974-x9tjx 1/1 Running 0 3h8m ### Build and run mirrord -To build this project, you will first need a [Protocol Buffer Compiler](https://grpc.io/docs/protoc-installation/) -installed. +To build this project, you will first need a [Protocol Buffer Compiler](https://grpc.io/docs/protoc-installation/) installed. #### macOS - ```bash scripts/build_fat_mac.sh ``` #### Linux - ```bash cargo build ``` @@ -303,34 +288,34 @@ Sample web server - `app.js` (present at `sample/node/app.mjs` in the repo) sample/node/app.mjs ```js -import {Buffer} from "node:buffer"; -import {createServer} from "net"; -import {open, readFile} from "fs/promises"; +import { Buffer } from "node:buffer"; +import { createServer } from "net"; +import { open, readFile } from "fs/promises"; async function debug_file_ops() { - try { - const readOnlyFile = await open("/var/log/dpkg.log", "r"); - console.log(">>>>> open readOnlyFile ", readOnlyFile); + try { + const readOnlyFile = await open("/var/log/dpkg.log", "r"); + console.log(">>>>> open readOnlyFile ", readOnlyFile); - let buffer = Buffer.alloc(128); - let bufferResult = await readOnlyFile.read(buffer); - console.log(">>>>> read readOnlyFile returned with ", bufferResult); + let buffer = Buffer.alloc(128); + let bufferResult = await readOnlyFile.read(buffer); + console.log(">>>>> read readOnlyFile returned with ", bufferResult); - const sampleFile = await open("/tmp/node_sample.txt", "w+"); - console.log(">>>>> open file ", sampleFile); + const sampleFile = await open("/tmp/node_sample.txt", "w+"); + console.log(">>>>> open file ", sampleFile); - const written = await sampleFile.write("mirrord sample node"); - console.log(">>>>> written ", written, " bytes to file ", sampleFile); + const written = await sampleFile.write("mirrord sample node"); + console.log(">>>>> written ", written, " bytes to file ", sampleFile); - let sampleBuffer = Buffer.alloc(32); - let sampleBufferResult = await sampleFile.read(buffer); - console.log(">>>>> read ", sampleBufferResult, " bytes from ", sampleFile); + let sampleBuffer = Buffer.alloc(32); + let sampleBufferResult = await sampleFile.read(buffer); + console.log(">>>>> read ", sampleBufferResult, " bytes from ", sampleFile); - readOnlyFile.close(); - sampleFile.close(); - } catch (fail) { - console.error("!!! Failed file operation with ", fail); - } + readOnlyFile.close(); + sampleFile.close(); + } catch (fail) { + console.error("!!! Failed file operation with ", fail); + } } // debug_file_ops(); @@ -338,34 +323,32 @@ async function debug_file_ops() { const server = createServer(); server.on("connection", handleConnection); server.listen( - { - host: "localhost", - port: 80, - }, - function () { - console.log("server listening to %j", server.address()); - } + { + host: "localhost", + port: 80, + }, + function () { + console.log("server listening to %j", server.address()); + } ); function handleConnection(conn) { - var remoteAddress = conn.remoteAddress + ":" + conn.remotePort; - console.log("new client connection from %s", remoteAddress); - conn.on("data", onConnData); - conn.once("close", onConnClose); - conn.on("error", onConnError); - - function onConnData(d) { - console.log("connection data from %s: %j", remoteAddress, d.toString()); - conn.write(d); - } - - function onConnClose() { - console.log("connection from %s closed", remoteAddress); - } - - function onConnError(err) { - console.log("Connection %s error: %s", remoteAddress, err.message); - } + var remoteAddress = conn.remoteAddress + ":" + conn.remotePort; + console.log("new client connection from %s", remoteAddress); + conn.on("data", onConnData); + conn.once("close", onConnClose); + conn.on("error", onConnError); + + function onConnData(d) { + console.log("connection data from %s: %j", remoteAddress, d.toString()); + conn.write(d); + } + function onConnClose() { + console.log("connection from %s closed", remoteAddress); + } + function onConnError(err) { + console.log("Connection %s error: %s", remoteAddress, err.message); + } } ``` @@ -375,9 +358,9 @@ function handleConnection(conn) { ```bash RUST_LOG=debug target/debug/mirrord exec -i test -l debug -c --target pod/py-serv-deployment-ff89b5974-x9tjx node sample/node/app.mjs ``` - > **Note:** You need to change the pod name here to the name of the pod created on your system. + ``` . . @@ -439,11 +422,9 @@ OK - GET: Request completed ## mirrord console -Debugging mirrord can get hard since we're running from another app flow, so the fact we're debugging might affect the -program and make it unusable/buggy (due to sharing stdout with scripts/other applications). +Debugging mirrord can get hard since we're running from another app flow, so the fact we're debugging might affect the program and make it unusable/buggy (due to sharing stdout with scripts/other applications). -The recommended way to do it is to use `mirrord-console`. It is a small application that receives log information from -different mirrord instances and prints it, controlled via `RUST_LOG` environment variable. +The recommended way to do it is to use `mirrord-console`. It is a small application that receives log information from different mirrord instances and prints it, controlled via `RUST_LOG` environment variable. To use mirrord console, run it: `cargo run --bin mirrord-console --features binary` @@ -461,29 +442,24 @@ To debug it with a debugger: ```Rust tokio::time::sleep(Duration::from_secs(20)).await; ``` - to [somewhere](https://github.com/metalbear-co/mirrord/blob/fa2af7f1e77a9254fb0908be40b0dae5da53d298/mirrord/cli/src/internal_proxy.rs#L145) - in the start of the intproxy code. + to [somewhere](https://github.com/metalbear-co/mirrord/blob/fa2af7f1e77a9254fb0908be40b0dae5da53d298/mirrord/cli/src/internal_proxy.rs#L145) in the start of the intproxy code. 2. Set breakpoints in vscode in the relevant lines of the intproxy code. 3. Build mirrord. 4. Run mirrord. -5. Attach a debugger in vscode to the inproxy process. On macOS you can do that with `Cmd` + `Shift` + `P` -> - `LLDB: Attach to Process...` -> type `intproxy` and choose the `mirrord intproxy` process. The sleep you added at the - start of the intproxy is time for you to attach the debugger. +5. Attach a debugger in vscode to the inproxy process. On macOS you can do that with `Cmd` + `Shift` + `P` -> `LLDB: Attach to Process...` -> type `intproxy` and choose the `mirrord intproxy` process. The sleep you added at the start of the intproxy is time for you to attach the debugger. ## Retrieving Agent Logs -By default, the agent's pod will complete and disappear shortly after the agent exits. In order to be able to retrieve +By default, the agent's pod will complete and disappear shortly after the agent exits. In order to be able to retrieve the agent's logs after it crashes, set the agent's pod's TTL to a comfortable number of seconds. This configuration can be specified either as a command line argument (`--agent-ttl`), environment variable (`MIRRORD_AGENT_TTL`), or in a configuration file: - ```toml [agent] ttl = 30 ``` Then, when running with some reasonable TTL, you can retrieve the agent log like this: - ```bash kubectl logs -l app=mirrord --tail=-1 | less -R ``` @@ -491,11 +467,9 @@ kubectl logs -l app=mirrord --tail=-1 | less -R This will retrieve the logs from all running mirrord agents, so it is only useful when just one agent pod exists. If there are currently multiple agent pods running on your cluster, you would have to run - ```bash kubectl get pods ``` - and find the name of the agent pod you're interested in, then run ```bash @@ -506,48 +480,36 @@ where you would replace `` with the name of the pod. # New Hook Guidelines -Adding a feature to mirrord that introduces a new hook (file system, network) can be tricky and there are a lot of edge -cases we might need to cover. +Adding a feature to mirrord that introduces a new hook (file system, network) can be tricky and there are a lot of edge cases we might need to cover. In order to have a more structured approach, here’s the flow you should follow when working on such a feature. -1. Start with the use case. Write an example use case of the feature, for example “App needs to read credentials from a - file”. -2. Write a minimal app that implements the use case - so in the case of our example, an app that reads credentials from - a file. Start with either Node or Python as those are most common. -3. Figure out what functions need to be hooked in order for the behavior to be run through the mirrord-agent instead of - locally. We suggest using `strace`. +1. Start with the use case. Write an example use case of the feature, for example “App needs to read credentials from a file”. +2. Write a minimal app that implements the use case - so in the case of our example, an app that reads credentials from a file. Start with either Node or Python as those are most common. +3. Figure out what functions need to be hooked in order for the behavior to be run through the mirrord-agent instead of locally. We suggest using `strace`. 4. Write a doc on how you would hook and handle the cases, for example: 1. To implement use case “App needs to read credentials from a file*” 2. I will hook `open` and `read` handling calls only with flags O_RDONLY. - 3. Once `open` is called, I’ll send a blocking request to the agent to open the remote file, returning the return - code of the operation. - 4. Create an fd using `memfd`. The result will be returned to the local app, and if successful we’ll save that fd - into a HashMap that matches between local fd and remote fd/identifier. - 5. When `read` is called, I will check if the fd being read was previously opened by us, and if it is we’ll send a - blocking `read` request to the agent. The result will be sent back to the caller. + 3. Once `open` is called, I’ll send a blocking request to the agent to open the remote file, returning the return code of the operation. + 4. Create an fd using `memfd`. The result will be returned to the local app, and if successful we’ll save that fd into a HashMap that matches between local fd and remote fd/identifier. + 5. When `read` is called, I will check if the fd being read was previously opened by us, and if it is we’ll send a blocking `read` request to the agent. The result will be sent back to the caller. 6. And so on. 5. This doc should go later on to our mirrord docs for advanced developers so people can understand how stuff works 6. After approval of the implementation, you can start writing code, and add relevant e2e tests. # Compliling on MacOS -The `mirrord-agent` crate makes use of the `#[cfg(target_os = "linux")]` attribute to allow the whole repo to compile on -MacOS when you run `cargo build`. +The `mirrord-agent` crate makes use of the `#[cfg(target_os = "linux")]` attribute to allow the whole repo to compile on MacOS when you run `cargo build`. To enable `mirrord-agent` code analysis with rust-analyzer: - 1. Install additional targets - ```sh rustup target add x86_64-unknown-linux-gnu rustup target add aarch64-apple-darwin rustup target add x86_64-apple-darwin rustup target add aarch64-unknown-linux-gnu ``` - 2. Add additional targets to your local `.cargo/config.toml` block: - ```toml [build] target = [ @@ -559,15 +521,14 @@ target = [ ``` If you're using rust-analyzer VSCode extension, put this block in `.vscode/settings.json` as well: - ```json { - "rust-analyzer.check.targets": [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu" - ] + "rust-analyzer.check.targets": [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu" + ] } ``` From c733263687fbb58407d936f08b5097c7f7ffefcc Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 10 Dec 2024 19:30:35 +0100 Subject: [PATCH 10/72] Use IpAddr instad of Ipv4Addr for pod IPs --- mirrord/kube/src/api/runtime.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mirrord/kube/src/api/runtime.rs b/mirrord/kube/src/api/runtime.rs index a77afd1b5b5..a431a3b4c03 100644 --- a/mirrord/kube/src/api/runtime.rs +++ b/mirrord/kube/src/api/runtime.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeMap, convert::Infallible, fmt::{self, Display, Formatter}, - net::Ipv4Addr, + net::IpAddr, ops::FromResidual, str::FromStr, }; @@ -71,7 +71,7 @@ impl Display for ContainerRuntime { #[derive(Debug)] pub struct RuntimeData { pub pod_name: String, - pub pod_ips: Vec, + pub pod_ips: Vec, pub pod_namespace: Option, pub node_name: String, pub container_id: String, @@ -128,9 +128,9 @@ impl RuntimeData { .filter_map(|pod_ip| { pod_ip .ip - .parse::() + .parse::() .inspect_err(|e| { - tracing::warn!("failed to parse pod IP {ip}: {e:?}", ip = pod_ip.ip); + tracing::warn!("failed to parse pod IP {ip}: {e:#?}", ip = pod_ip.ip); }) .ok() }) From c97dd1d5e921767eb9422a74e383c0702fa8b578 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 12 Dec 2024 17:31:45 +0100 Subject: [PATCH 11/72] E2E test with portforwarding --- tests/src/env.rs | 2 +- tests/src/file_ops.rs | 38 +++++-- tests/src/http.rs | 14 ++- tests/src/issue1317.rs | 9 +- tests/src/operator/concurrent_steal.rs | 15 ++- tests/src/operator/policies.rs | 4 +- tests/src/traffic.rs | 138 +++++++++++++++++-------- tests/src/traffic/steal.rs | 51 ++++++--- tests/src/utils.rs | 51 +++------ tests/src/utils/ipv6.rs | 94 +++++++++++++++++ 10 files changed, 297 insertions(+), 119 deletions(-) create mode 100644 tests/src/utils/ipv6.rs diff --git a/tests/src/env.rs b/tests/src/env.rs index 7e192ad9f94..8f4b4a8e919 100644 --- a/tests/src/env.rs +++ b/tests/src/env.rs @@ -36,7 +36,7 @@ mod env_tests { let service = service.await; let mut process = run_exec_with_target( application.command(), - &service.target, + &service.pod_container_target(), None, application.mirrord_args(), None, diff --git a/tests/src/file_ops.rs b/tests/src/file_ops.rs index 826b6b64ec9..4b67092b974 100644 --- a/tests/src/file_ops.rs +++ b/tests/src/file_ops.rs @@ -33,7 +33,7 @@ mod file_ops_tests { let env = vec![("MIRRORD_FILE_READ_WRITE_PATTERN", "/tmp/**")]; let mut process = run_exec_with_target( command, - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(args), Some(env), @@ -63,7 +63,7 @@ mod file_ops_tests { let mut process = run_exec_with_target( python_command, - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(args), Some(env), @@ -97,7 +97,7 @@ mod file_ops_tests { let mut process = run_exec_with_target( python_command, - &service.target, + &service.pod_container_target(), Some(&service.namespace), None, None, @@ -117,8 +117,14 @@ mod file_ops_tests { pub async fn bash_file_exists(#[future] service: KubeService) { let service = service.await; let bash_command = vec!["bash", "bash-e2e/file.sh", "exists"]; - let mut process = - run_exec_with_target(bash_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + bash_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -135,8 +141,14 @@ mod file_ops_tests { pub async fn bash_file_read(#[future] service: KubeService) { let service = service.await; let bash_command = vec!["bash", "bash-e2e/file.sh", "read"]; - let mut process = - run_exec_with_target(bash_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + bash_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -151,8 +163,14 @@ mod file_ops_tests { let service = service.await; let bash_command = vec!["bash", "bash-e2e/file.sh", "write"]; let args = vec!["--rw"]; - let mut process = - run_exec_with_target(bash_command, &service.target, None, Some(args), None).await; + let mut process = run_exec_with_target( + bash_command, + &service.pod_container_target(), + None, + Some(args), + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -183,7 +201,7 @@ mod file_ops_tests { let mut process = run_exec_with_target( command, - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(args), None, diff --git a/tests/src/http.rs b/tests/src/http.rs index 388bc6d2c89..e9e23911c3f 100644 --- a/tests/src/http.rs +++ b/tests/src/http.rs @@ -41,7 +41,12 @@ mod http_tests { let kube_client = kube_client.await; let url = get_service_url(kube_client.clone(), &service).await; let mut process = application - .run(&service.target, Some(&service.namespace), None, None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + None, + None, + ) .await; process .wait_for_line(Duration::from_secs(120), "daemon subscribed") @@ -80,7 +85,12 @@ mod http_tests { let kube_client = kube_client.await; let url = get_service_url(kube_client.clone(), &service).await; let mut process = application - .run(&service.target, Some(&service.namespace), None, None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + None, + None, + ) .await; process .wait_for_line(Duration::from_secs(300), "daemon subscribed") diff --git a/tests/src/issue1317.rs b/tests/src/issue1317.rs index 54b70cde154..28992e0fda2 100644 --- a/tests/src/issue1317.rs +++ b/tests/src/issue1317.rs @@ -81,7 +81,14 @@ mod issue1317_tests { .to_string_lossy() .to_string(); let executable = vec![app_path.as_str()]; - let mut process = run_exec_with_target(executable, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + executable, + &service.pod_container_target(), + None, + None, + None, + ) + .await; process .wait_for_line(Duration::from_secs(120), "daemon subscribed") diff --git a/tests/src/operator/concurrent_steal.rs b/tests/src/operator/concurrent_steal.rs index f7fcb44a952..8a142c9075e 100644 --- a/tests/src/operator/concurrent_steal.rs +++ b/tests/src/operator/concurrent_steal.rs @@ -31,7 +31,7 @@ pub async fn two_clients_steal_same_target( let mut client_a = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags.clone()), None, @@ -43,7 +43,12 @@ pub async fn two_clients_steal_same_target( .await; let mut client_b = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + None, + ) .await; let res = client_b.child.wait().await.unwrap(); @@ -84,7 +89,7 @@ pub async fn two_clients_steal_same_target_pod_deployment( let mut client_a = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags.clone()), None, @@ -148,7 +153,7 @@ pub async fn two_clients_steal_with_http_filter( let mut client_a = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags.clone()), Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -164,7 +169,7 @@ pub async fn two_clients_steal_with_http_filter( let mut client_b = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), diff --git a/tests/src/operator/policies.rs b/tests/src/operator/policies.rs index 114d42968e6..2b0a5d05e4c 100644 --- a/tests/src/operator/policies.rs +++ b/tests/src/operator/policies.rs @@ -282,7 +282,7 @@ async fn run_mirrord_and_verify_steal_result( let target = if target_deployment { format!("deploy/{}", kube_service.name) } else { - kube_service.target.clone() + kube_service.pod_container_target() }; let test_proc = application @@ -301,7 +301,7 @@ async fn run_mirrord_and_verify_steal_result( let test_proc = application .run( - &kube_service.target, + &kube_service.pod_container_target(), Some(&kube_service.namespace), Some(vec!["--config-file", config_path.to_str().unwrap()]), None, diff --git a/tests/src/traffic.rs b/tests/src/traffic.rs index 66d1df00238..abdaec6404c 100644 --- a/tests/src/traffic.rs +++ b/tests/src/traffic.rs @@ -30,8 +30,14 @@ mod traffic_tests { "node", "node-e2e/remote_dns/test_remote_dns_enabled_works.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -47,8 +53,14 @@ mod traffic_tests { "node", "node-e2e/remote_dns/test_remote_dns_lookup_google.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -66,8 +78,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_single_request.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -83,8 +101,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_single_request_ipv6.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -102,7 +126,7 @@ mod traffic_tests { let mirrord_args = vec!["--no-outgoing"]; let mut process = run_exec_with_target( node_command, - &service.target, + &service.pod_container_target(), None, Some(mirrord_args), None, @@ -122,8 +146,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_make_request_after_listen.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); } @@ -137,8 +167,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_make_request_localhost.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); } @@ -185,7 +221,7 @@ mod traffic_tests { let mirrord_no_outgoing = vec!["--no-outgoing"]; let mut process = run_exec_with_target( node_command.clone(), - &target_service.target, + &target_service.pod_container_target(), Some(&target_service.namespace), Some(mirrord_no_outgoing), None, @@ -193,18 +229,13 @@ mod traffic_tests { .await; let res = process.wait().await; assert!(res.success()); // The test does not fail, because UDP does not report dropped datagrams. - let stripped_target = internal_service - .target - .split('/') - .nth(1) - .expect("malformed target"); - let logs = pod_api.logs(stripped_target, &lp).await; + let logs = pod_api.logs(&internal_service.pod_name, &lp).await; assert_eq!(logs.unwrap(), ""); // Assert that the target service did not get the message. // Run mirrord with outgoing enabled. let mut process = run_exec_with_target( node_command, - &target_service.target, + &target_service.pod_container_target(), Some(&target_service.namespace), None, None, @@ -217,7 +248,7 @@ mod traffic_tests { lp.follow = true; // Follow log stream. let mut log_lines = pod_api - .log_stream(stripped_target, &lp) + .log_stream(&internal_service.pod_name, &lp) .await .unwrap() .lines(); @@ -278,7 +309,7 @@ mod traffic_tests { // If this verification fails, the test itself is invalid. let mut process = run_exec_with_target( node_command.clone(), - &target_service.target, + &target_service.pod_container_target(), Some(&target_service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -286,12 +317,7 @@ mod traffic_tests { .await; let res = process.wait().await; assert!(!res.success()); // Should fail because local process cannot reach service. - let stripped_target = internal_service - .target - .split('/') - .nth(1) - .expect("malformed target"); - let logs = pod_api.logs(stripped_target, &lp).await; + let logs = pod_api.logs(&internal_service.pod_name, &lp).await; assert_eq!(logs.unwrap(), ""); // Create remote filter file with service name so we can test DNS outgoing filter. @@ -326,7 +352,7 @@ mod traffic_tests { // Run mirrord with outgoing enabled. let mut process = run_exec_with_target( node_command, - &target_service.target, + &target_service.pod_container_target(), Some(&target_service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -339,7 +365,7 @@ mod traffic_tests { lp.follow = true; // Follow log stream. let mut log_lines = pod_api - .log_stream(stripped_target, &lp) + .log_stream(&internal_service.pod_name, &lp) .await .unwrap() .lines(); @@ -376,7 +402,7 @@ mod traffic_tests { let mirrord_args = vec!["--no-outgoing"]; let mut process = run_exec_with_target( node_command, - &service.target, + &service.pod_container_target(), None, Some(mirrord_args), None, @@ -395,7 +421,8 @@ mod traffic_tests { pub async fn test_go(service: impl Future, command: Vec<&str>) { let service = service.await; - let mut process = run_exec_with_target(command, &service.target, None, None, None).await; + let mut process = + run_exec_with_target(command, &service.pod_container_target(), None, None, None).await; let res = process.wait().await; assert!(res.success()); } @@ -458,8 +485,14 @@ mod traffic_tests { pub async fn listen_localhost(#[future] service: KubeService) { let service = service.await; let node_command = vec!["node", "node-e2e/listen/test_listen_localhost.mjs"]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); } @@ -471,8 +504,14 @@ mod traffic_tests { pub async fn gethostname_remote_result(#[future] hostname_service: KubeService) { let service = hostname_service.await; let node_command = vec!["python3", "-u", "python-e2e/hostname.py"]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -511,7 +550,9 @@ mod traffic_tests { "MIRRORD_OUTGOING_REMOTE_UNIX_STREAMS", "/app/unix-socket-server.sock", )]); - let mut process = run_exec_with_target(executable, &service.target, None, None, env).await; + let mut process = + run_exec_with_target(executable, &service.pod_container_target(), None, None, env) + .await; let res = process.wait().await; // The test application panics if it does not successfully connect to the socket, send data, @@ -534,7 +575,14 @@ mod traffic_tests { .to_string(); let executable = vec![app_path.as_str()]; - let mut process = run_exec_with_target(executable, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + executable, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; // The test application panics if it does not successfully connect to the socket, send data, @@ -551,8 +599,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_many_requests.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.child.wait().await.unwrap(); assert!(res.success()); @@ -571,7 +625,7 @@ mod traffic_tests { let mirrord_args = vec!["--no-outgoing"]; let mut process = run_exec_with_target( node_command, - &service.target, + &service.pod_container_target(), None, Some(mirrord_args), None, diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 76a8cda6bb8..ea07b8cf6c0 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -9,7 +9,10 @@ mod steal_tests { }; use futures_util::{SinkExt, StreamExt}; - use kube::Client; + use hyper::{body, client::conn, Request, StatusCode}; + use hyper_util::rt::TokioIo; + use k8s_openapi::api::core::v1::Pod; + use kube::{Api, Client}; use reqwest::{header::HeaderMap, Url}; use rstest::*; use tokio::time::sleep; @@ -19,7 +22,8 @@ mod steal_tests { }; use crate::utils::{ - config_dir, get_service_host_and_port, get_service_url, http2_service, ipv6_service, + config_dir, get_service_host_and_port, get_service_url, http2_service, + ipv6::{ipv6_service, portforward_http_requests}, kube_client, send_request, send_requests, service, tcp_echo_service, websocket_service, Application, KubeService, }; @@ -48,7 +52,12 @@ mod steal_tests { } let mut process = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + None, + ) .await; process @@ -73,7 +82,7 @@ mod steal_tests { let application = Application::PythonFastApiHTTPIPv6; let service = ipv6_service.await; let kube_client = kube_client.await; - let url = get_service_url(kube_client.clone(), &service).await; + let mut flags = vec!["--steal"]; // if cfg!(feature = "ephemeral") { @@ -82,7 +91,7 @@ mod steal_tests { let mut process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_INCOMING_ENABLE_IPV6", "true")]), @@ -92,7 +101,10 @@ mod steal_tests { process .wait_for_line(Duration::from_secs(40), "daemon subscribed") .await; - send_requests(&url, true, Default::default()).await; + + let api = Api::::namespaced(kube_client.clone(), &service.namespace); + portforward_http_requests(&api, service).await; + tokio::time::timeout(Duration::from_secs(40), process.wait()) .await .unwrap(); @@ -126,7 +138,7 @@ mod steal_tests { let mut process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "true")]), @@ -162,7 +174,12 @@ mod steal_tests { } let mut process = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + None, + ) .await; // Verify that we hooked the socket operations and the agent started stealing. @@ -245,7 +262,7 @@ mod steal_tests { let mut process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![ @@ -325,7 +342,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -366,7 +383,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -407,7 +424,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -460,7 +477,7 @@ mod steal_tests { let mut mirrored_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -531,7 +548,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -596,7 +613,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -666,7 +683,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -745,7 +762,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(vec!["--steal"]), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 3287e55020e..207a233bb26 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -40,6 +40,7 @@ use tokio::{ }; pub mod sqs_resources; +pub(crate) mod ipv6; const TEXT: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; pub const CONTAINER_NAME: &str = "test"; @@ -741,15 +742,19 @@ impl Drop for ResourceGuard { pub struct KubeService { pub name: String, pub namespace: String, - pub target: String, guards: Vec, namespace_guard: Option, + pub pod_name: String, } impl KubeService { pub fn deployment_target(&self) -> String { format!("deployment/{}", self.name) } + + pub fn pod_container_target(&self) -> String { + format!("pod/{}/container/{CONTAINER_NAME}", self.pod_name) + } } impl Drop for KubeService { @@ -1223,13 +1228,13 @@ async fn internal_service( .unwrap(); watch_resource_exists(&service_api, "default").await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1241,7 +1246,7 @@ async fn internal_service( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), + pod_name, guards: vec![pod_guard, service_guard], namespace_guard, } @@ -1334,13 +1339,13 @@ pub async fn service_for_mirrord_ls( .unwrap(); watch_resource_exists(&service_api, "default").await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1352,7 +1357,7 @@ pub async fn service_for_mirrord_ls( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), + pod_name, guards: vec![pod_guard, service_guard], namespace_guard, } @@ -1607,38 +1612,6 @@ pub async fn random_namespace_self_deleting_service(#[future] kube_client: Clien .await } -/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted -/// when the returned service is dropped, unless it is dropped during panic. -/// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. -/// * `randomize_name` - whether a random suffix should be added to the end of the resource names -#[fixture] -pub async fn ipv6_service( - #[default("default")] namespace: &str, - #[default("NodePort")] service_type: &str, - // #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, - #[default("docker.io/t4lz/mirrord-pytest:1204")] image: &str, - #[default("http-echo")] service_name: &str, - #[default(true)] randomize_name: bool, - #[future] kube_client: Client, -) -> KubeService { - internal_service( - namespace, - service_type, - image, - service_name, - randomize_name, - kube_client.await, - serde_json::json!([ - { - "name": "HOST", - "value": "::" - } - ]), - true, - ) - .await -} - pub fn resolve_node_host() -> String { if (cfg!(target_os = "linux") && !wsl::is_wsl()) || std::env::var("USE_MINIKUBE").is_ok() { let output = std::process::Command::new("minikube") diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs new file mode 100644 index 00000000000..ac2214c6f34 --- /dev/null +++ b/tests/src/utils/ipv6.rs @@ -0,0 +1,94 @@ +use http_body_util::{BodyExt, Empty}; +use hyper::{ + client::{conn, conn::http1::SendRequest}, + Request, +}; +use k8s_openapi::api::core::v1::Pod; +use kube::{Api, Client}; +use rstest::fixture; + +use crate::utils::{internal_service, kube_client, KubeService, PRESERVE_FAILED_ENV_NAME}; + +/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted +/// when the returned service is dropped, unless it is dropped during panic. +/// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. +/// * `randomize_name` - whether a random suffix should be added to the end of the resource names +#[fixture] +pub async fn ipv6_service( + #[default("default")] namespace: &str, + #[default("NodePort")] service_type: &str, + // #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, + #[default("docker.io/t4lz/mirrord-pytest:1204")] image: &str, + #[default("http-echo")] service_name: &str, + #[default(true)] randomize_name: bool, + #[future] kube_client: Client, +) -> KubeService { + internal_service( + namespace, + service_type, + image, + service_name, + randomize_name, + kube_client.await, + serde_json::json!([ + { + "name": "HOST", + "value": "::" + } + ]), + true, + ) + .await +} + +/// Send an HTTP request using the referenced `request_sender`, with the provided `method`, +/// then verify a success status code, and a response body that is the used method. +/// +/// # Panics +/// - If the request cannot be sent. +/// - If the response's code is not OK +/// - If the response's body is not the method's name. +pub async fn send_request_with_method( + method: &str, + request_sender: &mut SendRequest>, +) { + let req = Request::builder() + .method(method) + .body(Empty::::new()) + .unwrap(); + + let res = request_sender.send_request(req).await.unwrap(); + println!("Response: {:?}", res); + assert_eq!(res.status(), hyper::StatusCode::OK); + let bytes = res.collect().await.unwrap().to_bytes(); + let response_string = String::from_utf8(bytes.to_vec()).unwrap(); + assert_eq!(response_string, method); +} + +/// Create a portforward to the pod of the test service, and send HTTP requests over it. +/// Send four HTTP request (GET, POST, PUT, DELETE), using the referenced `request_sender`, with the +/// provided `method`, verify OK status, and a response body that is the used method. +/// +/// # Panics +/// - If a request cannot be sent. +/// - If a response's code is not OK +/// - If a response's body is not the method's name. +pub async fn portforward_http_requests(api: &Api, service: KubeService) { + let mut portforwarder = api + .portforward(&service.pod_name, &[80]) + .await + .expect("Failed to start portforward to test pod"); + + let stream = portforwarder.take_stream(80).unwrap(); + let stream = hyper_util::rt::TokioIo::new(stream); + + let (mut request_sender, connection) = conn::http1::handshake(stream).await.unwrap(); + tokio::spawn(async move { + if let Err(err) = connection.await { + eprintln!("Error in connection from test function to deployed test app {err:#?}"); + } + }); + for method in ["GET", "POST", "PUT", "DELETE"] { + send_request_with_method(method, &mut request_sender).await; + } +} From ef96b640af40bd72f28ecf32e5b6ecf5b91d0034 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 13 Dec 2024 10:50:35 +0100 Subject: [PATCH 12/72] fix tests import --- tests/src/utils/ipv6.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs index ac2214c6f34..8e9aa6affdd 100644 --- a/tests/src/utils/ipv6.rs +++ b/tests/src/utils/ipv6.rs @@ -7,11 +7,13 @@ use k8s_openapi::api::core::v1::Pod; use kube::{Api, Client}; use rstest::fixture; -use crate::utils::{internal_service, kube_client, KubeService, PRESERVE_FAILED_ENV_NAME}; +use crate::utils::{internal_service, kube_client, KubeService}; /// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted /// when the returned service is dropped, unless it is dropped during panic. -/// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. +/// This behavior can be changed, see +/// [`PRESERVE_FAILED_ENV_NAME`](crate::utils::PRESERVE_FAILED_ENV_NAME). +/// /// * `randomize_name` - whether a random suffix should be added to the end of the resource names #[fixture] pub async fn ipv6_service( From 7e06ba82b83cb3d4dfd4638e9c424d1919d7b895 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 13 Dec 2024 12:10:57 +0100 Subject: [PATCH 13/72] move ipv6 config up to network --- mirrord/config/src/feature/network.rs | 16 +++++++++- .../config/src/feature/network/incoming.rs | 31 +------------------ 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index 2f2b7901aee..55e5e2de4da 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -6,10 +6,12 @@ use serde::Serialize; use self::{incoming::*, outgoing::*}; use crate::{ - config::{ConfigContext, ConfigError}, + config::{from_env::FromEnv, source::MirrordConfigSource, ConfigContext, ConfigError}, util::MirrordToggleableConfig, }; +const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; + pub mod dns; pub mod filter; pub mod incoming; @@ -67,14 +69,26 @@ pub struct NetworkConfig { /// ### feature.network.dns {#feature-network-dns} #[config(toggleable, nested)] pub dns: DnsConfig, + + /// ### feature.network.ipv6 {#feature-network-dns} + /// + /// Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. + #[config(env = IPV6_ENV_VAR)] + pub ipv6: bool, } impl MirrordToggleableConfig for NetworkFileConfig { fn disabled_config(context: &mut ConfigContext) -> Result { + let ipv6 = FromEnv::new(IPV6_ENV_VAR) + .source_value(context) + .transpose()? + .unwrap_or_default(); + Ok(NetworkConfig { incoming: IncomingFileConfig::disabled_config(context)?, dns: DnsFileConfig::disabled_config(context)?, outgoing: OutgoingFileConfig::disabled_config(context)?, + ipv6, }) } } diff --git a/mirrord/config/src/feature/network/incoming.rs b/mirrord/config/src/feature/network/incoming.rs index a0906c5a5f1..fddd46260d5 100644 --- a/mirrord/config/src/feature/network/incoming.rs +++ b/mirrord/config/src/feature/network/incoming.rs @@ -18,8 +18,6 @@ pub mod http_filter; use http_filter::*; -const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; - /// ## incoming (network) /// /// Controls the incoming TCP traffic feature. @@ -61,8 +59,7 @@ const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; /// "port_mapping": [[ 7777, 8888 ]], /// "ignore_localhost": false, /// "ignore_ports": [9999, 10000], -/// "listen_ports": [[80, 8111]], -/// "ipv6": false +/// "listen_ports": [[80, 8111]] /// } /// } /// } @@ -103,10 +100,6 @@ impl MirrordConfig for IncomingFileConfig { .source_value(context) .transpose()? .unwrap_or_default(), - ipv6: FromEnv::new(IPV6_ENV_VAR) - .source_value(context) - .transpose()? // error on invalid env var value - .unwrap_or_default(), // if not set either by env or file - set to false. ..Default::default() }, IncomingFileConfig::Advanced(advanced) => IncomingConfig { @@ -139,11 +132,6 @@ impl MirrordConfig for IncomingFileConfig { .transpose()? .unwrap_or_default(), ports: advanced.ports.map(|ports| ports.into_iter().collect()), - ipv6: FromEnv::new(IPV6_ENV_VAR) - .source_value(context) - .transpose()? // error on invalid env var value - .or(advanced.ipv6) // only use file if env var not set. - .unwrap_or_default(), // if not set either by env or file - set to false. }, }; @@ -156,11 +144,6 @@ impl MirrordToggleableConfig for IncomingFileConfig { .source_value(context) .unwrap_or_else(|| Ok(IncomingMode::Off))?; - let ipv6 = FromEnv::new(IPV6_ENV_VAR) - .source_value(context) - .transpose()? - .unwrap_or_default(); - let on_concurrent_steal = FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) @@ -171,7 +154,6 @@ impl MirrordToggleableConfig for IncomingFileConfig { mode, on_concurrent_steal, http_filter: HttpFilterFileConfig::disabled_config(context)?, - ipv6, ..Default::default() }) } @@ -322,11 +304,6 @@ pub struct IncomingAdvancedFileConfig { /// /// Mutually exclusive with [`ignore_ports`](###ignore_ports). pub ports: Option>, - - /// ### ipv6 - /// - /// Enable ipv6 support. Turn on if your applications listens to incoming traffic over IPv6. - pub ipv6: Option, } fn serialize_bi_map(map: &BiMap, serializer: S) -> Result @@ -470,11 +447,6 @@ pub struct IncomingConfig { /// Mutually exclusive with /// [`feature.network.incoming.ignore_ports`](#feature-network-ignore_ports). pub ports: Option>, - - /// #### feature.network.incoming.ipv6 {#feature-network-incoming-ipv6} - /// - /// Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. - pub ipv6: bool, } impl IncomingConfig { @@ -639,6 +611,5 @@ impl CollectAnalytics for &IncomingConfig { analytics.add("ignore_localhost", self.ignore_localhost); analytics.add("ignore_ports_count", self.ignore_ports.len()); analytics.add("http", &self.http_filter); - analytics.add("ipv6", self.ipv6); } } From e6da4537de5708707a09107547bdfffad18b813f Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 13 Dec 2024 13:02:13 +0100 Subject: [PATCH 14/72] Propagate ipv6 setting to an agent arg --- mirrord/agent/src/cli.rs | 11 ++++++++++- mirrord/agent/src/entrypoint.rs | 1 + mirrord/config/src/feature/network.rs | 1 + mirrord/kube/src/api/container.rs | 9 ++++++++- mirrord/kube/src/api/container/job.rs | 2 ++ mirrord/kube/src/api/container/util.rs | 3 ++- mirrord/kube/src/api/kubernetes.rs | 10 ++++++++-- mirrord/layer/src/socket/ops.rs | 2 +- mirrord/protocol/src/lib.rs | 2 ++ 9 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mirrord/agent/src/cli.rs b/mirrord/agent/src/cli.rs index 6c5b11e65a2..07c8aa97503 100644 --- a/mirrord/agent/src/cli.rs +++ b/mirrord/agent/src/cli.rs @@ -1,7 +1,9 @@ #![deny(missing_docs)] use clap::{Parser, Subcommand}; -use mirrord_protocol::{MeshVendor, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; +use mirrord_protocol::{ + MeshVendor, AGENT_IPV6_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV, +}; const DEFAULT_RUNTIME: &str = "containerd"; @@ -50,6 +52,13 @@ pub struct Args { env = "MIRRORD_AGENT_IN_SERVICE_MESH" )] pub is_mesh: bool, + + /// Enable support for IPv6-only clusters + /// + /// Only when this option is set will take the needed steps to run on an IPv6 single stack + /// cluster. + #[arg(long, default_value_t = false, env = AGENT_IPV6_ENV)] + pub ipv6: bool, } impl Args { diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index f345fa0d5c9..4e5e17c768a 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -492,6 +492,7 @@ impl ClientConnectionHandler { async fn start_agent(args: Args) -> Result<()> { trace!("start_agent -> Starting agent with args: {args:?}"); + // listen for client connections let listener = TcpListener::bind(SocketAddrV4::new( Ipv4Addr::UNSPECIFIED, args.communicate_port, diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index 55e5e2de4da..f901a3f6b97 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -98,6 +98,7 @@ impl CollectAnalytics for &NetworkConfig { analytics.add("incoming", &self.incoming); analytics.add("outgoing", &self.outgoing); analytics.add("dns", &self.dns); + analytics.add("ipv6", self.ipv6); } } diff --git a/mirrord/kube/src/api/container.rs b/mirrord/kube/src/api/container.rs index b87a088a412..a651dc13458 100644 --- a/mirrord/kube/src/api/container.rs +++ b/mirrord/kube/src/api/container.rs @@ -44,10 +44,16 @@ pub struct ContainerParams { /// the agent container. pub tls_cert: Option, pub pod_ips: Option, + /// Support IPv6-only clusters + pub support_ipv6: bool, } impl ContainerParams { - pub fn new(tls_cert: Option, pod_ips: Option) -> ContainerParams { + pub fn new( + tls_cert: Option, + pod_ips: Option, + support_ipv6: bool, + ) -> ContainerParams { let port: u16 = rand::thread_rng().gen_range(30000..=65535); let gid: u16 = rand::thread_rng().gen_range(3000..u16::MAX); @@ -64,6 +70,7 @@ impl ContainerParams { port, tls_cert, pod_ips, + support_ipv6, } } } diff --git a/mirrord/kube/src/api/container/job.rs b/mirrord/kube/src/api/container/job.rs index 907aefffeeb..63e072935b0 100644 --- a/mirrord/kube/src/api/container/job.rs +++ b/mirrord/kube/src/api/container/job.rs @@ -247,6 +247,7 @@ mod test { gid: 13, tls_cert: None, pod_ips: None, + support_ipv6: false, }; let update = JobVariant::new(&agent, ¶ms).as_update(); @@ -336,6 +337,7 @@ mod test { gid: 13, tls_cert: None, pod_ips: None, + support_ipv6: false, }; let update = JobTargetedVariant::new( diff --git a/mirrord/kube/src/api/container/util.rs b/mirrord/kube/src/api/container/util.rs index 77f917378ce..23fd752181b 100644 --- a/mirrord/kube/src/api/container/util.rs +++ b/mirrord/kube/src/api/container/util.rs @@ -4,7 +4,7 @@ use futures::{AsyncBufReadExt, TryStreamExt}; use k8s_openapi::api::core::v1::{EnvVar, Pod, Toleration}; use kube::{api::LogParams, Api}; use mirrord_config::agent::{AgentConfig, LinuxCapability}; -use mirrord_protocol::{AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; +use mirrord_protocol::{AGENT_IPV6_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; use regex::Regex; use tracing::warn; @@ -59,6 +59,7 @@ pub(super) fn agent_env(agent: &AgentConfig, params: &&ContainerParams) -> Vec, + support_ipv6: bool, ) -> Result<(ContainerParams, Option), KubeApiError> { let runtime_data = match target.path.as_ref().unwrap_or(&Target::Targetless) { Target::Targetless => None, @@ -187,7 +188,7 @@ impl KubernetesAPI { .join(",") }); - let params = ContainerParams::new(tls_cert, pod_ips); + let params = ContainerParams::new(tls_cert, pod_ips, support_ipv6); Ok((params, runtime_data)) } @@ -209,7 +210,12 @@ impl KubernetesAPI { where P: Progress + Send + Sync, { - let (params, runtime_data) = self.create_agent_params(target, tls_cert).await?; + let support_ipv6 = config + .map(|layer_conf| layer_conf.feature.network.ipv6) + .unwrap_or_default(); + let (params, runtime_data) = self + .create_agent_params(target, tls_cert, support_ipv6) + .await?; if let Some(RuntimeData { guessed_container: true, container_name, diff --git a/mirrord/layer/src/socket/ops.rs b/mirrord/layer/src/socket/ops.rs index 4ff4062f9b0..58930ea0b41 100644 --- a/mirrord/layer/src/socket/ops.rs +++ b/mirrord/layer/src/socket/ops.rs @@ -131,7 +131,7 @@ pub(super) fn socket(domain: c_int, type_: c_int, protocol: c_int) -> Detour Date: Fri, 13 Dec 2024 19:44:34 +0100 Subject: [PATCH 15/72] fallback agent listener --- mirrord/agent/src/entrypoint.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index 4e5e17c768a..df563fd5d6f 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, mem, - net::{Ipv4Addr, SocketAddrV4}, + net::{Ipv4Addr, Ipv6Addr, SocketAddrV4}, path::PathBuf, sync::{ atomic::{AtomicU32, Ordering}, @@ -493,11 +493,30 @@ async fn start_agent(args: Args) -> Result<()> { trace!("start_agent -> Starting agent with args: {args:?}"); // listen for client connections - let listener = TcpListener::bind(SocketAddrV4::new( + let ipv4_listener_result = TcpListener::bind(SocketAddrV4::new( Ipv4Addr::UNSPECIFIED, args.communicate_port, )) - .await?; + .await; + + let listener = if args.ipv6 && ipv4_listener_result.is_err() { + debug!("IPv6 Support enabled, and IPv4 bind failed, binding IPv6 listener"); + TcpListener::bind(SocketAddrV4::new( + Ipv6Addr::UNSPECIFIED, + args.communicate_port, + )) + .await + } else { + ipv4_listener_result + }?; + + match listener.local_addr() { + Ok(addr) => debug!( + client_listener_address = addr.to_string(), + "Created listener." + ), + Err(err) => error!(%err, "listener local address error"), + } let state = State::new(&args).await?; From fdcaced698beb451934eacf58751dc3b0806ec96 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 18 Dec 2024 00:57:56 +0100 Subject: [PATCH 16/72] stealer, iptables - start --- mirrord/agent/src/entrypoint.rs | 16 +++--- mirrord/agent/src/steal/connection.rs | 26 +++++++-- mirrord/agent/src/steal/ip_tables.rs | 14 +++++ mirrord/agent/src/steal/subscriptions.rs | 72 ++++++++++++++++++++---- 4 files changed, 105 insertions(+), 23 deletions(-) diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index df563fd5d6f..82d157dea9b 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -586,13 +586,15 @@ async fn start_agent(args: Args) -> Result<()> { let cancellation_token = cancellation_token.clone(); let watched_task = WatchedTask::new( TcpConnectionStealer::TASK_NAME, - TcpConnectionStealer::new(stealer_command_rx).and_then(|stealer| async move { - let res = stealer.start(cancellation_token).await; - if let Err(err) = res.as_ref() { - error!("Stealer failed: {err}"); - } - res - }), + TcpConnectionStealer::new(stealer_command_rx, args.ipv6).and_then( + |stealer| async move { + let res = stealer.start(cancellation_token).await; + if let Err(err) = res.as_ref() { + error!("Stealer failed: {err}"); + } + res + }, + ), ); let status = watched_task.status(); let task = run_thread_in_namespace( diff --git a/mirrord/agent/src/steal/connection.rs b/mirrord/agent/src/steal/connection.rs index 463c61f88d0..37435176b8b 100644 --- a/mirrord/agent/src/steal/connection.rs +++ b/mirrord/agent/src/steal/connection.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, }; use fancy_regex::Regex; @@ -289,6 +289,9 @@ pub(crate) struct TcpConnectionStealer { /// Set of active connections stolen by [`Self::port_subscriptions`]. connections: StolenConnections, + + /// Shen set, the stealer will use IPv6 if needed. + support_ipv6: bool, } impl TcpConnectionStealer { @@ -297,14 +300,21 @@ impl TcpConnectionStealer { /// Initializes a new [`TcpConnectionStealer`], but doesn't start the actual work. /// You need to call [`TcpConnectionStealer::start`] to do so. #[tracing::instrument(level = "trace")] - pub(crate) async fn new(command_rx: Receiver) -> Result { + pub(crate) async fn new( + command_rx: Receiver, + support_ipv6: bool, + ) -> Result { let config = envy::prefixed("MIRRORD_AGENT_") .from_env::() .unwrap_or_default(); let port_subscriptions = { - let redirector = - IpTablesRedirector::new(config.stealer_flush_connections, config.pod_ips).await?; + let redirector = IpTablesRedirector::new( + config.stealer_flush_connections, + config.pod_ips, + support_ipv6, + ) + .await?; PortSubscriptions::new(redirector, 4) }; @@ -315,6 +325,7 @@ impl TcpConnectionStealer { clients: HashMap::with_capacity(8), clients_closed: Default::default(), connections: StolenConnections::with_capacity(8), + support_ipv6, }) } @@ -371,9 +382,14 @@ impl TcpConnectionStealer { #[tracing::instrument(level = "trace", skip(self))] async fn incoming_connection(&mut self, stream: TcpStream, peer: SocketAddr) -> Result<()> { let mut real_address = orig_dst::orig_dst_addr(&stream)?; + let localhost = if self.support_ipv6 && real_address.is_ipv6() { + IpAddr::V6(Ipv6Addr::LOCALHOST) + } else { + IpAddr::V4(Ipv4Addr::LOCALHOST) + }; // If we use the original IP we would go through prerouting and hit a loop. // localhost should always work. - real_address.set_ip(IpAddr::V4(Ipv4Addr::LOCALHOST)); + real_address.set_ip(localhost); let Some(port_subscription) = self.port_subscriptions.get(real_address.port()).cloned() else { diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 5583b485000..a7e4765e896 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -111,6 +111,20 @@ pub fn new_iptables() -> iptables::IPTables { .expect("IPTables initialization may not fail!") } +/// wrapper around iptables::new that uses nft or legacy based on env +pub fn new_ip6tables() -> iptables::IPTables { + if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") + && val.to_lowercase() == "true" + { + // TODO: check if there is such a binary. + iptables::new_with_cmd("/usr/sbin/ip6tables-nft") + } else { + // TODO: check if there is such a binary. + iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") + } + .expect("IPTables initialization may not fail!") +} + impl Debug for IPTablesWrapper { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("IPTablesWrapper") diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 0ff0e1fa8ea..c022e3f9e76 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -47,17 +47,31 @@ pub trait PortRedirector { async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error>; } -/// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by -/// redirecting TCP packets to inner [`TcpListener`]. -pub(crate) struct IpTablesRedirector { +/// A TCP listener, together with an iptables wrapper to set rules that send traffic to the +/// listener. +pub(crate) struct IptablesListener { /// For altering iptables rules. iptables: Option>, - /// Whether exisiting connections should be flushed when adding new redirects. - flush_connections: bool, /// Port of [`IpTablesRedirector::listener`]. redirect_to: Port, /// Listener to which redirect all connections. listener: TcpListener, +} + +/// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by +/// redirecting TCP packets to inner [`TcpListener`]. +pub(crate) struct IpTablesRedirector { + /// Whether existing connections should be flushed when adding new redirects. + flush_connections: bool, + + /// TCP listener + iptables, for redirecting IPv4 connections. + /// Could be `None` if IPv6 support is enabled and we cannot bind an IPv4 address. + iptables_listener: Option, + + /// TCP listener + ip6tables, for redirecting IPv6 connections. + /// Will only be `Some` if IPv6 support is enabled, and the agent is able to listen on an IPv6 + /// address. + ip6tables_listener: Option, pod_ips: Option, } @@ -67,27 +81,63 @@ impl IpTablesRedirector { /// [`Ipv4Addr::UNSPECIFIED`] address and a random port. This listener will be used to accept /// redirected connections. /// + /// If `support_ipv6` is set, will also listen on IPv6, and a fail to listen over IPv4 will be + /// accepted. + /// /// # Note /// /// Does not yet alter iptables. /// /// # Params /// - /// * `flush_connections` - whether exisitng connections should be flushed when adding new + /// * `flush_connections` - whether existing connections should be flushed when adding new /// redirects pub(crate) async fn new( flush_connections: bool, pod_ips: Option, + support_ipv6: bool, ) -> Result { - let listener = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await?; - let redirect_to = listener.local_addr()?.port(); + let bind_res = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await; + let (iptables_listener, ip6tables_listener) = if support_ipv6 { + let listener4 = bind_res + .inspect_err( + |err| traceing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), + ) + .ok() + .and_then(|listener| { + let redirect_to = listener.local_addr()?.port(); + Some(IptablesListener { + iptables: None, + redirect_to, + listener, + }) + }); + let listener = TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await?; + let redirect_to = listener.local_addr()?.port(); + let listener6 = Some(IptablesListener { + iptables: None, + redirect_to, + listener, + }); + (litener, listener6) + } else { + let listener = bind_res?; + let redirect_to = listener.local_addr()?.port(); + ( + Some(IptablesListener { + iptables: None, // populated lazily + listener, + redirect_to, + }), + None, + ) + }; Ok(Self { - iptables: None, flush_connections, - redirect_to, - listener, pod_ips, + iptables_listener, + ip6tables_listener, }) } } From 19a5c386fc9c40043e73876158d882228c768a75 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 20 Dec 2024 20:28:07 +0200 Subject: [PATCH 17/72] add ipv6 listener and iptables, still need to adapt more places --- mirrord/agent/src/entrypoint.rs | 6 +- mirrord/agent/src/steal/subscriptions.rs | 91 +++++++++++++++++------- tests/src/utils.rs | 7 +- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index 82d157dea9b..fff9ee7192f 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, mem, - net::{Ipv4Addr, Ipv6Addr, SocketAddrV4}, + net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}, path::PathBuf, sync::{ atomic::{AtomicU32, Ordering}, @@ -501,9 +501,11 @@ async fn start_agent(args: Args) -> Result<()> { let listener = if args.ipv6 && ipv4_listener_result.is_err() { debug!("IPv6 Support enabled, and IPv4 bind failed, binding IPv6 listener"); - TcpListener::bind(SocketAddrV4::new( + TcpListener::bind(SocketAddrV6::new( Ipv6Addr::UNSPECIFIED, args.communicate_port, + 0, + 0, )) .await } else { diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index c022e3f9e76..6191f1ea2bf 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -1,6 +1,7 @@ use std::{ collections::{hash_map::Entry, HashMap}, - net::{Ipv4Addr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + ops::Not, sync::Arc, }; @@ -56,14 +57,32 @@ pub(crate) struct IptablesListener { redirect_to: Port, /// Listener to which redirect all connections. listener: TcpListener, + /// Optional comma-seperated list of IPs of the pod, originating in the pod's `Status.PodIps` + pod_ips: Option, + /// Whether existing connections should be flushed when adding new redirects. + flush_connections: bool, +} + +impl IptablesListener { + async fn add_redirect(&mut self, from: Port) -> Result<(), AgentError> { + let iptables = if let Some(iptables) = self.iptables.as_ref() { + iptables + } else { + let safe = crate::steal::ip_tables::SafeIpTables::create( + new_iptables().into(), + self.flush_connections, + self.pod_ips.as_deref(), + ) + .await?; + self.iptables.insert(safe) + }; + iptables.add_redirect(from, self.redirect_to).await + } } /// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by /// redirecting TCP packets to inner [`TcpListener`]. pub(crate) struct IpTablesRedirector { - /// Whether existing connections should be flushed when adding new redirects. - flush_connections: bool, - /// TCP listener + iptables, for redirecting IPv4 connections. /// Could be `None` if IPv6 support is enabled and we cannot bind an IPv4 address. iptables_listener: Option, @@ -72,8 +91,6 @@ pub(crate) struct IpTablesRedirector { /// Will only be `Some` if IPv6 support is enabled, and the agent is able to listen on an IPv6 /// address. ip6tables_listener: Option, - - pod_ips: Option, } impl IpTablesRedirector { @@ -98,18 +115,46 @@ impl IpTablesRedirector { support_ipv6: bool, ) -> Result { let bind_res = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await; + let (pod_ips4, pod_ips6) = pod_ips.map_or_else( + || (None, None), + |ips| { + // TODO: probably nicer to split at the client and avoid the conversion to and back + // from a string. + let (ip4s, ip6s): (Vec<_>, Vec<_>) = ips.split(',').partition(|ip_str| { + ip_str + .parse::() + .inspect_err(|e| tracing::warn!(%e, "failed to parse pod IP {ip_str}")) + .as_ref() + .map(IpAddr::is_ipv4) + .unwrap_or_default() + }); + // Convert to options, `None` if vector is empty. + ( + ip4s.is_empty().not().then(|| ip4s.join(",")), + ip6s.is_empty().not().then(|| ip6s.join(",")), + ) + }, + ); let (iptables_listener, ip6tables_listener) = if support_ipv6 { let listener4 = bind_res .inspect_err( - |err| traceing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), + |err| tracing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), ) .ok() .and_then(|listener| { - let redirect_to = listener.local_addr()?.port(); + let redirect_to = listener + .local_addr() + .inspect_err( + |err| tracing::debug!(%err, "Get IPv4 listener address, continuing with IPv6 only."), + ) + .ok()? + .port(); Some(IptablesListener { iptables: None, redirect_to, listener, + pod_ips: pod_ips4, + flush_connections, }) }); let listener = TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await?; @@ -118,8 +163,10 @@ impl IpTablesRedirector { iptables: None, redirect_to, listener, + pod_ips: pod_ips6, + flush_connections, }); - (litener, listener6) + (listener4, listener6) } else { let listener = bind_res?; let redirect_to = listener.local_addr()?.port(); @@ -128,14 +175,14 @@ impl IpTablesRedirector { iptables: None, // populated lazily listener, redirect_to, + pod_ips: pod_ips4, + flush_connections, }), None, ) }; Ok(Self { - flush_connections, - pod_ips, iptables_listener, ip6tables_listener, }) @@ -147,21 +194,13 @@ impl PortRedirector for IpTablesRedirector { type Error = AgentError; async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { - let iptables = match self.iptables.as_ref() { - Some(iptables) => iptables, - None => { - let iptables = new_iptables(); - let safe = SafeIpTables::create( - iptables.into(), - self.flush_connections, - self.pod_ips.as_deref(), - ) - .await?; - self.iptables.insert(safe) - } - }; - - iptables.add_redirect(from, self.redirect_to).await + if let Some(ip4_listener) = self.iptables_listener.as_mut() { + ip4_listener.add_redirect(from).await?; + } + if let Some(ip6_listener) = self.ip6tables_listener.as_mut() { + ip6_listener.add_redirect(from).await?; + } + Ok(()) } async fn remove_redirection(&mut self, from: Port) -> Result<(), Self::Error> { diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 207a233bb26..b4732d81348 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -39,8 +39,8 @@ use tokio::{ task::JoinHandle, }; -pub mod sqs_resources; pub(crate) mod ipv6; +pub mod sqs_resources; const TEXT: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; pub const CONTAINER_NAME: &str = "test"; @@ -595,7 +595,10 @@ pub async fn run_exec( let mut base_env = HashMap::new(); // TODO: revert // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); - base_env.insert("MIRRORD_AGENT_IMAGE", "docker.io/t4lz/mirrord-agent:latest"); + base_env.insert( + "MIRRORD_AGENT_IMAGE", + "docker.io/t4lz/mirrord-agent:2025-12-13", + ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); From 1fdaf02d0be4fdd933c457263cc5536e3e887311 Mon Sep 17 00:00:00 2001 From: t4lz Date: Sun, 22 Dec 2024 16:14:21 +0200 Subject: [PATCH 18/72] iptable listeners --- mirrord/agent/src/error.rs | 4 + mirrord/agent/src/steal/subscriptions.rs | 183 ++++++++++++++++------- 2 files changed, 134 insertions(+), 53 deletions(-) diff --git a/mirrord/agent/src/error.rs b/mirrord/agent/src/error.rs index ad04e49c8c5..d9ae7cb8b9d 100644 --- a/mirrord/agent/src/error.rs +++ b/mirrord/agent/src/error.rs @@ -84,6 +84,10 @@ pub(crate) enum AgentError { /// Temporary error for vpn feature #[error("Generic error in vpn: {0}")] VpnError(String), + + /// When we neither create a redirector for IPv4, nor for IPv6 + #[error("Could not create a listener for stolen connections")] + CannotListenForStolenConnections, } impl From> for AgentError { diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 6191f1ea2bf..880627e4575 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -7,7 +7,10 @@ use std::{ use dashmap::{mapref::entry::Entry as DashMapEntry, DashMap}; use mirrord_protocol::{Port, RemoteResult, ResponseError}; -use tokio::net::{TcpListener, TcpStream}; +use tokio::{ + net::{TcpListener, TcpStream}, + select, +}; use super::{ http::HttpFilter, @@ -63,8 +66,10 @@ pub(crate) struct IptablesListener { flush_connections: bool, } -impl IptablesListener { - async fn add_redirect(&mut self, from: Port) -> Result<(), AgentError> { +#[async_trait::async_trait] +impl PortRedirector for IptablesListener { + type Error = AgentError; + async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let iptables = if let Some(iptables) = self.iptables.as_ref() { iptables } else { @@ -78,19 +83,40 @@ impl IptablesListener { }; iptables.add_redirect(from, self.redirect_to).await } + + async fn remove_redirection(&mut self, from: Port) -> Result<(), Self::Error> { + if let Some(iptables) = self.iptables.as_ref() { + iptables.remove_redirect(from, self.redirect_to).await?; + } + + Ok(()) + } + + async fn cleanup(&mut self) -> Result<(), Self::Error> { + if let Some(iptables) = self.iptables.take() { + iptables.cleanup().await?; + } + + Ok(()) + } + + async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error> { + self.listener.accept().await.map_err(Into::into) + } } /// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by /// redirecting TCP packets to inner [`TcpListener`]. -pub(crate) struct IpTablesRedirector { - /// TCP listener + iptables, for redirecting IPv4 connections. - /// Could be `None` if IPv6 support is enabled and we cannot bind an IPv4 address. - iptables_listener: Option, - - /// TCP listener + ip6tables, for redirecting IPv6 connections. - /// Will only be `Some` if IPv6 support is enabled, and the agent is able to listen on an IPv6 - /// address. - ip6tables_listener: Option, +/// +/// Holds TCP listeners + iptables, for redirecting IPv4 and/or IPv6 connections. +pub(crate) enum IpTablesRedirector { + Ipv4Only(IptablesListener), + /// Could be used if IPv6 support is enabled, and we cannot bind an IPv4 address. + Ipv6Only(IptablesListener), + Dual { + ipv4_listener: IptablesListener, + ipv6_listener: IptablesListener, + }, } impl IpTablesRedirector { @@ -114,12 +140,11 @@ impl IpTablesRedirector { pod_ips: Option, support_ipv6: bool, ) -> Result { - let bind_res = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await; let (pod_ips4, pod_ips6) = pod_ips.map_or_else( || (None, None), |ips| { // TODO: probably nicer to split at the client and avoid the conversion to and back - // from a string. + // from a string. let (ip4s, ip6s): (Vec<_>, Vec<_>) = ips.split(',').partition(|ip_str| { ip_str .parse::() @@ -135,8 +160,8 @@ impl IpTablesRedirector { ) }, ); - let (iptables_listener, ip6tables_listener) = if support_ipv6 { - let listener4 = bind_res + + let listener4 = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await .inspect_err( |err| tracing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), ) @@ -157,35 +182,69 @@ impl IpTablesRedirector { flush_connections, }) }); - let listener = TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await?; - let redirect_to = listener.local_addr()?.port(); - let listener6 = Some(IptablesListener { - iptables: None, - redirect_to, - listener, - pod_ips: pod_ips6, - flush_connections, - }); - (listener4, listener6) + let listener6 = if support_ipv6 { + TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await + .inspect_err( + |err| tracing::debug!(%err, "Could not bind IPv6, continuing with IPv4 only."), + ) + .ok() + .and_then(|listener| { + let redirect_to = listener + .local_addr() + .inspect_err( + |err| tracing::debug!(%err, "Get IPv6 listener address, continuing with IPv4 only."), + ) + .ok()? + .port(); + Some(IptablesListener { + iptables: None, + redirect_to, + listener, + pod_ips: pod_ips6, + flush_connections, + }) + }) } else { - let listener = bind_res?; - let redirect_to = listener.local_addr()?.port(); - ( - Some(IptablesListener { - iptables: None, // populated lazily - listener, - redirect_to, - pod_ips: pod_ips4, - flush_connections, - }), - None, - ) + None }; + match (listener4, listener6) { + (None, None) => Err(AgentError::CannotListenForStolenConnections), + (Some(ipv4_listener), None) => Ok(Self::Ipv4Only(ipv4_listener)), + (None, Some(ipv6_listener)) => Ok(Self::Ipv6Only(ipv6_listener)), + (Some(ipv4_listener), Some(ipv6_listener)) => Ok(Self::Dual { + ipv4_listener, + ipv6_listener, + }), + } + } - Ok(Self { - iptables_listener, - ip6tables_listener, - }) + pub(crate) fn get_ipv4_listener_mut(&mut self) -> Option<&mut IptablesListener> { + match self { + IpTablesRedirector::Ipv6Only(_) => None, + IpTablesRedirector::Dual { ipv4_listener, .. } + | IpTablesRedirector::Ipv4Only(ipv4_listener) => Some(ipv4_listener), + } + } + + pub(crate) fn get_ipv6_listener_mut(&mut self) -> Option<&mut IptablesListener> { + match self { + IpTablesRedirector::Ipv4Only(_) => None, + IpTablesRedirector::Dual { ipv6_listener, .. } + | IpTablesRedirector::Ipv6Only(ipv6_listener) => Some(ipv6_listener), + } + } + + pub(crate) fn get_listeners_mut( + &mut self, + ) -> (Option<&mut IptablesListener>, Option<&mut IptablesListener>) { + match self { + IpTablesRedirector::Ipv4Only(ipv4_listener) => (Some(ipv4_listener), None), + IpTablesRedirector::Ipv6Only(ipv6_listener) => (None, Some(ipv6_listener)), + IpTablesRedirector::Dual { + ipv4_listener, + ipv6_listener, + } => (Some(ipv4_listener), Some(ipv6_listener)), + } } } @@ -194,33 +253,51 @@ impl PortRedirector for IpTablesRedirector { type Error = AgentError; async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { - if let Some(ip4_listener) = self.iptables_listener.as_mut() { - ip4_listener.add_redirect(from).await?; + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + ip4_listener.add_redirection(from).await?; } - if let Some(ip6_listener) = self.ip6tables_listener.as_mut() { - ip6_listener.add_redirect(from).await?; + if let Some(ip6_listener) = ipv6_listener { + ip6_listener.add_redirection(from).await?; } Ok(()) } async fn remove_redirection(&mut self, from: Port) -> Result<(), Self::Error> { - if let Some(iptables) = self.iptables.as_ref() { - iptables.remove_redirect(from, self.redirect_to).await?; + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + ip4_listener.remove_redirection(from).await?; + } + if let Some(ip6_listener) = ipv6_listener { + ip6_listener.remove_redirection(from).await?; } - Ok(()) } async fn cleanup(&mut self) -> Result<(), Self::Error> { - if let Some(iptables) = self.iptables.take() { - iptables.cleanup().await?; + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + ip4_listener.cleanup().await?; + } + if let Some(ip6_listener) = ipv6_listener { + ip6_listener.cleanup().await?; } - Ok(()) } async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error> { - self.listener.accept().await.map_err(Into::into) + match self { + Self::Dual { + ipv4_listener, + ipv6_listener, + } => { + select! { + con = ipv4_listener.next_connection() => con, + con = ipv6_listener.next_connection() => con, + } + } + Self::Ipv4Only(listener) | Self::Ipv6Only(listener) => listener.next_connection().await, + } } } From 6aff1c43669cb6e96196541a40d409aed40c028f Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 23 Dec 2024 10:05:59 +0200 Subject: [PATCH 19/72] use filter table for ipv6 --- mirrord/agent/src/steal/ip_tables.rs | 12 +++++++++--- mirrord/agent/src/steal/subscriptions.rs | 13 +++++++++++-- tests/src/utils.rs | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index a7e4765e896..3eb0d174448 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -76,6 +76,7 @@ pub static IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL: LazyLock = LazyLock::ne }); const IPTABLES_TABLE_NAME: &str = "nat"; +const IP6TABLES_TABLE_NAME: &str = "filter"; #[cfg_attr(test, allow(clippy::indexing_slicing))] // `mockall::automock` violates our clippy rules #[cfg_attr(test, mockall::automock)] @@ -112,8 +113,8 @@ pub fn new_iptables() -> iptables::IPTables { } /// wrapper around iptables::new that uses nft or legacy based on env -pub fn new_ip6tables() -> iptables::IPTables { - if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") +pub fn new_ip6tables_wrapper() -> IPTablesWrapper { + let ip6tables = if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") && val.to_lowercase() == "true" { // TODO: check if there is such a binary. @@ -122,7 +123,11 @@ pub fn new_ip6tables() -> iptables::IPTables { // TODO: check if there is such a binary. iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") } - .expect("IPTables initialization may not fail!") + .expect("IPTables initialization may not fail!"); + IPTablesWrapper { + table_name: IP6TABLES_TABLE_NAME, + tables: Arc::new(ip6tables), + } } impl Debug for IPTablesWrapper { @@ -219,6 +224,7 @@ pub(crate) enum Redirects { /// Wrapper struct for IPTables so it flushes on drop. pub(crate) struct SafeIpTables { redirect: Redirects, + ipv6: bool, } /// Wrapper for using iptables. This creates a a new chain on creation and deletes it on drop. diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 880627e4575..7ac6d7491d1 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -14,7 +14,7 @@ use tokio::{ use super::{ http::HttpFilter, - ip_tables::{new_iptables, IPTablesWrapper, SafeIpTables}, + ip_tables::{new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables}, }; use crate::{error::AgentError, util::ClientId}; @@ -64,6 +64,8 @@ pub(crate) struct IptablesListener { pod_ips: Option, /// Whether existing connections should be flushed when adding new redirects. flush_connections: bool, + /// Is this for connections incoming over IPv6 + ipv6: bool, } #[async_trait::async_trait] @@ -74,7 +76,11 @@ impl PortRedirector for IptablesListener { iptables } else { let safe = crate::steal::ip_tables::SafeIpTables::create( - new_iptables().into(), + if self.ipv6 { + new_iptables().into() + } else { + new_ip6tables_wrapper() + }, self.flush_connections, self.pod_ips.as_deref(), ) @@ -160,6 +166,7 @@ impl IpTablesRedirector { ) }, ); + tracing::debug!("pod IPv4 addresses: {pod_ips4:?}, pod IPv6 addresses: {pod_ips6:?}"); let listener4 = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await .inspect_err( @@ -180,6 +187,7 @@ impl IpTablesRedirector { listener, pod_ips: pod_ips4, flush_connections, + ipv6: false, }) }); let listener6 = if support_ipv6 { @@ -202,6 +210,7 @@ impl IpTablesRedirector { listener, pod_ips: pod_ips6, flush_connections, + ipv6: true, }) }) } else { diff --git a/tests/src/utils.rs b/tests/src/utils.rs index b4732d81348..f8b9a034eb2 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -597,7 +597,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2025-12-13", + "docker.io/t4lz/mirrord-agent:2024-12-22_2", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); From 3e1d7ab04be9028ef7f7699be42a95d145f0a765 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 23 Dec 2024 13:20:17 +0200 Subject: [PATCH 20/72] oh no --- mirrord/agent/src/entrypoint.rs | 14 +++-- mirrord/agent/src/steal/ip_tables.rs | 24 +++++--- mirrord/agent/src/steal/ip_tables/mesh.rs | 2 +- .../agent/src/steal/ip_tables/mesh/istio.rs | 2 +- .../agent/src/steal/ip_tables/prerouting.rs | 24 ++++++-- mirrord/agent/src/steal/ip_tables/standard.rs | 15 +++-- mirrord/agent/src/steal/subscriptions.rs | 60 +++++++++---------- 7 files changed, 80 insertions(+), 61 deletions(-) diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index fff9ee7192f..8ae2fe62502 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -39,9 +39,10 @@ use crate::{ sniffer::{api::TcpSnifferApi, messages::SnifferCommand, TcpConnectionSniffer}, steal::{ ip_tables::{ - new_iptables, IPTablesWrapper, SafeIpTables, IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL, - IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL_ENV, IPTABLE_MESH, IPTABLE_MESH_ENV, - IPTABLE_PREROUTING, IPTABLE_PREROUTING_ENV, IPTABLE_STANDARD, IPTABLE_STANDARD_ENV, + new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables, + IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL, IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL_ENV, + IPTABLE_MESH, IPTABLE_MESH_ENV, IPTABLE_PREROUTING, IPTABLE_PREROUTING_ENV, + IPTABLE_STANDARD, IPTABLE_STANDARD_ENV, }, StealerCommand, TcpConnectionStealer, TcpStealerApi, }, @@ -748,9 +749,12 @@ async fn start_agent(args: Args) -> Result<()> { } async fn clear_iptable_chain() -> Result<()> { - let ipt = new_iptables(); + SafeIpTables::<_, false>::load(IPTablesWrapper::from(new_iptables()), false) + .await? + .cleanup() + .await?; - SafeIpTables::load(IPTablesWrapper::from(ipt), false) + SafeIpTables::<_, true>::load(new_ip6tables_wrapper(), false) .await? .cleanup() .await?; diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 3eb0d174448..7d4975c9c4f 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -13,7 +13,7 @@ use crate::{ steal::ip_tables::{ flush_connections::FlushConnections, mesh::{istio::AmbientRedirect, MeshRedirect, MeshVendorExt}, - prerouting::PreroutingRedirect, + prerouting::{ChainName, PreroutingRedirect}, redirect::Redirect, standard::StandardRedirect, }, @@ -213,18 +213,23 @@ impl IPTables for IPTablesWrapper { } #[enum_dispatch(Redirect)] -pub(crate) enum Redirects { +pub(crate) enum Redirects +where + PreroutingRedirect: ChainName, +{ Ambient(AmbientRedirect), - Standard(StandardRedirect), + Standard(StandardRedirect), Mesh(MeshRedirect), - FlushConnections(FlushConnections>), - PrerouteFallback(PreroutingRedirect), + FlushConnections(FlushConnections>), + PrerouteFallback(PreroutingRedirect), } /// Wrapper struct for IPTables so it flushes on drop. -pub(crate) struct SafeIpTables { - redirect: Redirects, - ipv6: bool, +pub(crate) struct SafeIpTables +where + PreroutingRedirect: ChainName, +{ + redirect: Redirects, } /// Wrapper for using iptables. This creates a a new chain on creation and deletes it on drop. @@ -232,9 +237,10 @@ pub(crate) struct SafeIpTables { /// original chain (fallback) and adds a rule in the "PREROUTING" table that jumps to the new chain. /// Connections will go then PREROUTING -> OUR_CHAIN -> IF MATCH REDIRECT -> IF NOT MATCH FALLBACK /// -> ORIGINAL_CHAIN -impl SafeIpTables +impl SafeIpTables where IPT: IPTables + Send + Sync, + PreroutingRedirect: ChainName, { pub(super) async fn create( ipt: IPT, diff --git a/mirrord/agent/src/steal/ip_tables/mesh.rs b/mirrord/agent/src/steal/ip_tables/mesh.rs index 88fdff5d0b1..09997115472 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh.rs @@ -20,7 +20,7 @@ static TCP_SKIP_PORTS_LOOKUP_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"-p tcp -m tcp --dport ([\d:,]+)").unwrap()); pub(crate) struct MeshRedirect { - prerouting: PreroutingRedirect, + prerouting: PreroutingRedirect, output: OutputRedirect, vendor: MeshVendor, } diff --git a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs index cd3d4b06fa9..bb8ed6d8c29 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs @@ -12,7 +12,7 @@ use crate::{ }; pub(crate) struct AmbientRedirect { - prerouting: PreroutingRedirect, + prerouting: PreroutingRedirect, output: OutputRedirect, } diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 486b0ca1b51..cd853484705 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -8,16 +8,27 @@ use crate::{ steal::ip_tables::{chain::IPTableChain, IPTables, Redirect, IPTABLE_PREROUTING}, }; -pub(crate) struct PreroutingRedirect { +pub(crate) struct PreroutingRedirect { managed: IPTableChain, } -impl PreroutingRedirect +pub(crate) trait ChainName { + const ENTRYPOINT: &'static str; +} + +impl ChainName for PreroutingRedirect { + const ENTRYPOINT: &'static str = "PREROUTING"; +} + +impl ChainName for PreroutingRedirect { + const ENTRYPOINT: &'static str = "INPUT"; +} + +impl PreroutingRedirect where IPT: IPTables, + Self: ChainName, { - const ENTRYPOINT: &'static str = "PREROUTING"; - pub fn create(ipt: Arc) -> Result { let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; @@ -32,9 +43,10 @@ where } #[async_trait] -impl Redirect for PreroutingRedirect +impl Redirect for PreroutingRedirect where IPT: IPTables + Send + Sync, + Self: ChainName, { async fn mount_entrypoint(&self) -> Result<()> { self.managed.inner().add_rule( @@ -73,7 +85,7 @@ where } } -impl Deref for PreroutingRedirect +impl Deref for PreroutingRedirect where IPT: IPTables, { diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 3302b05c02e..1975f57565c 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -6,19 +6,21 @@ use mirrord_protocol::Port; use crate::{ error::Result, steal::ip_tables::{ - output::OutputRedirect, prerouting::PreroutingRedirect, IPTables, Redirect, - IPTABLE_STANDARD, + output::OutputRedirect, + prerouting::{ChainName, PreroutingRedirect}, + IPTables, Redirect, IPTABLE_STANDARD, }, }; -pub(crate) struct StandardRedirect { - prerouting: PreroutingRedirect, +pub(crate) struct StandardRedirect { + prerouting: PreroutingRedirect, output: OutputRedirect, } -impl StandardRedirect +impl StandardRedirect where IPT: IPTables, + PreroutingRedirect: ChainName, { pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { let prerouting = PreroutingRedirect::create(ipt.clone())?; @@ -38,9 +40,10 @@ where /// This wrapper adds a new rule to the NAT OUTPUT chain to redirect "localhost" traffic as well /// Note: OUTPUT chain is only traversed for packets produced by local applications #[async_trait] -impl Redirect for StandardRedirect +impl Redirect for StandardRedirect where IPT: IPTables + Send + Sync, + PreroutingRedirect: ChainName, { async fn mount_entrypoint(&self) -> Result<()> { self.prerouting.mount_entrypoint().await?; diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 7ac6d7491d1..8f43814e083 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -16,7 +16,11 @@ use super::{ http::HttpFilter, ip_tables::{new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables}, }; -use crate::{error::AgentError, util::ClientId}; +use crate::{ + error::AgentError, + steal::ip_tables::prerouting::{ChainName, PreroutingRedirect}, + util::ClientId, +}; /// For stealing incoming TCP connections. #[async_trait::async_trait] @@ -53,9 +57,12 @@ pub trait PortRedirector { /// A TCP listener, together with an iptables wrapper to set rules that send traffic to the /// listener. -pub(crate) struct IptablesListener { +pub(crate) struct IptablesListener +where + PreroutingRedirect<_, IPV6>: ChainName, +{ /// For altering iptables rules. - iptables: Option>, + iptables: Option>, /// Port of [`IpTablesRedirector::listener`]. redirect_to: Port, /// Listener to which redirect all connections. @@ -64,19 +71,20 @@ pub(crate) struct IptablesListener { pod_ips: Option, /// Whether existing connections should be flushed when adding new redirects. flush_connections: bool, - /// Is this for connections incoming over IPv6 - ipv6: bool, } #[async_trait::async_trait] -impl PortRedirector for IptablesListener { +impl PortRedirector for IptablesListener +where + PreroutingRedirect<_, IPV6>: ChainName, +{ type Error = AgentError; async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let iptables = if let Some(iptables) = self.iptables.as_ref() { iptables } else { let safe = crate::steal::ip_tables::SafeIpTables::create( - if self.ipv6 { + if IPV6 { new_iptables().into() } else { new_ip6tables_wrapper() @@ -116,12 +124,12 @@ impl PortRedirector for IptablesListener { /// /// Holds TCP listeners + iptables, for redirecting IPv4 and/or IPv6 connections. pub(crate) enum IpTablesRedirector { - Ipv4Only(IptablesListener), + Ipv4Only(IptablesListener), /// Could be used if IPv6 support is enabled, and we cannot bind an IPv4 address. - Ipv6Only(IptablesListener), + Ipv6Only(IptablesListener), Dual { - ipv4_listener: IptablesListener, - ipv6_listener: IptablesListener, + ipv4_listener: IptablesListener, + ipv6_listener: IptablesListener, }, } @@ -181,13 +189,12 @@ impl IpTablesRedirector { ) .ok()? .port(); - Some(IptablesListener { + Some(IptablesListener:: { iptables: None, redirect_to, listener, pod_ips: pod_ips4, flush_connections, - ipv6: false, }) }); let listener6 = if support_ipv6 { @@ -204,13 +211,12 @@ impl IpTablesRedirector { ) .ok()? .port(); - Some(IptablesListener { + Some(IptablesListener:: { iptables: None, redirect_to, listener, pod_ips: pod_ips6, flush_connections, - ipv6: true, }) }) } else { @@ -227,25 +233,12 @@ impl IpTablesRedirector { } } - pub(crate) fn get_ipv4_listener_mut(&mut self) -> Option<&mut IptablesListener> { - match self { - IpTablesRedirector::Ipv6Only(_) => None, - IpTablesRedirector::Dual { ipv4_listener, .. } - | IpTablesRedirector::Ipv4Only(ipv4_listener) => Some(ipv4_listener), - } - } - - pub(crate) fn get_ipv6_listener_mut(&mut self) -> Option<&mut IptablesListener> { - match self { - IpTablesRedirector::Ipv4Only(_) => None, - IpTablesRedirector::Dual { ipv6_listener, .. } - | IpTablesRedirector::Ipv6Only(ipv6_listener) => Some(ipv6_listener), - } - } - pub(crate) fn get_listeners_mut( &mut self, - ) -> (Option<&mut IptablesListener>, Option<&mut IptablesListener>) { + ) -> ( + Option<&mut IptablesListener>, + Option<&mut IptablesListener>, + ) { match self { IpTablesRedirector::Ipv4Only(ipv4_listener) => (Some(ipv4_listener), None), IpTablesRedirector::Ipv6Only(ipv6_listener) => (None, Some(ipv6_listener)), @@ -305,7 +298,8 @@ impl PortRedirector for IpTablesRedirector { con = ipv6_listener.next_connection() => con, } } - Self::Ipv4Only(listener) | Self::Ipv6Only(listener) => listener.next_connection().await, + Self::Ipv4Only(listener) => listener.next_connection().await, + Self::Ipv6Only(listener) => listener.next_connection().await, } } } From 1222eeeae1dcd31a540835ba8a81a8e6eb85fc2d Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 23 Dec 2024 13:20:30 +0200 Subject: [PATCH 21/72] Revert "oh no" This reverts commit 8fa0954f95c434497b2839db4483269fc2db5132. --- mirrord/agent/src/entrypoint.rs | 14 ++--- mirrord/agent/src/steal/ip_tables.rs | 24 +++----- mirrord/agent/src/steal/ip_tables/mesh.rs | 2 +- .../agent/src/steal/ip_tables/mesh/istio.rs | 2 +- .../agent/src/steal/ip_tables/prerouting.rs | 24 ++------ mirrord/agent/src/steal/ip_tables/standard.rs | 15 ++--- mirrord/agent/src/steal/subscriptions.rs | 60 ++++++++++--------- 7 files changed, 61 insertions(+), 80 deletions(-) diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index 8ae2fe62502..fff9ee7192f 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -39,10 +39,9 @@ use crate::{ sniffer::{api::TcpSnifferApi, messages::SnifferCommand, TcpConnectionSniffer}, steal::{ ip_tables::{ - new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables, - IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL, IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL_ENV, - IPTABLE_MESH, IPTABLE_MESH_ENV, IPTABLE_PREROUTING, IPTABLE_PREROUTING_ENV, - IPTABLE_STANDARD, IPTABLE_STANDARD_ENV, + new_iptables, IPTablesWrapper, SafeIpTables, IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL, + IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL_ENV, IPTABLE_MESH, IPTABLE_MESH_ENV, + IPTABLE_PREROUTING, IPTABLE_PREROUTING_ENV, IPTABLE_STANDARD, IPTABLE_STANDARD_ENV, }, StealerCommand, TcpConnectionStealer, TcpStealerApi, }, @@ -749,12 +748,9 @@ async fn start_agent(args: Args) -> Result<()> { } async fn clear_iptable_chain() -> Result<()> { - SafeIpTables::<_, false>::load(IPTablesWrapper::from(new_iptables()), false) - .await? - .cleanup() - .await?; + let ipt = new_iptables(); - SafeIpTables::<_, true>::load(new_ip6tables_wrapper(), false) + SafeIpTables::load(IPTablesWrapper::from(ipt), false) .await? .cleanup() .await?; diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 7d4975c9c4f..3eb0d174448 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -13,7 +13,7 @@ use crate::{ steal::ip_tables::{ flush_connections::FlushConnections, mesh::{istio::AmbientRedirect, MeshRedirect, MeshVendorExt}, - prerouting::{ChainName, PreroutingRedirect}, + prerouting::PreroutingRedirect, redirect::Redirect, standard::StandardRedirect, }, @@ -213,23 +213,18 @@ impl IPTables for IPTablesWrapper { } #[enum_dispatch(Redirect)] -pub(crate) enum Redirects -where - PreroutingRedirect: ChainName, -{ +pub(crate) enum Redirects { Ambient(AmbientRedirect), - Standard(StandardRedirect), + Standard(StandardRedirect), Mesh(MeshRedirect), - FlushConnections(FlushConnections>), - PrerouteFallback(PreroutingRedirect), + FlushConnections(FlushConnections>), + PrerouteFallback(PreroutingRedirect), } /// Wrapper struct for IPTables so it flushes on drop. -pub(crate) struct SafeIpTables -where - PreroutingRedirect: ChainName, -{ - redirect: Redirects, +pub(crate) struct SafeIpTables { + redirect: Redirects, + ipv6: bool, } /// Wrapper for using iptables. This creates a a new chain on creation and deletes it on drop. @@ -237,10 +232,9 @@ where /// original chain (fallback) and adds a rule in the "PREROUTING" table that jumps to the new chain. /// Connections will go then PREROUTING -> OUR_CHAIN -> IF MATCH REDIRECT -> IF NOT MATCH FALLBACK /// -> ORIGINAL_CHAIN -impl SafeIpTables +impl SafeIpTables where IPT: IPTables + Send + Sync, - PreroutingRedirect: ChainName, { pub(super) async fn create( ipt: IPT, diff --git a/mirrord/agent/src/steal/ip_tables/mesh.rs b/mirrord/agent/src/steal/ip_tables/mesh.rs index 09997115472..88fdff5d0b1 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh.rs @@ -20,7 +20,7 @@ static TCP_SKIP_PORTS_LOOKUP_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"-p tcp -m tcp --dport ([\d:,]+)").unwrap()); pub(crate) struct MeshRedirect { - prerouting: PreroutingRedirect, + prerouting: PreroutingRedirect, output: OutputRedirect, vendor: MeshVendor, } diff --git a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs index bb8ed6d8c29..cd3d4b06fa9 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs @@ -12,7 +12,7 @@ use crate::{ }; pub(crate) struct AmbientRedirect { - prerouting: PreroutingRedirect, + prerouting: PreroutingRedirect, output: OutputRedirect, } diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index cd853484705..486b0ca1b51 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -8,27 +8,16 @@ use crate::{ steal::ip_tables::{chain::IPTableChain, IPTables, Redirect, IPTABLE_PREROUTING}, }; -pub(crate) struct PreroutingRedirect { +pub(crate) struct PreroutingRedirect { managed: IPTableChain, } -pub(crate) trait ChainName { - const ENTRYPOINT: &'static str; -} - -impl ChainName for PreroutingRedirect { - const ENTRYPOINT: &'static str = "PREROUTING"; -} - -impl ChainName for PreroutingRedirect { - const ENTRYPOINT: &'static str = "INPUT"; -} - -impl PreroutingRedirect +impl PreroutingRedirect where IPT: IPTables, - Self: ChainName, { + const ENTRYPOINT: &'static str = "PREROUTING"; + pub fn create(ipt: Arc) -> Result { let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; @@ -43,10 +32,9 @@ where } #[async_trait] -impl Redirect for PreroutingRedirect +impl Redirect for PreroutingRedirect where IPT: IPTables + Send + Sync, - Self: ChainName, { async fn mount_entrypoint(&self) -> Result<()> { self.managed.inner().add_rule( @@ -85,7 +73,7 @@ where } } -impl Deref for PreroutingRedirect +impl Deref for PreroutingRedirect where IPT: IPTables, { diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 1975f57565c..3302b05c02e 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -6,21 +6,19 @@ use mirrord_protocol::Port; use crate::{ error::Result, steal::ip_tables::{ - output::OutputRedirect, - prerouting::{ChainName, PreroutingRedirect}, - IPTables, Redirect, IPTABLE_STANDARD, + output::OutputRedirect, prerouting::PreroutingRedirect, IPTables, Redirect, + IPTABLE_STANDARD, }, }; -pub(crate) struct StandardRedirect { - prerouting: PreroutingRedirect, +pub(crate) struct StandardRedirect { + prerouting: PreroutingRedirect, output: OutputRedirect, } -impl StandardRedirect +impl StandardRedirect where IPT: IPTables, - PreroutingRedirect: ChainName, { pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { let prerouting = PreroutingRedirect::create(ipt.clone())?; @@ -40,10 +38,9 @@ where /// This wrapper adds a new rule to the NAT OUTPUT chain to redirect "localhost" traffic as well /// Note: OUTPUT chain is only traversed for packets produced by local applications #[async_trait] -impl Redirect for StandardRedirect +impl Redirect for StandardRedirect where IPT: IPTables + Send + Sync, - PreroutingRedirect: ChainName, { async fn mount_entrypoint(&self) -> Result<()> { self.prerouting.mount_entrypoint().await?; diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 8f43814e083..7ac6d7491d1 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -16,11 +16,7 @@ use super::{ http::HttpFilter, ip_tables::{new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables}, }; -use crate::{ - error::AgentError, - steal::ip_tables::prerouting::{ChainName, PreroutingRedirect}, - util::ClientId, -}; +use crate::{error::AgentError, util::ClientId}; /// For stealing incoming TCP connections. #[async_trait::async_trait] @@ -57,12 +53,9 @@ pub trait PortRedirector { /// A TCP listener, together with an iptables wrapper to set rules that send traffic to the /// listener. -pub(crate) struct IptablesListener -where - PreroutingRedirect<_, IPV6>: ChainName, -{ +pub(crate) struct IptablesListener { /// For altering iptables rules. - iptables: Option>, + iptables: Option>, /// Port of [`IpTablesRedirector::listener`]. redirect_to: Port, /// Listener to which redirect all connections. @@ -71,20 +64,19 @@ where pod_ips: Option, /// Whether existing connections should be flushed when adding new redirects. flush_connections: bool, + /// Is this for connections incoming over IPv6 + ipv6: bool, } #[async_trait::async_trait] -impl PortRedirector for IptablesListener -where - PreroutingRedirect<_, IPV6>: ChainName, -{ +impl PortRedirector for IptablesListener { type Error = AgentError; async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let iptables = if let Some(iptables) = self.iptables.as_ref() { iptables } else { let safe = crate::steal::ip_tables::SafeIpTables::create( - if IPV6 { + if self.ipv6 { new_iptables().into() } else { new_ip6tables_wrapper() @@ -124,12 +116,12 @@ where /// /// Holds TCP listeners + iptables, for redirecting IPv4 and/or IPv6 connections. pub(crate) enum IpTablesRedirector { - Ipv4Only(IptablesListener), + Ipv4Only(IptablesListener), /// Could be used if IPv6 support is enabled, and we cannot bind an IPv4 address. - Ipv6Only(IptablesListener), + Ipv6Only(IptablesListener), Dual { - ipv4_listener: IptablesListener, - ipv6_listener: IptablesListener, + ipv4_listener: IptablesListener, + ipv6_listener: IptablesListener, }, } @@ -189,12 +181,13 @@ impl IpTablesRedirector { ) .ok()? .port(); - Some(IptablesListener:: { + Some(IptablesListener { iptables: None, redirect_to, listener, pod_ips: pod_ips4, flush_connections, + ipv6: false, }) }); let listener6 = if support_ipv6 { @@ -211,12 +204,13 @@ impl IpTablesRedirector { ) .ok()? .port(); - Some(IptablesListener:: { + Some(IptablesListener { iptables: None, redirect_to, listener, pod_ips: pod_ips6, flush_connections, + ipv6: true, }) }) } else { @@ -233,12 +227,25 @@ impl IpTablesRedirector { } } + pub(crate) fn get_ipv4_listener_mut(&mut self) -> Option<&mut IptablesListener> { + match self { + IpTablesRedirector::Ipv6Only(_) => None, + IpTablesRedirector::Dual { ipv4_listener, .. } + | IpTablesRedirector::Ipv4Only(ipv4_listener) => Some(ipv4_listener), + } + } + + pub(crate) fn get_ipv6_listener_mut(&mut self) -> Option<&mut IptablesListener> { + match self { + IpTablesRedirector::Ipv4Only(_) => None, + IpTablesRedirector::Dual { ipv6_listener, .. } + | IpTablesRedirector::Ipv6Only(ipv6_listener) => Some(ipv6_listener), + } + } + pub(crate) fn get_listeners_mut( &mut self, - ) -> ( - Option<&mut IptablesListener>, - Option<&mut IptablesListener>, - ) { + ) -> (Option<&mut IptablesListener>, Option<&mut IptablesListener>) { match self { IpTablesRedirector::Ipv4Only(ipv4_listener) => (Some(ipv4_listener), None), IpTablesRedirector::Ipv6Only(ipv6_listener) => (None, Some(ipv6_listener)), @@ -298,8 +305,7 @@ impl PortRedirector for IpTablesRedirector { con = ipv6_listener.next_connection() => con, } } - Self::Ipv4Only(listener) => listener.next_connection().await, - Self::Ipv6Only(listener) => listener.next_connection().await, + Self::Ipv4Only(listener) | Self::Ipv6Only(listener) => listener.next_connection().await, } } } From 08d02c03dbe955ef8d83d5aba0bb97031a5da571 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 23 Dec 2024 13:26:02 +0200 Subject: [PATCH 22/72] try with flush connections --- mirrord/agent/src/steal/subscriptions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 7ac6d7491d1..419f9222967 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -81,7 +81,7 @@ impl PortRedirector for IptablesListener { } else { new_ip6tables_wrapper() }, - self.flush_connections, + self.flush_connections || self.ipv6, self.pod_ips.as_deref(), ) .await?; From 2fc813831f4b7837849b930b3ba0ab2adeb3365a Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 24 Dec 2024 15:39:28 +0200 Subject: [PATCH 23/72] use input chain for IPv6 --- mirrord/agent/src/steal/ip_tables.rs | 8 ++-- mirrord/agent/src/steal/ip_tables/mesh.rs | 4 +- .../agent/src/steal/ip_tables/mesh/istio.rs | 4 +- .../agent/src/steal/ip_tables/prerouting.rs | 44 ++++++++++++++----- mirrord/agent/src/steal/ip_tables/standard.rs | 10 +++-- mirrord/agent/src/steal/subscriptions.rs | 1 + tests/src/utils.rs | 2 +- 7 files changed, 51 insertions(+), 22 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 3eb0d174448..8379ba0bc3d 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -224,7 +224,6 @@ pub(crate) enum Redirects { /// Wrapper struct for IPTables so it flushes on drop. pub(crate) struct SafeIpTables { redirect: Redirects, - ipv6: bool, } /// Wrapper for using iptables. This creates a a new chain on creation and deletes it on drop. @@ -240,6 +239,7 @@ where ipt: IPT, flush_connections: bool, pod_ips: Option<&str>, + ipv6: bool, ) -> Result { let ipt = Arc::new(ipt); @@ -251,11 +251,11 @@ where _ => Redirects::Mesh(MeshRedirect::create(ipt.clone(), vendor, pod_ips)?), } } else { - match StandardRedirect::create(ipt.clone(), pod_ips) { + match StandardRedirect::create(ipt.clone(), pod_ips, ipv6) { Err(err) => { warn!("Unable to create StandardRedirect chain: {err}"); - Redirects::PrerouteFallback(PreroutingRedirect::create(ipt.clone())?) + Redirects::PrerouteFallback(PreroutingRedirect::create_prerouting(ipt.clone())?) } Ok(standard) => Redirects::Standard(standard), } @@ -284,7 +284,7 @@ where Err(err) => { warn!("Unable to load StandardRedirect chain: {err}"); - Redirects::PrerouteFallback(PreroutingRedirect::load(ipt.clone())?) + Redirects::PrerouteFallback(PreroutingRedirect::load_prerouting(ipt.clone())?) } Ok(standard) => Redirects::Standard(standard), } diff --git a/mirrord/agent/src/steal/ip_tables/mesh.rs b/mirrord/agent/src/steal/ip_tables/mesh.rs index 88fdff5d0b1..0ddd4f77921 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh.rs @@ -30,7 +30,7 @@ where IPT: IPTables, { pub fn create(ipt: Arc, vendor: MeshVendor, pod_ips: Option<&str>) -> Result { - let prerouting = PreroutingRedirect::create(ipt.clone())?; + let prerouting = PreroutingRedirect::create_prerouting(ipt.clone())?; for port in Self::get_skip_ports(&ipt, &vendor)? { prerouting.add_rule(&format!("-m multiport -p tcp ! --dports {port} -j RETURN"))?; @@ -46,7 +46,7 @@ where } pub fn load(ipt: Arc, vendor: MeshVendor) -> Result { - let prerouting = PreroutingRedirect::load(ipt.clone())?; + let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; Ok(MeshRedirect { diff --git a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs index cd3d4b06fa9..a4e9a57a3e1 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs @@ -21,14 +21,14 @@ where IPT: IPTables, { pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { - let prerouting = PreroutingRedirect::create(ipt.clone())?; + let prerouting = PreroutingRedirect::create_prerouting(ipt.clone())?; let output = OutputRedirect::create(ipt, IPTABLE_MESH.to_string(), pod_ips)?; Ok(AmbientRedirect { prerouting, output }) } pub fn load(ipt: Arc) -> Result { - let prerouting = PreroutingRedirect::load(ipt.clone())?; + let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; Ok(AmbientRedirect { prerouting, output }) diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 486b0ca1b51..10a801bc5a3 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -10,24 +10,45 @@ use crate::{ pub(crate) struct PreroutingRedirect { managed: IPTableChain, + chain_name: &'static str, } impl PreroutingRedirect where IPT: IPTables, { - const ENTRYPOINT: &'static str = "PREROUTING"; + pub fn create_prerouting(ipt: Arc) -> Result { + Self::create(ipt, "PREROUTING") + } + + pub fn create_input(ipt: Arc) -> Result { + Self::create(ipt, "INPUT") + } - pub fn create(ipt: Arc) -> Result { + pub fn create(ipt: Arc, chain_name: &'static str) -> Result { let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; - Ok(PreroutingRedirect { managed }) + Ok(PreroutingRedirect { + managed, + chain_name, + }) + } + + pub fn load_prerouting(ipt: Arc) -> Result { + Self::load(ipt, "PREROUTING") + } + + pub fn load_input(ipt: Arc) -> Result { + Self::load(ipt, "INPUT") } - pub fn load(ipt: Arc) -> Result { + pub fn load(ipt: Arc, chain_name: &'static str) -> Result { let managed = IPTableChain::load(ipt, IPTABLE_PREROUTING.to_string())?; - Ok(PreroutingRedirect { managed }) + Ok(PreroutingRedirect { + managed, + chain_name, + }) } } @@ -38,7 +59,7 @@ where { async fn mount_entrypoint(&self) -> Result<()> { self.managed.inner().add_rule( - Self::ENTRYPOINT, + &self.chain_name, &format!("-j {}", self.managed.chain_name()), )?; @@ -47,7 +68,7 @@ where async fn unmount_entrypoint(&self) -> Result<()> { self.managed.inner().remove_rule( - Self::ENTRYPOINT, + &self.chain_name, &format!("-j {}", self.managed.chain_name()), )?; @@ -114,7 +135,8 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); + let prerouting = + PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.add_redirect(69, 420).await.is_ok()); } @@ -151,7 +173,8 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); + let prerouting = + PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.add_redirect(69, 420).await.is_ok()); assert!(prerouting.add_redirect(169, 1420).await.is_ok()); @@ -179,7 +202,8 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); + let prerouting = + PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.remove_redirect(69, 420).await.is_ok()); } diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 3302b05c02e..1c200b1a168 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -20,15 +20,19 @@ impl StandardRedirect where IPT: IPTables, { - pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { - let prerouting = PreroutingRedirect::create(ipt.clone())?; + pub fn create(ipt: Arc, pod_ips: Option<&str>, ipv6: bool) -> Result { + let prerouting = if ipv6 { + PreroutingRedirect::create_input(ipt.clone())? + } else { + PreroutingRedirect::create_prerouting(ipt.clone())? + }; let output = OutputRedirect::create(ipt, IPTABLE_STANDARD.to_string(), pod_ips)?; Ok(StandardRedirect { prerouting, output }) } pub fn load(ipt: Arc) -> Result { - let prerouting = PreroutingRedirect::load(ipt.clone())?; + let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_STANDARD.to_string())?; Ok(StandardRedirect { prerouting, output }) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 419f9222967..ddb7f9e1aba 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -83,6 +83,7 @@ impl PortRedirector for IptablesListener { }, self.flush_connections || self.ipv6, self.pod_ips.as_deref(), + self.ipv6, ) .await?; self.iptables.insert(safe) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index f8b9a034eb2..c9fc6ed5c83 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -597,7 +597,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-22_2", + "docker.io/t4lz/mirrord-agent:2024-12-23", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); From c5e85686601ce18a80a01ecf8f0b92c458c4dfb7 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 11:54:52 +0100 Subject: [PATCH 24/72] fix dumb bug (ip6tables command switch) --- mirrord/agent/src/steal/ip_tables.rs | 3 ++- mirrord/agent/src/steal/ip_tables/standard.rs | 1 + mirrord/agent/src/steal/subscriptions.rs | 12 +++++++++--- tests/src/utils.rs | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 8379ba0bc3d..7fdc7f3d464 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -159,7 +159,7 @@ impl IPTables for IPTablesWrapper { } } - #[tracing::instrument(level = "trace")] + #[tracing::instrument(level = "debug", skip(self), ret, fields(table_name=%self.table_name))] // TODO: change to trace. fn create_chain(&self, name: &str) -> Result<()> { self.tables .new_chain(self.table_name, name) @@ -251,6 +251,7 @@ where _ => Redirects::Mesh(MeshRedirect::create(ipt.clone(), vendor, pod_ips)?), } } else { + tracing::debug!(ipv6 = ipv6, "creating standard redirect"); // TODO: change to trace. match StandardRedirect::create(ipt.clone(), pod_ips, ipv6) { Err(err) => { warn!("Unable to create StandardRedirect chain: {err}"); diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 1c200b1a168..c6c104fb6ff 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -20,6 +20,7 @@ impl StandardRedirect where IPT: IPTables, { + #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] pub fn create(ipt: Arc, pod_ips: Option<&str>, ipv6: bool) -> Result { let prerouting = if ipv6 { PreroutingRedirect::create_input(ipt.clone())? diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index ddb7f9e1aba..21b3a5bc111 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -71,17 +71,19 @@ pub(crate) struct IptablesListener { #[async_trait::async_trait] impl PortRedirector for IptablesListener { type Error = AgentError; + + #[tracing::instrument(skip(self), err, level=tracing::Level::DEBUG, fields(self.ipv6 = %self.ipv6))] async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let iptables = if let Some(iptables) = self.iptables.as_ref() { iptables } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { - new_iptables().into() - } else { new_ip6tables_wrapper() + } else { + new_iptables().into() }, - self.flush_connections || self.ipv6, + self.flush_connections, self.pod_ips.as_deref(), self.ipv6, ) @@ -169,6 +171,7 @@ impl IpTablesRedirector { ); tracing::debug!("pod IPv4 addresses: {pod_ips4:?}, pod IPv6 addresses: {pod_ips6:?}"); + tracing::debug!("Creating IPv4 iptables redirection listener"); let listener4 = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await .inspect_err( |err| tracing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), @@ -191,6 +194,7 @@ impl IpTablesRedirector { ipv6: false, }) }); + tracing::debug!("Creating IPv6 iptables redirection listener"); let listener6 = if support_ipv6 { TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await .inspect_err( @@ -265,9 +269,11 @@ impl PortRedirector for IpTablesRedirector { async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); if let Some(ip4_listener) = ipv4_listener { + tracing::debug!("Adding IPv4 redirection from port {from}"); ip4_listener.add_redirection(from).await?; } if let Some(ip6_listener) = ipv6_listener { + tracing::debug!("Adding IPv6 redirection from port {from}"); ip6_listener.add_redirection(from).await?; } Ok(()) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index c9fc6ed5c83..10242d3f638 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -597,7 +597,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-23", + "docker.io/t4lz/mirrord-agent:2024-12-27_4", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); From 8fba85830a1dd38ca79666437a9a404fc411e221 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 12:06:00 +0100 Subject: [PATCH 25/72] add debug logs --- mirrord/agent/src/steal/ip_tables/output.rs | 1 + mirrord/agent/src/steal/ip_tables/prerouting.rs | 1 + tests/src/utils.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mirrord/agent/src/steal/ip_tables/output.rs b/mirrord/agent/src/steal/ip_tables/output.rs index 944bc26f95b..fa9c534d413 100644 --- a/mirrord/agent/src/steal/ip_tables/output.rs +++ b/mirrord/agent/src/steal/ip_tables/output.rs @@ -20,6 +20,7 @@ where { const ENTRYPOINT: &'static str = "OUTPUT"; + #[tracing::instrument(skip(ipt), level = "debug", ret)] // TODO: change to trace. pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> Result { let managed = IPTableChain::create(ipt, chain_name)?; diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 10a801bc5a3..8eb7a78f954 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -25,6 +25,7 @@ where Self::create(ipt, "INPUT") } + #[tracing::instrument(level = "debug", skip(ipt), ret)] pub fn create(ipt: Arc, chain_name: &'static str) -> Result { let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 10242d3f638..281b22024e6 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -597,7 +597,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-27_4", + "docker.io/t4lz/mirrord-agent:2024-12-30_1", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); From 65236e6fd6365248a704baf71b8ed258ca1a11e1 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 12:16:45 +0100 Subject: [PATCH 26/72] add debug logs --- mirrord/agent/src/steal/ip_tables.rs | 2 +- mirrord/agent/src/steal/ip_tables/output.rs | 6 ++++-- mirrord/agent/src/steal/ip_tables/prerouting.rs | 7 +++++-- mirrord/agent/src/steal/ip_tables/standard.rs | 1 + mirrord/agent/src/steal/subscriptions.rs | 2 +- tests/src/utils.rs | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 7fdc7f3d464..2d8d9817e08 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -301,7 +301,7 @@ where /// Adds the redirect rule to iptables. /// /// Used to redirect packets when mirrord incoming feature is set to `steal`. - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = tracing::Level::DEBUG, skip(self))] pub(super) async fn add_redirect( &self, redirected_port: Port, diff --git a/mirrord/agent/src/steal/ip_tables/output.rs b/mirrord/agent/src/steal/ip_tables/output.rs index fa9c534d413..bd045166eb6 100644 --- a/mirrord/agent/src/steal/ip_tables/output.rs +++ b/mirrord/agent/src/steal/ip_tables/output.rs @@ -20,9 +20,11 @@ where { const ENTRYPOINT: &'static str = "OUTPUT"; - #[tracing::instrument(skip(ipt), level = "debug", ret)] // TODO: change to trace. + #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] // TODO: change to trace. pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> Result { - let managed = IPTableChain::create(ipt, chain_name)?; + let managed = IPTableChain::create(ipt, chain_name.clone()).inspect_err( + |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), + )?; let exclude_source_ips = pod_ips .map(|pod_ips| format!("! -s {pod_ips}")) diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 8eb7a78f954..58a99187877 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -25,9 +25,11 @@ where Self::create(ipt, "INPUT") } - #[tracing::instrument(level = "debug", skip(ipt), ret)] + #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] // TODO: change to trace. pub fn create(ipt: Arc, chain_name: &'static str) -> Result { - let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; + let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string()).inspect_err( + |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), + )?; Ok(PreroutingRedirect { managed, @@ -76,6 +78,7 @@ where Ok(()) } + #[tracing::instrument(skip(self), level = tracing::Level::DEBUG)] // TODO: change to trace. async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { let redirect_rule = format!("-m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}"); diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index c6c104fb6ff..75bebe91e5a 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -61,6 +61,7 @@ where Ok(()) } + #[tracing::instrument(level = tracing::Level::DEBUG, skip(self), ret, err)] async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { self.prerouting .add_redirect(redirected_port, target_port) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 21b3a5bc111..cff1707fbf7 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -72,7 +72,7 @@ pub(crate) struct IptablesListener { impl PortRedirector for IptablesListener { type Error = AgentError; - #[tracing::instrument(skip(self), err, level=tracing::Level::DEBUG, fields(self.ipv6 = %self.ipv6))] + #[tracing::instrument(skip(self), err, ret, level=tracing::Level::DEBUG, fields(self.ipv6 = %self.ipv6))] async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { let iptables = if let Some(iptables) = self.iptables.as_ref() { iptables diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 281b22024e6..6c4d1b88c2e 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -597,7 +597,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-30_1", + "docker.io/t4lz/mirrord-agent:2024-12-30_3", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); From 5660ef6fc3abbb3d1460d313ecfead3caf38148f Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 21:36:01 +0100 Subject: [PATCH 27/72] revert some stuff --- mirrord/agent/src/steal/ip_tables.rs | 6 +-- mirrord/agent/src/steal/ip_tables/mesh.rs | 4 +- .../agent/src/steal/ip_tables/mesh/istio.rs | 4 +- .../agent/src/steal/ip_tables/prerouting.rs | 50 ++++--------------- mirrord/agent/src/steal/ip_tables/standard.rs | 12 ++--- 5 files changed, 21 insertions(+), 55 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 2d8d9817e08..d5eb3f1057f 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -252,11 +252,11 @@ where } } else { tracing::debug!(ipv6 = ipv6, "creating standard redirect"); // TODO: change to trace. - match StandardRedirect::create(ipt.clone(), pod_ips, ipv6) { + match StandardRedirect::create(ipt.clone(), pod_ips) { Err(err) => { warn!("Unable to create StandardRedirect chain: {err}"); - Redirects::PrerouteFallback(PreroutingRedirect::create_prerouting(ipt.clone())?) + Redirects::PrerouteFallback(PreroutingRedirect::create(ipt.clone())?) } Ok(standard) => Redirects::Standard(standard), } @@ -285,7 +285,7 @@ where Err(err) => { warn!("Unable to load StandardRedirect chain: {err}"); - Redirects::PrerouteFallback(PreroutingRedirect::load_prerouting(ipt.clone())?) + Redirects::PrerouteFallback(PreroutingRedirect::load(ipt.clone())?) } Ok(standard) => Redirects::Standard(standard), } diff --git a/mirrord/agent/src/steal/ip_tables/mesh.rs b/mirrord/agent/src/steal/ip_tables/mesh.rs index 0ddd4f77921..88fdff5d0b1 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh.rs @@ -30,7 +30,7 @@ where IPT: IPTables, { pub fn create(ipt: Arc, vendor: MeshVendor, pod_ips: Option<&str>) -> Result { - let prerouting = PreroutingRedirect::create_prerouting(ipt.clone())?; + let prerouting = PreroutingRedirect::create(ipt.clone())?; for port in Self::get_skip_ports(&ipt, &vendor)? { prerouting.add_rule(&format!("-m multiport -p tcp ! --dports {port} -j RETURN"))?; @@ -46,7 +46,7 @@ where } pub fn load(ipt: Arc, vendor: MeshVendor) -> Result { - let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; + let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; Ok(MeshRedirect { diff --git a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs index a4e9a57a3e1..cd3d4b06fa9 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs @@ -21,14 +21,14 @@ where IPT: IPTables, { pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { - let prerouting = PreroutingRedirect::create_prerouting(ipt.clone())?; + let prerouting = PreroutingRedirect::create(ipt.clone())?; let output = OutputRedirect::create(ipt, IPTABLE_MESH.to_string(), pod_ips)?; Ok(AmbientRedirect { prerouting, output }) } pub fn load(ipt: Arc) -> Result { - let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; + let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; Ok(AmbientRedirect { prerouting, output }) diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 58a99187877..486b0ca1b51 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -10,48 +10,24 @@ use crate::{ pub(crate) struct PreroutingRedirect { managed: IPTableChain, - chain_name: &'static str, } impl PreroutingRedirect where IPT: IPTables, { - pub fn create_prerouting(ipt: Arc) -> Result { - Self::create(ipt, "PREROUTING") - } - - pub fn create_input(ipt: Arc) -> Result { - Self::create(ipt, "INPUT") - } - - #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] // TODO: change to trace. - pub fn create(ipt: Arc, chain_name: &'static str) -> Result { - let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string()).inspect_err( - |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), - )?; - - Ok(PreroutingRedirect { - managed, - chain_name, - }) - } + const ENTRYPOINT: &'static str = "PREROUTING"; - pub fn load_prerouting(ipt: Arc) -> Result { - Self::load(ipt, "PREROUTING") - } + pub fn create(ipt: Arc) -> Result { + let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; - pub fn load_input(ipt: Arc) -> Result { - Self::load(ipt, "INPUT") + Ok(PreroutingRedirect { managed }) } - pub fn load(ipt: Arc, chain_name: &'static str) -> Result { + pub fn load(ipt: Arc) -> Result { let managed = IPTableChain::load(ipt, IPTABLE_PREROUTING.to_string())?; - Ok(PreroutingRedirect { - managed, - chain_name, - }) + Ok(PreroutingRedirect { managed }) } } @@ -62,7 +38,7 @@ where { async fn mount_entrypoint(&self) -> Result<()> { self.managed.inner().add_rule( - &self.chain_name, + Self::ENTRYPOINT, &format!("-j {}", self.managed.chain_name()), )?; @@ -71,14 +47,13 @@ where async fn unmount_entrypoint(&self) -> Result<()> { self.managed.inner().remove_rule( - &self.chain_name, + Self::ENTRYPOINT, &format!("-j {}", self.managed.chain_name()), )?; Ok(()) } - #[tracing::instrument(skip(self), level = tracing::Level::DEBUG)] // TODO: change to trace. async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { let redirect_rule = format!("-m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}"); @@ -139,8 +114,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = - PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); + let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.add_redirect(69, 420).await.is_ok()); } @@ -177,8 +151,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = - PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); + let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.add_redirect(69, 420).await.is_ok()); assert!(prerouting.add_redirect(169, 1420).await.is_ok()); @@ -206,8 +179,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let prerouting = - PreroutingRedirect::create_prerouting(Arc::new(mock)).expect("Unable to create"); + let prerouting = PreroutingRedirect::create(Arc::new(mock)).expect("Unable to create"); assert!(prerouting.remove_redirect(69, 420).await.is_ok()); } diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 75bebe91e5a..3302b05c02e 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -20,20 +20,15 @@ impl StandardRedirect where IPT: IPTables, { - #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] - pub fn create(ipt: Arc, pod_ips: Option<&str>, ipv6: bool) -> Result { - let prerouting = if ipv6 { - PreroutingRedirect::create_input(ipt.clone())? - } else { - PreroutingRedirect::create_prerouting(ipt.clone())? - }; + pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { + let prerouting = PreroutingRedirect::create(ipt.clone())?; let output = OutputRedirect::create(ipt, IPTABLE_STANDARD.to_string(), pod_ips)?; Ok(StandardRedirect { prerouting, output }) } pub fn load(ipt: Arc) -> Result { - let prerouting = PreroutingRedirect::load_prerouting(ipt.clone())?; + let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_STANDARD.to_string())?; Ok(StandardRedirect { prerouting, output }) @@ -61,7 +56,6 @@ where Ok(()) } - #[tracing::instrument(level = tracing::Level::DEBUG, skip(self), ret, err)] async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { self.prerouting .add_redirect(redirected_port, target_port) From 33501113ecc0230881278ec7ae96c090e6e8898b Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 23:35:20 +0100 Subject: [PATCH 28/72] use nat table in ip6tables --- mirrord/agent/src/steal/ip_tables.rs | 11 +++-------- mirrord/agent/src/steal/subscriptions.rs | 9 +++++---- tests/src/utils.rs | 3 ++- tests/src/utils/ipv6.rs | 2 ++ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index d5eb3f1057f..9d809032bb4 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -76,7 +76,6 @@ pub static IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL: LazyLock = LazyLock::ne }); const IPTABLES_TABLE_NAME: &str = "nat"; -const IP6TABLES_TABLE_NAME: &str = "filter"; #[cfg_attr(test, allow(clippy::indexing_slicing))] // `mockall::automock` violates our clippy rules #[cfg_attr(test, mockall::automock)] @@ -113,8 +112,8 @@ pub fn new_iptables() -> iptables::IPTables { } /// wrapper around iptables::new that uses nft or legacy based on env -pub fn new_ip6tables_wrapper() -> IPTablesWrapper { - let ip6tables = if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") +pub fn new_ip6tables() -> iptables::IPTables { + if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") && val.to_lowercase() == "true" { // TODO: check if there is such a binary. @@ -123,11 +122,7 @@ pub fn new_ip6tables_wrapper() -> IPTablesWrapper { // TODO: check if there is such a binary. iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") } - .expect("IPTables initialization may not fail!"); - IPTablesWrapper { - table_name: IP6TABLES_TABLE_NAME, - tables: Arc::new(ip6tables), - } + .expect("IPTables initialization may not fail!") } impl Debug for IPTablesWrapper { diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index cff1707fbf7..d385f152338 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -14,7 +14,7 @@ use tokio::{ use super::{ http::HttpFilter, - ip_tables::{new_ip6tables_wrapper, new_iptables, IPTablesWrapper, SafeIpTables}, + ip_tables::{new_ip6tables, new_iptables, IPTablesWrapper, SafeIpTables}, }; use crate::{error::AgentError, util::ClientId}; @@ -79,10 +79,11 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { - new_ip6tables_wrapper() + new_ip6tables() } else { - new_iptables().into() - }, + new_iptables() + } + .into(), self.flush_connections, self.pod_ips.as_deref(), self.ipv6, diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 6c4d1b88c2e..4313ec8cbbf 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -404,6 +404,7 @@ impl Application { Application::PythonFastApiHTTPIPv6 => { vec![ "uvicorn", + "--log-level=trace", // TODO: delete? "--port=80", "--host=::", "--app-dir=./python-e2e/", @@ -597,7 +598,7 @@ pub async fn run_exec( // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); base_env.insert( "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-30_3", + "docker.io/t4lz/mirrord-agent:2024-12-30_5", ); base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete base_env.insert("MIRRORD_CHECK_VERSION", "false"); diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs index 8e9aa6affdd..089cd0be7d4 100644 --- a/tests/src/utils/ipv6.rs +++ b/tests/src/utils/ipv6.rs @@ -59,6 +59,8 @@ pub async fn send_request_with_method( .body(Empty::::new()) .unwrap(); + println!("Request: {:?}", req); + let res = request_sender.send_request(req).await.unwrap(); println!("Response: {:?}", res); assert_eq!(res.status(), hyper::StatusCode::OK); From 1ce84bac47cc9774a1989c629c564e4ce4e856e5 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 30 Dec 2024 23:35:36 +0100 Subject: [PATCH 29/72] ipv6 manual test app --- tests/ipv6-app.yaml | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/ipv6-app.yaml diff --git a/tests/ipv6-app.yaml b/tests/ipv6-app.yaml new file mode 100644 index 00000000000..96446e516c8 --- /dev/null +++ b/tests/ipv6-app.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: py-serv-deployment + labels: + app: py-serv +spec: + replicas: 1 + selector: + matchLabels: + app: py-serv + template: + metadata: + labels: + app: py-serv + spec: + containers: + - name: py-serv + image: docker.io/t4lz/mirrord-pytest:1204 # TODO: change to metalbear's image. + ports: + - containerPort: 80 + env: + - name: MIRRORD_FAKE_VAR_FIRST + value: mirrord.is.running + - name: MIRRORD_FAKE_VAR_SECOND + value: "7777" + - name: HOST + value: "::" + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: py-serv + name: py-serv +spec: + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv6 + ports: + - port: 80 + protocol: TCP + targetPort: 80 + nodePort: 30000 + selector: + app: py-serv + sessionAffinity: None + type: NodePort From e730e4c4ec30c812a7c6d661a25df4b466804440 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 00:59:16 +0100 Subject: [PATCH 30/72] fix test request --- tests/src/utils.rs | 1 - tests/src/utils/ipv6.rs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 4313ec8cbbf..5045b1400ca 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -404,7 +404,6 @@ impl Application { Application::PythonFastApiHTTPIPv6 => { vec![ "uvicorn", - "--log-level=trace", // TODO: delete? "--port=80", "--host=::", "--app-dir=./python-e2e/", diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs index 089cd0be7d4..bdf5fa02b3d 100644 --- a/tests/src/utils/ipv6.rs +++ b/tests/src/utils/ipv6.rs @@ -56,6 +56,7 @@ pub async fn send_request_with_method( ) { let req = Request::builder() .method(method) + .header("Host", "::") .body(Empty::::new()) .unwrap(); From 1fcf661eb4029d9fd88481701b8c5f921c74edd0 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 09:33:03 +0100 Subject: [PATCH 31/72] fix doc? --- mirrord/agent/src/steal/subscriptions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index d385f152338..47ab01db4f8 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -56,7 +56,7 @@ pub trait PortRedirector { pub(crate) struct IptablesListener { /// For altering iptables rules. iptables: Option>, - /// Port of [`IpTablesRedirector::listener`]. + /// Port of [`listener`](Self::listener). redirect_to: Port, /// Listener to which redirect all connections. listener: TcpListener, From a560193960003022fea6d8de030cd7948268cb58 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 09:38:57 +0100 Subject: [PATCH 32/72] thanks clippy --- mirrord/layer/src/socket/ops.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mirrord/layer/src/socket/ops.rs b/mirrord/layer/src/socket/ops.rs index 58930ea0b41..efc0095c8a4 100644 --- a/mirrord/layer/src/socket/ops.rs +++ b/mirrord/layer/src/socket/ops.rs @@ -130,10 +130,8 @@ pub(super) fn socket(domain: c_int, type_: c_int, protocol: c_int) -> Detour Date: Tue, 31 Dec 2024 09:50:43 +0100 Subject: [PATCH 33/72] ignore ipv6 test --- tests/src/traffic/steal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index ea07b8cf6c0..a334d34f9ed 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -71,7 +71,7 @@ mod steal_tests { application.assert(&process).await; } - #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] + #[ignore] // Needs special cluster setup, so ignore by default. #[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[timeout(Duration::from_secs(240))] From 5c80d23772715d157ca045ee82d6cca56794e63f Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 09:52:17 +0100 Subject: [PATCH 34/72] fix config test --- mirrord/config/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index 8ffd1d48c29..055adac98d2 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -878,6 +878,7 @@ mod tests { udp: Some(false), ..Default::default() })), + ipv6: None, })), copy_target: None, hostname: None, From 8d634f9598835033bb742aa51156c00857cfd92a Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 09:56:58 +0100 Subject: [PATCH 35/72] cfg test for ipv6 utils --- tests/src/utils/ipv6.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs index bdf5fa02b3d..a18d15c53e2 100644 --- a/tests/src/utils/ipv6.rs +++ b/tests/src/utils/ipv6.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use http_body_util::{BodyExt, Empty}; use hyper::{ client::{conn, conn::http1::SendRequest}, From e7144d72e92147cb7fc5f77fc3b8c9b82d92ca89 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 09:58:09 +0100 Subject: [PATCH 36/72] easy way out --- tests/src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 5045b1400ca..4dffd88b4f5 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -1142,6 +1142,7 @@ pub async fn service_with_env( /// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. /// * `randomize_name` - whether a random suffix should be added to the end of the resource names /// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. +#[allow(clippy::too_many_arguments)] async fn internal_service( namespace: &str, service_type: &str, From 065a5160a082ae3fd0feffdf0b058aa6c142560f Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 10:19:52 +0100 Subject: [PATCH 37/72] fix tests utils --- tests/src/utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 4dffd88b4f5..0979cfa7932 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -1497,13 +1497,13 @@ pub async fn service_for_mirrord_ls( .unwrap(); watch_resource_exists(&job_api, &name).await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1515,7 +1515,6 @@ pub async fn service_for_mirrord_ls( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), guards: vec![ pod_guard, service_guard, @@ -1524,6 +1523,7 @@ pub async fn service_for_mirrord_ls( job_guard, ], namespace_guard, + pod_name, } } From 227c1f0e0c7ec8cadd16ad5f3e176db3f48ad0d9 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 10:26:52 +0100 Subject: [PATCH 38/72] ipv6 support default to false --- mirrord/config/src/feature/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index f901a3f6b97..227bd82a915 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -73,7 +73,7 @@ pub struct NetworkConfig { /// ### feature.network.ipv6 {#feature-network-dns} /// /// Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. - #[config(env = IPV6_ENV_VAR)] + #[config(env = IPV6_ENV_VAR, default = false)] pub ipv6: bool, } From 06e432e8f90803235381b265d2d8ae82f0f371c3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 10:30:06 +0100 Subject: [PATCH 39/72] fix iptables tests --- mirrord/agent/src/steal/ip_tables.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 9d809032bb4..e70ae40dc0b 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -424,7 +424,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let ipt = SafeIpTables::create(mock, false, None) + let ipt = SafeIpTables::create(mock, false, None, false) .await .expect("Create Failed"); @@ -557,7 +557,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let ipt = SafeIpTables::create(mock, false, None) + let ipt = SafeIpTables::create(mock, false, None, false) .await .expect("Create Failed"); From b11208deb0da6067bd6fe10479d8f84d9e5fc065 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 10:31:07 +0100 Subject: [PATCH 40/72] remove unused methods --- mirrord/agent/src/steal/subscriptions.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 47ab01db4f8..0468719bc9c 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -233,22 +233,6 @@ impl IpTablesRedirector { } } - pub(crate) fn get_ipv4_listener_mut(&mut self) -> Option<&mut IptablesListener> { - match self { - IpTablesRedirector::Ipv6Only(_) => None, - IpTablesRedirector::Dual { ipv4_listener, .. } - | IpTablesRedirector::Ipv4Only(ipv4_listener) => Some(ipv4_listener), - } - } - - pub(crate) fn get_ipv6_listener_mut(&mut self) -> Option<&mut IptablesListener> { - match self { - IpTablesRedirector::Ipv4Only(_) => None, - IpTablesRedirector::Dual { ipv6_listener, .. } - | IpTablesRedirector::Ipv6Only(ipv6_listener) => Some(ipv6_listener), - } - } - pub(crate) fn get_listeners_mut( &mut self, ) -> (Option<&mut IptablesListener>, Option<&mut IptablesListener>) { From 71c6d02cb33f1620cdf5e403e0d1847fc220eeb2 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 11:21:48 +0100 Subject: [PATCH 41/72] fix policies test --- tests/src/operator/policies.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/operator/policies.rs b/tests/src/operator/policies.rs index 2b0a5d05e4c..3684a97fc20 100644 --- a/tests/src/operator/policies.rs +++ b/tests/src/operator/policies.rs @@ -355,7 +355,7 @@ async fn run_mirrord_and_verify_mirror_result(kube_service: &KubeService, expect let test_proc = application .run( - &kube_service.target, + &kube_service.pod_container_target(), Some(&kube_service.namespace), Some(vec!["--fs-mode=local"]), None, From 14703809ca16b3914aa2837ea021707db175f268 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 11:23:47 +0100 Subject: [PATCH 42/72] update schema --- mirrord-schema.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mirrord-schema.json b/mirrord-schema.json index 82a1538b9a7..46306a4ed28 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -1268,7 +1268,7 @@ }, "IncomingFileConfig": { "title": "incoming (network)", - "description": "Controls the incoming TCP traffic feature.\n\nSee the incoming [reference](https://mirrord.dev/docs/reference/traffic/#incoming) for more details.\n\nIncoming traffic supports 2 modes of operation:\n\n1. Mirror (**default**): Sniffs the TCP data from a port, and forwards a copy to the interested listeners;\n\n2. Steal: Captures the TCP data from a port, and forwards it to the local process, see [`steal`](##steal);\n\n### Minimal `incoming` config\n\n```json { \"feature\": { \"network\": { \"incoming\": \"steal\" } } } ```\n\n### Advanced `incoming` config\n\n```json { \"feature\": { \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_filter\": { \"header_filter\": \"host: api\\\\..+\" }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] \"listen_ports\": [[80, 8111]] } } } } ```", + "description": "Controls the incoming TCP traffic feature.\n\nSee the incoming [reference](https://mirrord.dev/docs/reference/traffic/#incoming) for more details.\n\nIncoming traffic supports 2 modes of operation:\n\n1. Mirror (**default**): Sniffs the TCP data from a port, and forwards a copy to the interested listeners;\n\n2. Steal: Captures the TCP data from a port, and forwards it to the local process, see [`steal`](##steal);\n\n### Minimal `incoming` config\n\n```json { \"feature\": { \"network\": { \"incoming\": \"steal\" } } } ```\n\n### Advanced `incoming` config\n\n```json { \"feature\": { \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_filter\": { \"header_filter\": \"host: api\\\\..+\" }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000], \"listen_ports\": [[80, 8111]] } } } } ```", "anyOf": [ { "anyOf": [ @@ -1474,6 +1474,14 @@ } ] }, + "ipv6": { + "title": "feature.network.ipv6 {#feature-network-dns}", + "description": "Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6.", + "type": [ + "boolean", + "null" + ] + }, "outgoing": { "title": "feature.network.outgoing {#feature-network-outgoing}", "anyOf": [ From 14d98539c3f682b14f7e52501ed316f76d3cdd42 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 11:24:37 +0100 Subject: [PATCH 43/72] run medschool --- mirrord/config/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md index 93dab79c3c0..3a3f8b4fa57 100644 --- a/mirrord/config/configuration.md +++ b/mirrord/config/configuration.md @@ -1266,6 +1266,10 @@ List of ports to mirror/steal traffic from. Other ports will remain local. Mutually exclusive with [`feature.network.incoming.ignore_ports`](#feature-network-ignore_ports). +### feature.network.ipv6 {#feature-network-dns} + +Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. + ### feature.network.outgoing {#feature-network-outgoing} Tunnel outgoing network operations through mirrord. From d7566446ae046a891611b69a7c1a8d3dcea811cd Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 31 Dec 2024 11:40:13 +0100 Subject: [PATCH 44/72] fix kube UT --- mirrord/kube/src/api/container/job.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mirrord/kube/src/api/container/job.rs b/mirrord/kube/src/api/container/job.rs index 63e072935b0..d9958e6620b 100644 --- a/mirrord/kube/src/api/container/job.rs +++ b/mirrord/kube/src/api/container/job.rs @@ -241,13 +241,14 @@ mod test { fn targetless() -> Result<(), Box> { let mut config_context = ConfigContext::default(); let agent = AgentFileConfig::default().generate_config(&mut config_context)?; + let support_ipv6 = false; let params = ContainerParams { name: "foobar".to_string(), port: 3000, gid: 13, tls_cert: None, pod_ips: None, - support_ipv6: false, + support_ipv6, }; let update = JobVariant::new(&agent, ¶ms).as_update(); @@ -299,7 +300,8 @@ mod test { { "name": "RUST_LOG", "value": agent.log_level }, { "name": "MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "value": agent.flush_connections.to_string() }, { "name": "MIRRORD_AGENT_NFTABLES", "value": agent.nftables.to_string() }, - { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) } + { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) }, + { "name": "MIRRORD_AGENT_SUPPORT_IPV6", "value": Some(support_ipv6.to_string()) } ], "resources": // Add requests to avoid getting defaulted https://github.com/metalbear-co/mirrord/issues/579 @@ -331,13 +333,14 @@ mod test { fn targeted() -> Result<(), Box> { let mut config_context = ConfigContext::default(); let agent = AgentFileConfig::default().generate_config(&mut config_context)?; + let support_ipv6 = false; let params = ContainerParams { name: "foobar".to_string(), port: 3000, gid: 13, tls_cert: None, pod_ips: None, - support_ipv6: false, + support_ipv6, }; let update = JobTargetedVariant::new( @@ -434,7 +437,8 @@ mod test { { "name": "RUST_LOG", "value": agent.log_level }, { "name": "MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "value": agent.flush_connections.to_string() }, { "name": "MIRRORD_AGENT_NFTABLES", "value": agent.nftables.to_string() }, - { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) } + { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) }, + { "name": "MIRRORD_AGENT_SUPPORT_IPV6", "value": Some(support_ipv6.to_string()) } ], "resources": // Add requests to avoid getting defaulted https://github.com/metalbear-co/mirrord/issues/579 { From eeba1f380c88d0fe90dacb5db3be49cb03742df1 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 13:29:07 +0100 Subject: [PATCH 45/72] use test image agent --- tests/src/utils.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 0979cfa7932..bd4520e82ec 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -593,13 +593,8 @@ pub async fn run_exec( // docker build -t test . -f mirrord/agent/Dockerfile // minikube load image test:latest let mut base_env = HashMap::new(); - // TODO: revert - // base_env.insert("MIRRORD_AGENT_IMAGE", "test"); - base_env.insert( - "MIRRORD_AGENT_IMAGE", - "docker.io/t4lz/mirrord-agent:2024-12-30_5", - ); - base_env.insert("MIRRORD_AGENT_TTL", "180"); // TODO: delete + base_env.insert("MIRRORD_AGENT_IMAGE", "test"); + // base_env.insert("MIRRORD_AGENT_TTL", "180"); // Uncomment for getting logs after failed tests base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From f87e7f30f1a53dbef74e09362ee1c41b5de335b3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 13:31:14 +0100 Subject: [PATCH 46/72] changelog --- changelog.d/2956.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2956.added.md diff --git a/changelog.d/2956.added.md b/changelog.d/2956.added.md new file mode 100644 index 00000000000..5cadfd68a24 --- /dev/null +++ b/changelog.d/2956.added.md @@ -0,0 +1 @@ +Support for stealing incoming connections that are over IPv6. From 22d3c026e4a76f2cdbb0ddd11a9afdf42c01ad4b Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 13:38:08 +0100 Subject: [PATCH 47/72] use published test image again --- tests/ipv6-app.yaml | 2 +- tests/src/utils/ipv6.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ipv6-app.yaml b/tests/ipv6-app.yaml index 96446e516c8..9a044cbde4d 100644 --- a/tests/ipv6-app.yaml +++ b/tests/ipv6-app.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: py-serv - image: docker.io/t4lz/mirrord-pytest:1204 # TODO: change to metalbear's image. + image: ghcr.io/metalbear-co/mirrord-pytest:latest ports: - containerPort: 80 env: diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs index a18d15c53e2..4b766e7f42d 100644 --- a/tests/src/utils/ipv6.rs +++ b/tests/src/utils/ipv6.rs @@ -21,8 +21,7 @@ use crate::utils::{internal_service, kube_client, KubeService}; pub async fn ipv6_service( #[default("default")] namespace: &str, #[default("NodePort")] service_type: &str, - // #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, - #[default("docker.io/t4lz/mirrord-pytest:1204")] image: &str, + #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, #[default("http-echo")] service_name: &str, #[default(true)] randomize_name: bool, #[future] kube_client: Client, From ed0cd1c9a090ba4e9251c835e3793f57f2d003b3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 13:55:32 +0100 Subject: [PATCH 48/72] TODOs --- mirrord/agent/src/steal/ip_tables.rs | 6 ++---- mirrord/agent/src/steal/ip_tables/output.rs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index e70ae40dc0b..68bddb6a406 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -116,10 +116,8 @@ pub fn new_ip6tables() -> iptables::IPTables { if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") && val.to_lowercase() == "true" { - // TODO: check if there is such a binary. iptables::new_with_cmd("/usr/sbin/ip6tables-nft") } else { - // TODO: check if there is such a binary. iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") } .expect("IPTables initialization may not fail!") @@ -154,7 +152,7 @@ impl IPTables for IPTablesWrapper { } } - #[tracing::instrument(level = "debug", skip(self), ret, fields(table_name=%self.table_name))] // TODO: change to trace. + #[tracing::instrument(level = tracing::Level::TRACE, skip(self), ret, fields(table_name=%self.table_name))] fn create_chain(&self, name: &str) -> Result<()> { self.tables .new_chain(self.table_name, name) @@ -246,7 +244,7 @@ where _ => Redirects::Mesh(MeshRedirect::create(ipt.clone(), vendor, pod_ips)?), } } else { - tracing::debug!(ipv6 = ipv6, "creating standard redirect"); // TODO: change to trace. + tracing::trace!(ipv6 = ipv6, "creating standard redirect"); match StandardRedirect::create(ipt.clone(), pod_ips) { Err(err) => { warn!("Unable to create StandardRedirect chain: {err}"); diff --git a/mirrord/agent/src/steal/ip_tables/output.rs b/mirrord/agent/src/steal/ip_tables/output.rs index bd045166eb6..2286469c00c 100644 --- a/mirrord/agent/src/steal/ip_tables/output.rs +++ b/mirrord/agent/src/steal/ip_tables/output.rs @@ -20,7 +20,7 @@ where { const ENTRYPOINT: &'static str = "OUTPUT"; - #[tracing::instrument(skip(ipt), level = tracing::Level::DEBUG)] // TODO: change to trace. + #[tracing::instrument(skip(ipt), level = tracing::Level::TRACE)] pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> Result { let managed = IPTableChain::create(ipt, chain_name.clone()).inspect_err( |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), From 9d198513c57306dfbfca80a710deb599a12e7f71 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 15:07:58 +0100 Subject: [PATCH 49/72] add ipv6 test to CI --- .github/workflows/ci.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 54115cef62f..f30c19a2c6d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -551,6 +551,30 @@ jobs: kubectl describe pods docker exec minikube find /var/log/pods -print -exec cat {} \; + ipv6_e2e_tests: + runs-on: ubuntu-24.04 + name: e2e + needs: [ test_agent_image, changed_files ] + if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' || needs.changed_files.outputs.dockerfile_changed == 'true'}} + env: + MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug" + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust. + - uses: actions/setup-python@v3 + - run: pip3 install fastapi uvicorn[standard] + shell: bash + - name: download image + uses: actions/download-artifact@v4 + with: + name: test + path: /tmp + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1 + - run: kind load image-archive /tmp/test.tar + - run: | + cargo test --ignored --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --test-threads=6 + lint_markdown: runs-on: ubuntu-24.04 needs: changed_files @@ -620,6 +644,7 @@ jobs: integration_tests, e2e, test_agent, + ipv6_e2e_tests, lint, lint_markdown, check-rust-docs, @@ -637,6 +662,7 @@ jobs: (needs.macos_tests.result == 'success' || needs.macos_tests.result == 'skipped') && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') && (needs.e2e.result == 'success' || needs.e2e.result == 'skipped') && + (needs.ipv6_e2e_tests.result == 'success' || needs.ipv6_e2e_tests.result == 'skipped') && (needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') && From 7f5e5e540f0565ce662f205494ab344134012563 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 15:29:57 +0100 Subject: [PATCH 50/72] add kind cluster config for IPv6 --- .github/workflows/ci.yaml | 2 ++ tests/kind-cluster-ipv6-config.yaml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 tests/kind-cluster-ipv6-config.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f30c19a2c6d..e3ea2a4cbe7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -571,6 +571,8 @@ jobs: path: /tmp - name: Create k8s Kind Cluster uses: helm/kind-action@v1 + with: + config: tests/kind-cluster-ipv6-config.yaml - run: kind load image-archive /tmp/test.tar - run: | cargo test --ignored --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --test-threads=6 diff --git a/tests/kind-cluster-ipv6-config.yaml b/tests/kind-cluster-ipv6-config.yaml new file mode 100644 index 00000000000..17bf7b7635d --- /dev/null +++ b/tests/kind-cluster-ipv6-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +ipFamily: ipv6 +apiServerAddress: 127.0.0.1 \ No newline at end of file From 14afc5bd3a661568d0d27df03b449de59f0adbac Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 22:31:31 +0100 Subject: [PATCH 51/72] fix cluster config --- CONTRIBUTING.md | 4 ++-- tests/kind-cluster-ipv6-config.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fbf717d7069..1b99ddc4c65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -124,8 +124,8 @@ In order to test IPv6 on a local cluster on macOS, you can use Kind: kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: - ipFamily: ipv6 - apiServerAddress: 127.0.0.1 + ipFamily: ipv6 + apiServerAddress: 127.0.0.1 EOF ``` 3. `kind create cluster --config kind-config.yaml` diff --git a/tests/kind-cluster-ipv6-config.yaml b/tests/kind-cluster-ipv6-config.yaml index 17bf7b7635d..2ad5f7f8b3c 100644 --- a/tests/kind-cluster-ipv6-config.yaml +++ b/tests/kind-cluster-ipv6-config.yaml @@ -1,5 +1,5 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: -ipFamily: ipv6 -apiServerAddress: 127.0.0.1 \ No newline at end of file + ipFamily: ipv6 + apiServerAddress: 127.0.0.1 From 441ad714cd8738034536d22fb4c05c4f37ae5ff8 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 22:37:25 +0100 Subject: [PATCH 52/72] CI IPv6 job name --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3ea2a4cbe7..1a6c136bf8e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -553,7 +553,6 @@ jobs: ipv6_e2e_tests: runs-on: ubuntu-24.04 - name: e2e needs: [ test_agent_image, changed_files ] if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' || needs.changed_files.outputs.dockerfile_changed == 'true'}} env: From 5ddc69fbbd532a61af9ab76c1c4732c534c96895 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 23:03:20 +0100 Subject: [PATCH 53/72] patch kind config to fix fail --- tests/kind-cluster-ipv6-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kind-cluster-ipv6-config.yaml b/tests/kind-cluster-ipv6-config.yaml index 2ad5f7f8b3c..29898284d86 100644 --- a/tests/kind-cluster-ipv6-config.yaml +++ b/tests/kind-cluster-ipv6-config.yaml @@ -3,3 +3,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: ipv6 apiServerAddress: 127.0.0.1 +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" From 5a68d7b95504d50be5d45a874daf3a9c5eb29b51 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 23:20:07 +0100 Subject: [PATCH 54/72] use kind bash script --- .github/workflows/ci.yaml | 7 +- .../workflows/kind-with-registry-and-ipv6.sh | 71 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100755 .github/workflows/kind-with-registry-and-ipv6.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1a6c136bf8e..a4818aeb993 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -569,9 +569,10 @@ jobs: name: test path: /tmp - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 - with: - config: tests/kind-cluster-ipv6-config.yaml + # uses: helm/kind-action@v1 + # with: + # config: tests/kind-cluster-ipv6-config.yaml + run: .github/workflows/kind-with-registry-and-ipv6.sh - run: kind load image-archive /tmp/test.tar - run: | cargo test --ignored --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --test-threads=6 diff --git a/.github/workflows/kind-with-registry-and-ipv6.sh b/.github/workflows/kind-with-registry-and-ipv6.sh new file mode 100755 index 00000000000..334e9013438 --- /dev/null +++ b/.github/workflows/kind-with-registry-and-ipv6.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +# taken from https://kind.sigs.k8s.io/docs/user/local-registry/ + +set -o errexit + + +# 1. Create registry container unless it already exists +reg_name='kind-registry' +reg_port='5001' +if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \ + registry:2 +fi + +# 2. Create kind cluster with containerd registry config dir enabled +# TODO: kind will eventually enable this by default and this patch will +# be unnecessary. +# +# See: +# https://github.com/kubernetes-sigs/kind/issues/2875 +# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration +# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md +cat < Date: Fri, 3 Jan 2025 23:32:49 +0100 Subject: [PATCH 55/72] fix cargo test command --- .github/workflows/ci.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a4818aeb993..e9c098f19dd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -569,13 +569,10 @@ jobs: name: test path: /tmp - name: Create k8s Kind Cluster - # uses: helm/kind-action@v1 - # with: - # config: tests/kind-cluster-ipv6-config.yaml run: .github/workflows/kind-with-registry-and-ipv6.sh - run: kind load image-archive /tmp/test.tar - run: | - cargo test --ignored --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --test-threads=6 + cargo test --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --ignored --test-threads=6 lint_markdown: runs-on: ubuntu-24.04 From dcdf89fc69f76d2d06274ad27e278b7ea360cba0 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 3 Jan 2025 23:41:11 +0100 Subject: [PATCH 56/72] agent logs? --- .github/workflows/ci.yaml | 6 ++++++ tests/src/utils.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e9c098f19dd..71b866cfdcc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -573,6 +573,12 @@ jobs: - run: kind load image-archive /tmp/test.tar - run: | cargo test --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --ignored --test-threads=6 + - name: fetch agent logs + if: ${{ failure() }} + run: | + kubectl logs -l app==mirrord --tail -1 + kubectl get all + kubectl describe pods lint_markdown: runs-on: ubuntu-24.04 diff --git a/tests/src/utils.rs b/tests/src/utils.rs index bd4520e82ec..11b57e90773 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -594,7 +594,7 @@ pub async fn run_exec( // minikube load image test:latest let mut base_env = HashMap::new(); base_env.insert("MIRRORD_AGENT_IMAGE", "test"); - // base_env.insert("MIRRORD_AGENT_TTL", "180"); // Uncomment for getting logs after failed tests + base_env.insert("MIRRORD_AGENT_TTL", "20"); base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From b40975496fa7168939b51ceee654350732f0c3da Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 10:56:01 +0100 Subject: [PATCH 57/72] maybe with a longer TTL I'll get some logs? --- tests/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 11b57e90773..e22f0c53fa3 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -594,7 +594,7 @@ pub async fn run_exec( // minikube load image test:latest let mut base_env = HashMap::new(); base_env.insert("MIRRORD_AGENT_IMAGE", "test"); - base_env.insert("MIRRORD_AGENT_TTL", "20"); + base_env.insert("MIRRORD_AGENT_TTL", "30"); base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From 42c04ed99192b0e6a9a2ec3c654838d55e891a7d Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 11:02:01 +0100 Subject: [PATCH 58/72] print intproxy logs on failure --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71b866cfdcc..2a212e32fbb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -579,6 +579,7 @@ jobs: kubectl logs -l app==mirrord --tail -1 kubectl get all kubectl describe pods + cat /tmp/mirrord-intproxy-*.log lint_markdown: runs-on: ubuntu-24.04 From c2559c5ed4d9d25e9fc07d8c946c453ed051959f Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 11:33:35 +0100 Subject: [PATCH 59/72] show nodes on failure --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a212e32fbb..87149b9be8c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -580,6 +580,7 @@ jobs: kubectl get all kubectl describe pods cat /tmp/mirrord-intproxy-*.log + kubectl get nodes -o wide lint_markdown: runs-on: ubuntu-24.04 From c119c88b7682f7866a5d18f83694790fcd16043d Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 12:24:12 +0100 Subject: [PATCH 60/72] modprobe? --- .github/workflows/ci.yaml | 2 +- Cargo.lock | 7 +++++++ mirrord/agent/Cargo.toml | 1 + mirrord/agent/src/steal/ip_tables.rs | 9 +++++++++ mirrord/agent/src/steal/subscriptions.rs | 9 +++++++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87149b9be8c..b123f6d021e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -573,7 +573,7 @@ jobs: - run: kind load image-archive /tmp/test.tar - run: | cargo test --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --ignored --test-threads=6 - - name: fetch agent logs + - name: fetch info on failure if: ${{ failure() }} run: | kubectl logs -l app==mirrord --tail -1 diff --git a/Cargo.lock b/Cargo.lock index 89528a03ca1..2c210786848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3838,6 +3838,12 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "liblmod" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b221585d3f0053b5b80768e115b50edcd5c423b703a9c2c5cbf4bcf417ab227c" + [[package]] name = "libloading" version = "0.8.6" @@ -4193,6 +4199,7 @@ dependencies = [ "iptables", "k8s-cri", "libc", + "liblmod", "mirrord-protocol", "mockall", "nix 0.29.0", diff --git a/mirrord/agent/Cargo.toml b/mirrord/agent/Cargo.toml index cdba788acf3..3179cca39a0 100644 --- a/mirrord/agent/Cargo.toml +++ b/mirrord/agent/Cargo.toml @@ -72,6 +72,7 @@ socket2.workspace = true iptables = { git = "https://github.com/metalbear-co/rust-iptables.git", rev = "e66c7332e361df3c61a194f08eefe3f40763d624" } rawsocket = { git = "https://github.com/metalbear-co/rawsocket.git" } procfs = "0.17.0" +liblmod = "0.2.0" [target.'cfg(target_os = "linux")'.dev-dependencies] rstest.workspace = true diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 68bddb6a406..7912804e499 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -1,5 +1,6 @@ use std::{ fmt::Debug, + process::Command, sync::{Arc, LazyLock}, }; @@ -118,6 +119,14 @@ pub fn new_ip6tables() -> iptables::IPTables { { iptables::new_with_cmd("/usr/sbin/ip6tables-nft") } else { + let output = Command::new("/usr/sbin/ip6tables-legacy") + .arg("--version") + .output() + .unwrap() + .stdout; + let version = String::from_utf8_lossy(&output); + tracing::info!("Using ip6tables-legacy, version: {version}"); + iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") } .expect("IPTables initialization may not fail!") diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 0468719bc9c..ec70ba644fd 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,6 +79,15 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { + liblmod::modprobe( + "ip6table_nat".to_string(), + "".to_string(), + liblmod::Selection::Current, + ) + .map_err(|e| { + tracing::warn!(%e, "modprobe ip6_tables failed"); + AgentError::IPTablesError(format!("modprobe failed: {e:?}")) + })?; new_ip6tables() } else { new_iptables() From 59a8b3e6db161b69d4ffa18dcd37582b4b994389 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 15:13:22 +0100 Subject: [PATCH 61/72] exec modprobe as command --- mirrord/agent/src/steal/subscriptions.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index ec70ba644fd..4ec30705389 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,6 +79,17 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { + let output = std::process::Command::new("modprobe") + .arg("ip6table_nat") + .output() + .map_err(|e| { + tracing::warn!(%e, "modprobe ip6_tables failed"); + AgentError::IPTablesError(format!("modprobe failed: {e:?}")) + })? + .stdout; + + let output = String::from_utf8_lossy(&output); + tracing::info!("modprobe output: {output}"); liblmod::modprobe( "ip6table_nat".to_string(), "".to_string(), @@ -87,7 +98,8 @@ impl PortRedirector for IptablesListener { .map_err(|e| { tracing::warn!(%e, "modprobe ip6_tables failed"); AgentError::IPTablesError(format!("modprobe failed: {e:?}")) - })?; + }) + .ok(); new_ip6tables() } else { new_iptables() From 6f618ac5c3ca006b2f9b11cad9d820f1a082a8b0 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 15:48:02 +0100 Subject: [PATCH 62/72] which modprobe --- mirrord/agent/src/steal/subscriptions.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 4ec30705389..e1606193ed1 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,15 +79,24 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { + let output = std::process::Command::new("which") + .arg("modprobe") + .output() + .map_err(|e| { + tracing::warn!(%e, "`which modprobe` failed"); + AgentError::IPTablesError(format!("which modprobe failed: {e:?}")) + })? + .stdout; + let output = String::from_utf8_lossy(&output); + tracing::info!("which modprobe: {output}"); let output = std::process::Command::new("modprobe") .arg("ip6table_nat") .output() .map_err(|e| { - tracing::warn!(%e, "modprobe ip6_tables failed"); - AgentError::IPTablesError(format!("modprobe failed: {e:?}")) + tracing::warn!(%e, "manual modprobe ip6_tables failed"); + AgentError::IPTablesError(format!("manual modprobe failed: {e:?}")) })? .stdout; - let output = String::from_utf8_lossy(&output); tracing::info!("modprobe output: {output}"); liblmod::modprobe( From c0ed13458544f312ad3f1c13655adc530eed0868 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 16:47:11 +0100 Subject: [PATCH 63/72] docker file install kmod --- mirrord/agent/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord/agent/Dockerfile b/mirrord/agent/Dockerfile index 83c97871cae..8fa93bafeb0 100644 --- a/mirrord/agent/Dockerfile +++ b/mirrord/agent/Dockerfile @@ -48,5 +48,6 @@ RUN cp /app/target/$(cat /.platform)/release/mirrord-agent /mirrord-agent FROM ghcr.io/metalbear-co/ci-agent-runtime:latest COPY --from=builder /mirrord-agent / +RUN apt install -y kmod CMD ["./mirrord-agent"] From ea871129d95f42abb3f8a6babbd3c73f8f09e95b Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 Jan 2025 16:54:24 +0100 Subject: [PATCH 64/72] modprobe ip6_tables --- mirrord/agent/src/steal/subscriptions.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index e1606193ed1..f0f65c04bc0 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -99,6 +99,15 @@ impl PortRedirector for IptablesListener { .stdout; let output = String::from_utf8_lossy(&output); tracing::info!("modprobe output: {output}"); + liblmod::modprobe( + "ip6_tables".to_string(), + "".to_string(), + liblmod::Selection::Current, + ) + .map_err(|e| { + tracing::warn!(%e, "modprobe ip6_tables failed"); + AgentError::IPTablesError(format!("modprobe failed: {e:?}")) + })?; liblmod::modprobe( "ip6table_nat".to_string(), "".to_string(), @@ -107,8 +116,7 @@ impl PortRedirector for IptablesListener { .map_err(|e| { tracing::warn!(%e, "modprobe ip6_tables failed"); AgentError::IPTablesError(format!("modprobe failed: {e:?}")) - }) - .ok(); + })?; new_ip6tables() } else { new_iptables() From 953997dec3c9a6829b589b5793b4394e6601142b Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 8 Jan 2025 10:54:02 +0100 Subject: [PATCH 65/72] load 3 modules --- Cargo.lock | 7 --- mirrord/agent/Cargo.toml | 1 - mirrord/agent/src/steal/subscriptions.rs | 58 +++++++++++++----------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c210786848..89528a03ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3838,12 +3838,6 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" -[[package]] -name = "liblmod" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b221585d3f0053b5b80768e115b50edcd5c423b703a9c2c5cbf4bcf417ab227c" - [[package]] name = "libloading" version = "0.8.6" @@ -4199,7 +4193,6 @@ dependencies = [ "iptables", "k8s-cri", "libc", - "liblmod", "mirrord-protocol", "mockall", "nix 0.29.0", diff --git a/mirrord/agent/Cargo.toml b/mirrord/agent/Cargo.toml index 3179cca39a0..cdba788acf3 100644 --- a/mirrord/agent/Cargo.toml +++ b/mirrord/agent/Cargo.toml @@ -72,7 +72,6 @@ socket2.workspace = true iptables = { git = "https://github.com/metalbear-co/rust-iptables.git", rev = "e66c7332e361df3c61a194f08eefe3f40763d624" } rawsocket = { git = "https://github.com/metalbear-co/rawsocket.git" } procfs = "0.17.0" -liblmod = "0.2.0" [target.'cfg(target_os = "linux")'.dev-dependencies] rstest.workspace = true diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index f0f65c04bc0..97382826990 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,44 +79,48 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { - let output = std::process::Command::new("which") - .arg("modprobe") + let output = std::process::Command::new("modprobe") + .arg("ip6table_nat") .output() .map_err(|e| { - tracing::warn!(%e, "`which modprobe` failed"); - AgentError::IPTablesError(format!("which modprobe failed: {e:?}")) + tracing::warn!(%e, "manual modprobe ip6_tables failed"); + AgentError::IPTablesError(format!( + "manual modprobe ip6table_nat failed: {e:?}" + )) })? .stdout; let output = String::from_utf8_lossy(&output); - tracing::info!("which modprobe: {output}"); + tracing::info!("modprobe output: {output}"); let output = std::process::Command::new("modprobe") - .arg("ip6table_nat") + .arg("ip6_tables") .output() .map_err(|e| { tracing::warn!(%e, "manual modprobe ip6_tables failed"); - AgentError::IPTablesError(format!("manual modprobe failed: {e:?}")) + AgentError::IPTablesError(format!( + "manual modprobe ip6_tables failed: {e:?}" + )) + })? + .stdout; + let output = std::process::Command::new("modprobe") + .arg("nf_nat_ipv6") + .output() + .map_err(|e| { + tracing::warn!(%e, "manual modprobe ip6_tables failed"); + AgentError::IPTablesError(format!( + "manual modprobe nf_nat_ipv6 failed: {e:?}" + )) + })? + .stdout; + let output = std::process::Command::new("modprobe") + .arg("nf_conntrack_ipv6") + .output() + .map_err(|e| { + tracing::warn!(%e, "manual modprobe ip6_tables failed"); + AgentError::IPTablesError(format!( + "manual modprobe nf_conntrack_ipv6 failed: {e:?}" + )) })? .stdout; - let output = String::from_utf8_lossy(&output); - tracing::info!("modprobe output: {output}"); - liblmod::modprobe( - "ip6_tables".to_string(), - "".to_string(), - liblmod::Selection::Current, - ) - .map_err(|e| { - tracing::warn!(%e, "modprobe ip6_tables failed"); - AgentError::IPTablesError(format!("modprobe failed: {e:?}")) - })?; - liblmod::modprobe( - "ip6table_nat".to_string(), - "".to_string(), - liblmod::Selection::Current, - ) - .map_err(|e| { - tracing::warn!(%e, "modprobe ip6_tables failed"); - AgentError::IPTablesError(format!("modprobe failed: {e:?}")) - })?; new_ip6tables() } else { new_iptables() From e5dca27e0d76896dca7e41c52b4e4bdad0e01986 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 8 Jan 2025 10:57:03 +0100 Subject: [PATCH 66/72] unused vars --- mirrord/agent/src/steal/subscriptions.rs | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 97382826990..a348f1c8424 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,7 +79,7 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { - let output = std::process::Command::new("modprobe") + std::process::Command::new("modprobe") .arg("ip6table_nat") .output() .map_err(|e| { @@ -87,11 +87,8 @@ impl PortRedirector for IptablesListener { AgentError::IPTablesError(format!( "manual modprobe ip6table_nat failed: {e:?}" )) - })? - .stdout; - let output = String::from_utf8_lossy(&output); - tracing::info!("modprobe output: {output}"); - let output = std::process::Command::new("modprobe") + })?; + std::process::Command::new("modprobe") .arg("ip6_tables") .output() .map_err(|e| { @@ -99,9 +96,8 @@ impl PortRedirector for IptablesListener { AgentError::IPTablesError(format!( "manual modprobe ip6_tables failed: {e:?}" )) - })? - .stdout; - let output = std::process::Command::new("modprobe") + })?; + std::process::Command::new("modprobe") .arg("nf_nat_ipv6") .output() .map_err(|e| { @@ -109,9 +105,8 @@ impl PortRedirector for IptablesListener { AgentError::IPTablesError(format!( "manual modprobe nf_nat_ipv6 failed: {e:?}" )) - })? - .stdout; - let output = std::process::Command::new("modprobe") + })?; + std::process::Command::new("modprobe") .arg("nf_conntrack_ipv6") .output() .map_err(|e| { @@ -119,8 +114,7 @@ impl PortRedirector for IptablesListener { AgentError::IPTablesError(format!( "manual modprobe nf_conntrack_ipv6 failed: {e:?}" )) - })? - .stdout; + })?; new_ip6tables() } else { new_iptables() From 8c604324f979f4526d98de6fe6fc20e50155593f Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 8 Jan 2025 15:48:57 +0100 Subject: [PATCH 67/72] undo modprobes --- mirrord/agent/Dockerfile | 1 - mirrord/agent/src/steal/ip_tables.rs | 9 ------ mirrord/agent/src/steal/subscriptions.rs | 36 ------------------------ 3 files changed, 46 deletions(-) diff --git a/mirrord/agent/Dockerfile b/mirrord/agent/Dockerfile index 8fa93bafeb0..83c97871cae 100644 --- a/mirrord/agent/Dockerfile +++ b/mirrord/agent/Dockerfile @@ -48,6 +48,5 @@ RUN cp /app/target/$(cat /.platform)/release/mirrord-agent /mirrord-agent FROM ghcr.io/metalbear-co/ci-agent-runtime:latest COPY --from=builder /mirrord-agent / -RUN apt install -y kmod CMD ["./mirrord-agent"] diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 7912804e499..68bddb6a406 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -1,6 +1,5 @@ use std::{ fmt::Debug, - process::Command, sync::{Arc, LazyLock}, }; @@ -119,14 +118,6 @@ pub fn new_ip6tables() -> iptables::IPTables { { iptables::new_with_cmd("/usr/sbin/ip6tables-nft") } else { - let output = Command::new("/usr/sbin/ip6tables-legacy") - .arg("--version") - .output() - .unwrap() - .stdout; - let version = String::from_utf8_lossy(&output); - tracing::info!("Using ip6tables-legacy, version: {version}"); - iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") } .expect("IPTables initialization may not fail!") diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index a348f1c8424..0468719bc9c 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -79,42 +79,6 @@ impl PortRedirector for IptablesListener { } else { let safe = crate::steal::ip_tables::SafeIpTables::create( if self.ipv6 { - std::process::Command::new("modprobe") - .arg("ip6table_nat") - .output() - .map_err(|e| { - tracing::warn!(%e, "manual modprobe ip6_tables failed"); - AgentError::IPTablesError(format!( - "manual modprobe ip6table_nat failed: {e:?}" - )) - })?; - std::process::Command::new("modprobe") - .arg("ip6_tables") - .output() - .map_err(|e| { - tracing::warn!(%e, "manual modprobe ip6_tables failed"); - AgentError::IPTablesError(format!( - "manual modprobe ip6_tables failed: {e:?}" - )) - })?; - std::process::Command::new("modprobe") - .arg("nf_nat_ipv6") - .output() - .map_err(|e| { - tracing::warn!(%e, "manual modprobe ip6_tables failed"); - AgentError::IPTablesError(format!( - "manual modprobe nf_nat_ipv6 failed: {e:?}" - )) - })?; - std::process::Command::new("modprobe") - .arg("nf_conntrack_ipv6") - .output() - .map_err(|e| { - tracing::warn!(%e, "manual modprobe ip6_tables failed"); - AgentError::IPTablesError(format!( - "manual modprobe nf_conntrack_ipv6 failed: {e:?}" - )) - })?; new_ip6tables() } else { new_iptables() From d3db592f86856b8825e0302b1928b2f4006bdc0a Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 8 Jan 2025 17:49:10 +0100 Subject: [PATCH 68/72] protocol cargo --- mirrord/protocol/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 1904cc97f1c..7d787e1e5c6 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -18,12 +18,12 @@ workspace = true [dependencies] actix-codec.workspace = true +bincode.workspace = true bytes.workspace = true thiserror.workspace = true hickory-resolver.workspace = true hickory-proto.workspace = true serde.workspace = true -bincode.workspace = true tracing.workspace = true hyper = { workspace = true, features = ["client"] } http-serde = "2" From 07fd967d762cf432eaac14079c636379c1977d7c Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 8 Jan 2025 18:00:41 +0100 Subject: [PATCH 69/72] don't test ipv6 on CI --- .github/workflows/ci.yaml | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b123f6d021e..54115cef62f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -551,37 +551,6 @@ jobs: kubectl describe pods docker exec minikube find /var/log/pods -print -exec cat {} \; - ipv6_e2e_tests: - runs-on: ubuntu-24.04 - needs: [ test_agent_image, changed_files ] - if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' || needs.changed_files.outputs.dockerfile_changed == 'true'}} - env: - MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug" - steps: - - uses: actions/checkout@v4 - - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust. - - uses: actions/setup-python@v3 - - run: pip3 install fastapi uvicorn[standard] - shell: bash - - name: download image - uses: actions/download-artifact@v4 - with: - name: test - path: /tmp - - name: Create k8s Kind Cluster - run: .github/workflows/kind-with-registry-and-ipv6.sh - - run: kind load image-archive /tmp/test.tar - - run: | - cargo test --target=x86_64-unknown-linux-gnu -p tests steal_http_ipv6_traffic -- --ignored --test-threads=6 - - name: fetch info on failure - if: ${{ failure() }} - run: | - kubectl logs -l app==mirrord --tail -1 - kubectl get all - kubectl describe pods - cat /tmp/mirrord-intproxy-*.log - kubectl get nodes -o wide - lint_markdown: runs-on: ubuntu-24.04 needs: changed_files @@ -651,7 +620,6 @@ jobs: integration_tests, e2e, test_agent, - ipv6_e2e_tests, lint, lint_markdown, check-rust-docs, @@ -669,7 +637,6 @@ jobs: (needs.macos_tests.result == 'success' || needs.macos_tests.result == 'skipped') && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') && (needs.e2e.result == 'success' || needs.e2e.result == 'skipped') && - (needs.ipv6_e2e_tests.result == 'success' || needs.ipv6_e2e_tests.result == 'skipped') && (needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') && From c49013cb6ff6dae6c9964e98c18cb1693c22d162 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 9 Jan 2025 11:15:23 +0100 Subject: [PATCH 70/72] delete kind cluster creation script, since not testing in CI --- .../workflows/kind-with-registry-and-ipv6.sh | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100755 .github/workflows/kind-with-registry-and-ipv6.sh diff --git a/.github/workflows/kind-with-registry-and-ipv6.sh b/.github/workflows/kind-with-registry-and-ipv6.sh deleted file mode 100755 index 334e9013438..00000000000 --- a/.github/workflows/kind-with-registry-and-ipv6.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh - -# taken from https://kind.sigs.k8s.io/docs/user/local-registry/ - -set -o errexit - - -# 1. Create registry container unless it already exists -reg_name='kind-registry' -reg_port='5001' -if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then - docker run \ - -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \ - registry:2 -fi - -# 2. Create kind cluster with containerd registry config dir enabled -# TODO: kind will eventually enable this by default and this patch will -# be unnecessary. -# -# See: -# https://github.com/kubernetes-sigs/kind/issues/2875 -# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration -# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md -cat < Date: Thu, 9 Jan 2025 15:48:35 +0100 Subject: [PATCH 71/72] CR --- tests/src/traffic/steal.rs | 6 +++--- tests/src/utils.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index a334d34f9ed..31b5f668a2b 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -85,9 +85,9 @@ mod steal_tests { let mut flags = vec!["--steal"]; - // if cfg!(feature = "ephemeral") { - // flags.extend(["-e"].into_iter()); - // } + if cfg!(feature = "ephemeral") { + flags.extend(["-e"].into_iter()); + } let mut process = application .run( diff --git a/tests/src/utils.rs b/tests/src/utils.rs index e22f0c53fa3..3b596efa357 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -594,7 +594,6 @@ pub async fn run_exec( // minikube load image test:latest let mut base_env = HashMap::new(); base_env.insert("MIRRORD_AGENT_IMAGE", "test"); - base_env.insert("MIRRORD_AGENT_TTL", "30"); base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From 1f5508be6013814332b6bde8ed005b0ecb5c97c1 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 9 Jan 2025 22:08:37 +0100 Subject: [PATCH 72/72] apply change to new policy test --- tests/src/operator/policies/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/operator/policies/fs.rs b/tests/src/operator/policies/fs.rs index d54ed7bb98d..afa2cdf62bc 100644 --- a/tests/src/operator/policies/fs.rs +++ b/tests/src/operator/policies/fs.rs @@ -59,7 +59,7 @@ pub async fn create_cluster_fs_policy_and_try_file_operations( let mut test_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(vec!["--fs-mode=write"]), None,