diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml index 39191005f66..607eb12e265 100644 --- a/.github/workflows/ci_rust.yml +++ b/.github/workflows/ci_rust.yml @@ -14,6 +14,7 @@ env: RUST_NIGHTLY_TOOLCHAIN: nightly-2024-01-01 ROOT_PATH: bindings/rust EXAMPLE_WORKSPACE: bindings/rust-examples + PCAP_TEST_PATH: tests/pcap jobs: generate: @@ -256,3 +257,29 @@ jobs: - name: Check MSRV of s2n-tokio run: grep "rust-version = \"$(cat ${{env.ROOT_PATH}}/rust-toolchain)\"" ${{env.ROOT_PATH}}/s2n-tls-tokio/Cargo.toml + pcaps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Install Rust toolchain + id: toolchain + run: | + rustup toolchain install stable --component clippy + rustup override set stable + + - name: Generate bindings + working-directory: ${{env.ROOT_PATH}} + run: ./generate.sh --skip-tests + + - name: Run lints + working-directory: ${{env.PCAP_TEST_PATH}} + run: | + cargo fmt --all -- --check + cargo clippy --all-targets -- -D warnings + + - name: Run tests + working-directory: ${{env.PCAP_TEST_PATH}} + run: cargo test diff --git a/tests/pcap/.gitignore b/tests/pcap/.gitignore new file mode 100644 index 00000000000..2c96eb1b651 --- /dev/null +++ b/tests/pcap/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock diff --git a/tests/pcap/Cargo.toml b/tests/pcap/Cargo.toml new file mode 100644 index 00000000000..bc5aa1bbc2a --- /dev/null +++ b/tests/pcap/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "pcap" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1.0.86" +hex = "0.4.3" +rtshark = "2.7.1" + +[dev-dependencies] +# We want to test against the latest, local version of s2n +s2n-tls-sys = { path = "../../bindings/rust/s2n-tls-sys" } +s2n-tls = { path = "../../bindings/rust/s2n-tls", features = ["unstable-fingerprint"] } diff --git a/tests/pcap/data/fragmented_ch.pcap b/tests/pcap/data/fragmented_ch.pcap new file mode 100644 index 00000000000..d5518d83666 Binary files /dev/null and b/tests/pcap/data/fragmented_ch.pcap differ diff --git a/tests/pcap/data/multiple_hellos.pcap b/tests/pcap/data/multiple_hellos.pcap new file mode 100644 index 00000000000..dd124cbe3ea Binary files /dev/null and b/tests/pcap/data/multiple_hellos.pcap differ diff --git a/tests/pcap/data/tls12.pcap b/tests/pcap/data/tls12.pcap new file mode 100644 index 00000000000..41b61cae468 Binary files /dev/null and b/tests/pcap/data/tls12.pcap differ diff --git a/tests/pcap/data/tls13.pcap b/tests/pcap/data/tls13.pcap new file mode 100644 index 00000000000..7c88292794c Binary files /dev/null and b/tests/pcap/data/tls13.pcap differ diff --git a/tests/pcap/src/capture.rs b/tests/pcap/src/capture.rs new file mode 100644 index 00000000000..e68e790c643 --- /dev/null +++ b/tests/pcap/src/capture.rs @@ -0,0 +1,42 @@ +use rtshark::Metadata; +use rtshark::Packet; +use rtshark::RTShark; + +pub(crate) fn read_all(mut tshark: RTShark) -> Vec { + let mut packets = Vec::new(); + while let Ok(Some(packet)) = tshark.read() { + packets.push(packet) + } + packets +} + +pub(crate) fn get_metadata<'a>(packet: &'a Packet, key: &'a str) -> Option<&'a str> { + let (layer_name, _) = key.split_once('.').expect("key is layer"); + packet + .layer_name(layer_name) + .and_then(|layer| layer.metadata(key)) + .map(Metadata::value) +} + +pub(crate) fn get_all_metadata<'a>(packet: &'a Packet, key: &'a str) -> Vec<&'a str> { + let (layer_name, _) = key.split_once('.').expect("key is layer"); + if let Some(layer) = packet.layer_name(layer_name) { + layer + .iter() + .filter(|metadata| metadata.name() == key) + .map(Metadata::value) + .collect() + } else { + Vec::new() + } +} + +pub fn all_pcaps() -> impl Iterator { + std::fs::read_dir("data") + .expect("Missing test pcap file") + .filter_map(Result::ok) + .filter_map(|entry| { + let path = entry.path(); + path.to_str().map(std::string::ToString::to_string) + }) +} diff --git a/tests/pcap/src/client_hello.rs b/tests/pcap/src/client_hello.rs new file mode 100644 index 00000000000..f8d9b835b2c --- /dev/null +++ b/tests/pcap/src/client_hello.rs @@ -0,0 +1,83 @@ +use crate::handshake_message::Builder as MessageBuilder; +use crate::handshake_message::Message; +use anyhow::*; +use std::option::Option; + +use crate::capture::*; + +#[derive(Debug, Clone, Default)] +pub struct ClientHello { + pub message: Message, + pub ja3_hash: Option, + pub ja3_str: Option, +} + +impl ClientHello { + const JA3_HASH: &'static str = "tls.handshake.ja3"; + const JA3_STR: &'static str = "tls.handshake.ja3_full"; + + fn from_message(message: Message) -> Self { + let packet = &message.packet; + let ja3_hash = get_metadata(packet, Self::JA3_HASH).map(str::to_string); + let ja3_str = get_metadata(packet, Self::JA3_STR).map(str::to_string); + Self { + message, + ja3_hash, + ja3_str, + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct Builder(MessageBuilder); + +impl Builder { + pub fn inner(&mut self) -> &mut MessageBuilder { + &mut self.0 + } + + pub fn build(mut self) -> Result> { + self.0.set_type(1); + + let mut client_hellos = Vec::new(); + for message in self.0.build()? { + client_hellos.push(ClientHello::from_message(message)); + } + Ok(client_hellos) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn multiple_hellos() -> Result<()> { + let mut builder = Builder::default(); + builder + .inner() + .set_capture_file("data/multiple_hellos.pcap"); + let hellos = builder.build()?; + assert_eq!(hellos.len(), 5); + Ok(()) + } + + #[test] + fn from_pcaps() -> Result<()> { + let pcaps = all_pcaps(); + + for pcap in pcaps { + let mut builder = Builder::default(); + builder.inner().set_capture_file(&pcap); + let hellos = builder.build().unwrap(); + + assert!(!hellos.is_empty()); + for hello in hellos { + assert!(hello.ja3_hash.is_some()); + assert!(hello.ja3_str.is_some()); + } + } + + Ok(()) + } +} diff --git a/tests/pcap/src/handshake_message.rs b/tests/pcap/src/handshake_message.rs new file mode 100644 index 00000000000..30bae8dc4e9 --- /dev/null +++ b/tests/pcap/src/handshake_message.rs @@ -0,0 +1,171 @@ +use anyhow::*; +use rtshark::Packet; +use rtshark::RTSharkBuilder; +use std::collections::HashSet; + +use crate::capture::*; + +#[derive(Debug, Clone, Default)] +pub struct Message { + pub message_type: String, + pub packet: Packet, + pub payloads: Vec, + pub frame_num: String, +} + +impl Message { + const FRAGMENT: &'static str = "tls.handshake.fragment"; + const FRAGMENTS_COUNT: &'static str = "tls.handshake.fragment.count"; + + pub fn to_bytes(&self) -> Result> { + let mut bytes = Vec::new(); + for payload in &self.payloads { + let hex = payload.replace(':', ""); + bytes.extend(&hex::decode(&hex)?[5..]); + } + Ok(bytes) + } + + fn from_packet(packet: Packet, frame_num: String, tcp_payloads: &Vec) -> Result { + let message_type = get_metadata(&packet, Builder::MESSAGE_TYPE) + .context("Missing handshake message type")? + .to_string(); + + let mut payload_frames = get_all_metadata(&packet, Self::FRAGMENT); + if payload_frames.is_empty() { + payload_frames.push(&frame_num); + } + let payload_frames: HashSet<&str> = HashSet::from_iter(payload_frames); + + let mut payloads = Vec::new(); + for packet in tcp_payloads { + if let Some(frame) = get_metadata(packet, Builder::FRAME_NUM) { + if payload_frames.contains(frame) { + let payload = get_metadata(packet, Builder::TCP_PAYLOAD) + .context("Missing tcp payload")?; + payloads.push(payload.to_string()); + } + } + } + + let count = get_metadata(&packet, Self::FRAGMENTS_COUNT).unwrap_or("1"); + if count != payloads.len().to_string() { + bail!("Unable to find all tcp payloads for tls message") + } + + Ok(Self { + message_type, + packet, + payloads, + frame_num, + }) + } +} + +#[derive(Debug, Clone, Default)] +pub struct Builder { + message_type: Option, + capture_file: Option, +} + +impl Builder { + const TCP_PAYLOAD: &'static str = "tcp.payload"; + const FRAME_NUM: &'static str = "frame.number"; + const MESSAGE_TYPE: &'static str = "tls.handshake.type"; + + pub(crate) fn set_type(&mut self, message_type: u16) -> &mut Self { + self.message_type = Some(message_type); + self + } + + pub fn set_capture_file(&mut self, file: &str) -> &mut Self { + self.capture_file = Some(file.to_string()); + self + } + + fn build_from_capture(self, capture: rtshark::RTSharkBuilderReady) -> Result> { + let tcp_capture = capture + .display_filter(Self::TCP_PAYLOAD) + .metadata_whitelist(Self::TCP_PAYLOAD) + .metadata_whitelist(Self::FRAME_NUM) + .spawn()?; + let payloads = read_all(tcp_capture); + + let filter = if let Some(message_type) = self.message_type { + format!("{} == {}", Self::MESSAGE_TYPE, message_type) + } else { + Self::MESSAGE_TYPE.to_string() + }; + let message_capture = capture.display_filter(&filter).spawn()?; + + let mut messages = Vec::new(); + for packet in read_all(message_capture) { + let frame_num = get_metadata(&packet, Builder::FRAME_NUM) + .context("Missing frame number")? + .to_string(); + + let context_msg = format!("Failed to parse frame {}", &frame_num); + let message = + Message::from_packet(packet, frame_num, &payloads).context(context_msg)?; + + messages.push(message); + } + Ok(messages) + } + + pub(crate) fn build(mut self) -> Result> { + let file = self + .capture_file + .take() + .context("No capture file provided")?; + let capture = RTSharkBuilder::builder().input_path(&file); + self.build_from_capture(capture) + .with_context(|| format!("Failed to parse capture file {}", &file)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fragmentation() -> Result<()> { + let mut builder = Builder::default(); + builder.set_capture_file("data/fragmented_ch.pcap"); + let messages = builder.build()?; + + let first = messages.first().unwrap(); + assert_eq!(first.payloads.len(), 3); + let bytes = first.to_bytes()?; + // Correct length read from wireshark + assert_eq!(bytes.len(), 16262); + + Ok(()) + } + + #[test] + fn multiple_handshakes() -> Result<()> { + let mut builder = Builder::default(); + builder.set_capture_file("data/multiple_hellos.pcap"); + let messages = builder.build()?; + let count = messages.iter().filter(|m| m.message_type == "1").count(); + assert_eq!(count, 5); + Ok(()) + } + + #[test] + fn from_pcaps() -> Result<()> { + let pcaps = all_pcaps(); + + for pcap in pcaps { + let pcap: String = pcap; + + let mut builder = Builder::default(); + builder.set_capture_file(&pcap); + let messages = builder.build().unwrap(); + assert!(!messages.is_empty()) + } + + Ok(()) + } +} diff --git a/tests/pcap/src/lib.rs b/tests/pcap/src/lib.rs new file mode 100644 index 00000000000..cc08faeb16f --- /dev/null +++ b/tests/pcap/src/lib.rs @@ -0,0 +1,3 @@ +pub mod capture; +pub mod client_hello; +pub mod handshake_message; diff --git a/tests/pcap/tests/s2n_client_hellos.rs b/tests/pcap/tests/s2n_client_hellos.rs new file mode 100644 index 00000000000..6bf3094f742 --- /dev/null +++ b/tests/pcap/tests/s2n_client_hellos.rs @@ -0,0 +1,56 @@ +use anyhow::*; +use pcap::capture::all_pcaps; +use pcap::client_hello::{Builder as PcapBuilder, ClientHello as PcapHello}; +use s2n_tls::client_hello::{ClientHello as S2NHello, FingerprintType}; + +fn get_s2n_hello(pcap_hello: &PcapHello) -> Result> { + let bytes = pcap_hello.message.to_bytes()?; + Ok(S2NHello::parse_client_hello(&bytes)?) +} + +fn test_all_client_hellos(test_fn: F) -> Result<()> +where + F: FnOnce(PcapHello, Box) -> Result<()> + Copy, +{ + let pcaps = all_pcaps(); + for pcap in pcaps { + let mut builder = PcapBuilder::default(); + builder.inner().set_capture_file(&pcap); + let hellos = builder.build()?; + + for hello in hellos { + println!( + "Testing ClientHello found in frame {} in {}", + hello.message.frame_num, pcap + ); + let s2n_hello = get_s2n_hello(&hello).context("s2n failed to parse ClientHello")?; + test_fn(hello, s2n_hello)?; + } + } + Ok(()) +} + +#[test] +fn parsing() -> Result<()> { + test_all_client_hellos(|_, _| Ok(())) +} + +#[test] +fn ja3_fingerprints() -> Result<()> { + test_all_client_hellos(|pcap_hello, s2n_hello| { + let mut s2n_ja3_hash = Vec::new(); + s2n_hello + .fingerprint_hash(FingerprintType::JA3, &mut s2n_ja3_hash) + .context("s2n failed to calculate ja3 hash")?; + let s2n_ja3_hash = hex::encode(s2n_ja3_hash); + + let mut s2n_ja3_str = String::with_capacity(1000); + s2n_hello + .fingerprint_string(FingerprintType::JA3, &mut s2n_ja3_str) + .context("s2n failed to calculate ja3 string")?; + + assert_eq!(pcap_hello.ja3_hash, Some(s2n_ja3_hash)); + assert_eq!(pcap_hello.ja3_str, Some(s2n_ja3_str)); + Ok(()) + }) +}