Merge pull request #2 from c410-f3r/misc

Add support for compression

.github/workflows/ci.yaml

@@ -6,39 +6,58 @@ on:
       - main
 
 jobs:
-  autobahn:
+  autobahn-fuzzingclient:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions-rs/toolchain@v1
         with:
           override: true
           profile: minimal
           toolchain: nightly-2023-08-01
       - uses: Swatinem/rust-cache@v2
 
-      - run: .scripts/autobahn.sh ci
-
-#  fuzz:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@v3
-#      - uses: actions-rs/toolchain@v1
-#        with:
-#          override: true
-#          profile: minimal
-#          toolchain: nightly-2023-08-01
-#      - uses: actions-rs/[email protected]
-#        with:
-#          crate: cargo-fuzz
-#          use-tool-cache: true
-#
-#      - run: .scripts/fuzz.sh
+      - run: .scripts/autobahn-fuzzingclient.sh ci
+    strategy:
+      fail-fast: true
+
+  autobahn-fuzzingserver:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: true
+          profile: minimal
+          toolchain: nightly-2023-08-01
+      - uses: Swatinem/rust-cache@v2
+
+      - run: .scripts/autobahn-fuzzingserver.sh ci
+    strategy:
+      fail-fast: true
+
+  fuzz:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: true
+          profile: minimal
+          toolchain: nightly-2023-08-01
+      - uses: actions-rs/[email protected]
+        with:
+          crate: cargo-fuzz
+          use-tool-cache: true
+
+      - run: .scripts/fuzz.sh
+    strategy:
+      fail-fast: true
 
   tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions-rs/toolchain@v1
         with:
           components: clippy, rustfmt
@@ -47,4 +66,6 @@ jobs:
           toolchain: nightly-2023-08-01
       - uses: Swatinem/rust-cache@v2
 
-      - run: .scripts/wtx.sh
+      - run: .scripts/internal-tests.sh
+    strategy:
+      fail-fast: true

.gitignore

@@ -2,7 +2,6 @@
 .vscode
 **/*.rs.bk
 **/artifacts
-**/Cargo.lock
 **/corpus
 **/target
 **/target

.scripts/autobahn-fuzzingclient.sh

@@ -0,0 +1,26 @@
+set -euxo pipefail
+
+ARG=${1:-""}
+if [ "$ARG" != "ci" ]; then
+	trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+fi;
+
+## fuzzingclient
+
+RUSTFLAGS='-C target-cpu=native' cargo build --bin autobahn-server --features flate2,tokio,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --bin autobahn-server --features flate2,tokio,web-socket-handshake --release & cargo_pid=$!
+mkdir -p .scripts/autobahn/reports/fuzzingclient
+podman run \
+	-v .scripts/autobahn/fuzzingclient-min.json:/fuzzingclient.json:ro \
+	-v .scripts/autobahn:/autobahn \
+	--name fuzzingclient \
+	--net=host \
+	--rm \
+	docker.io/crossbario/autobahn-testsuite:0.8.2 wstest -m fuzzingclient -s fuzzingclient.json
+podman rm --force --ignore fuzzingclient
+kill -9 $cargo_pid
+
+if [ $(grep -ci "failed" .scripts/autobahn/reports/fuzzingclient/index.json) -gt 0 ]
+then
+    exit 1
+fi

.scripts/autobahn-fuzzingserver.sh

@@ -0,0 +1,25 @@
+set -euxo pipefail
+
+ARG=${1:-""}
+if [ "$ARG" != "ci" ]; then
+	trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+fi;
+
+RUSTFLAGS='-C target-cpu=native' cargo build --bin autobahn-client --features flate2,tokio,web-socket-handshake --release
+mkdir -p .scripts/autobahn/reports/fuzzingserver
+podman run \
+	-d \
+	-p 9080:9080 \
+	-v .scripts/autobahn/fuzzingserver-min.json:/fuzzingserver.json:ro \
+	-v .scripts/autobahn:/autobahn \
+	--name fuzzingserver \
+	--net=host \
+	docker.io/crossbario/autobahn-testsuite:0.8.2 wstest -m fuzzingserver -s fuzzingserver.json
+sleep 5
+RUSTFLAGS='-C target-cpu=native' cargo run --bin autobahn-client --features flate2,tokio,web-socket-handshake --release -- 127.0.0.1:9080
+podman rm --force --ignore fuzzingserver
+
+if [ $(grep -ci "failed" .scripts/autobahn/reports/fuzzingserver/index.json) -gt 0 ]
+then
+    exit 1
+fi

.scripts/autobahn/fuzzingclient-min.json

@@ -0,0 +1,35 @@
+{
+  "cases": [
+    "1.*",
+    "2.*",
+    "3.*",
+    "4.*",
+    "5.*",
+    "6.*",
+    "7.*",
+    "8.*",
+    "9.*",
+    "10.*",
+    "12.1.1",
+    "12.2.1",
+    "12.3.1",
+    "12.4.1",
+    "12.5.1",
+    "13.1.1",
+    "13.2.1",
+    "13.3.1",
+    "13.4.1",
+    "13.5.1",
+    "13.6.1",
+    "13.7.1"
+  ],
+  "exclude-agent-cases": {},
+  "exclude-cases": [],
+  "outdir": "/autobahn/reports/fuzzingclient",
+  "servers": [
+    {
+      "agent": "wtx",
+      "url": "ws://127.0.0.1:8080"
+    }
+  ]
+}

.scripts/autobahn/fuzzingclient.json

@@ -1,5 +1,5 @@
 {
-  "cases": ["1.*", "2.*", "3.*", "4.*", "5.*", "6.*", "7.*", "9.*", "10.*"],
+  "cases": ["*"],
   "exclude-agent-cases": {},
   "exclude-cases": [],
   "outdir": "/autobahn/reports/fuzzingclient",

.scripts/autobahn/fuzzingserver-min.json

@@ -0,0 +1,30 @@
+{
+  "cases": [
+    "1.*",
+    "2.*",
+    "3.*",
+    "4.*",
+    "5.*",
+    "6.*",
+    "7.*",
+    "8.*",
+    "9.*",
+    "10.*",
+    "12.1.1",
+    "12.2.1",
+    "12.3.1",
+    "12.4.1",
+    "12.5.1",
+    "13.1.1",
+    "13.2.1",
+    "13.3.1",
+    "13.4.1",
+    "13.5.1",
+    "13.6.1",
+    "13.7.1"
+  ],
+  "exclude-agent-cases": {},
+  "exclude-cases": [],
+  "outdir": "/autobahn/reports/fuzzingserver",
+  "url": "ws://127.0.0.1:9080"
+}

.scripts/autobahn/fuzzingserver.json

@@ -1,5 +1,5 @@
 {
-  "cases": ["1.*", "2.*", "3.*", "4.*", "5.*", "6.*", "7.*", "9.*", "10.*"],
+  "cases": ["*"],
   "exclude-agent-cases": {},
   "exclude-cases": [],
   "outdir": "/autobahn/reports/fuzzingserver",

.scripts/fuzz.sh

@@ -2,5 +2,5 @@
 
 set -euxo pipefail
 
-cargo fuzz run --fuzz-dir wtx-fuzz unmask -- -runs=100000
-cargo fuzz run --fuzz-dir wtx-fuzz parse-frame -- -runs=100000
+cargo fuzz run --features="libfuzzer-sys/link_libfuzzer" --fuzz-dir wtx-fuzz parse-frame -- -max_total_time=30
+cargo fuzz run --features="libfuzzer-sys/link_libfuzzer" --fuzz-dir wtx-fuzz unmask -- -max_total_time=30

.scripts/gen-certs.sh

@@ -0,0 +1,16 @@
+# https://stackoverflow.com/questions/76049656/unexpected-notvalidforname-with-rusts-tonic-with-tls
+
+openssl req -newkey rsa:2048 -nodes -subj "/C=FI/CN=vahid" -keyout key.pem -out key.csr
+openssl x509 -signkey key.pem -in key.csr -req -days 365 -out cert.pem
+openssl req -x509 -sha256 -nodes -subj "/C=FI/CN=vahid" -days 1825 -newkey rsa:2048 -keyout root-ca.key -out root-ca.crt
+cat <<'EOF' >> localhost.ext
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = server
+IP.1 = 127.0.0.1
+EOF
+openssl x509 -req -CA root-ca.crt -CAkey root-ca.key -in key.csr -out cert.pem -days 365 -CAcreateserial -extfile localhost.ext
+rm localhost.ext
+rm key.csr

.scripts/wtx.sh → .scripts/internal-tests.sh

@@ -6,25 +6,29 @@ cargo install rust-tools --git https://github.com/c410-f3r/regular-crates
 
 rt='rust-tools --template you-rust'
 
-CARGO_TARGET_DIR="$($rt target-dir)"
-RUST_BACKTRACE=1
 RUSTFLAGS="$($rt rust-flags)"
 
+export CARGO_TARGET_DIR="$($rt target-dir)"
+export RUST_BACKTRACE=1
+export RUST_LOG=debug
+
 $rt rustfmt
 $rt clippy
 
 $rt test-generic wtx
 $rt test-with-features wtx async-std
-$rt test-with-features wtx async-trait
 $rt test-with-features wtx base64
+$rt test-with-features wtx flate2
 $rt test-with-features wtx futures-lite
 $rt test-with-features wtx glommio
 $rt test-with-features wtx http
 $rt test-with-features wtx httparse
-$rt test-with-features wtx hyper
+$rt test-with-features wtx rand
+$rt test-with-features wtx rustls-pemfile
 $rt test-with-features wtx sha1
 $rt test-with-features wtx simdutf8
+$rt test-with-features wtx smol
 $rt test-with-features wtx std
 $rt test-with-features wtx tokio
+$rt test-with-features wtx tokio-rustls
 $rt test-with-features wtx web-socket-handshake
-$rt test-with-features wtx web-socket-hyper

.scripts/wtx-bench.sh

@@ -35,17 +35,23 @@ fi
 ./EchoServer 8083 &
 popd
 
-RUSTFLAGS='-C target-cpu=native' cargo build --example web_socket_server_echo_hyper --features simdutf8,web-socket-hyper --release
-RUSTFLAGS='-C target-cpu=native' cargo run --example web_socket_server_echo_hyper --features simdutf8,web-socket-hyper --release 127.0.0.1:8084 &
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-async-std --features async-std/attributes,simdutf8,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-async-std --features async-std/attributes,simdutf8,web-socket-handshake --release 127.0.0.1:8084 &
 
-RUSTFLAGS='-C target-cpu=native' cargo build --example web_socket_server_echo_raw_async_std --features async-std,simdutf8,web-socket-handshake --release
-RUSTFLAGS='-C target-cpu=native' cargo run --example web_socket_server_echo_raw_async_std --features async-std,simdutf8,web-socket-handshake --release 127.0.0.1:8085 &
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-glommio --features glommio,simdutf8,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-glommio --features glommio,simdutf8,web-socket-handshake --release 127.0.0.1:8085 &
 
-RUSTFLAGS='-C target-cpu=native' cargo build --example web_socket_server_echo_raw_glommio --features glommio,simdutf8,web-socket-handshake --release
-RUSTFLAGS='-C target-cpu=native' cargo run --example web_socket_server_echo_raw_glommio --features glommio,simdutf8,web-socket-handshake --release 127.0.0.1:8086 &
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-monoio --features monoio/iouring,monoio/legacy,monoio/macros,simdutf8,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-monoio --features monoio/iouring,monoio/legacy,monoio/macros,simdutf8,web-socket-handshake --release 127.0.0.1:8086 &
 
-RUSTFLAGS='-C target-cpu=native' cargo build --example web_socket_server_echo_raw_tokio --features simdutf8,tokio,web-socket-handshake --release
-RUSTFLAGS='-C target-cpu=native' cargo run --example web_socket_server_echo_raw_tokio --features simdutf8,tokio,web-socket-handshake --release 127.0.0.1:8087 &
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-smol --features simdutf8,smol,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-smol --features simdutf8,smol,web-socket-handshake --release 127.0.0.1:8087 &
+
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-tokio --features simdutf8,tokio,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-tokio --features simdutf8,tokio,web-socket-handshake --release 127.0.0.1:8088 &
+
+RUSTFLAGS='-C target-cpu=native' cargo build --example web-socket-server-echo-raw-tokio-uring --features simdutf8,tokio-uring,web-socket-handshake --release
+RUSTFLAGS='-C target-cpu=native' cargo run --example web-socket-server-echo-raw-tokio-uring --features simdutf8,tokio-uring,web-socket-handshake --release 127.0.0.1:8089 &
 
 sleep 1
 
@@ -54,7 +60,9 @@ RUSTFLAGS='-C target-cpu=native' cargo run --bin wtx-bench --release -- \
     http://127.0.0.1:8081/gorilla-websocket \
     http://127.0.0.1:8082/tokio-tungstenite \
     http://127.0.0.1:8083/uWebSockets \
-    http://127.0.0.1:8084/wtx-hyper \
-    http://127.0.0.1:8085/wtx-raw-async-std \
-    http://127.0.0.1:8086/wtx-raw-glommio \
-    http://127.0.0.1:8087/wtx-raw-tokio
+    http://127.0.0.1:8084/wtx-raw-async-std \
+    http://127.0.0.1:8085/wtx-raw-glommio \
+    http://127.0.0.1:8086/wtx-raw-monoio \
+    http://127.0.0.1:8087/wtx-raw-smol \
+    http://127.0.0.1:8088/wtx-raw-tokio \
+    http://127.0.0.1:8089/wtx-raw-tokio-uring \