From 03aa56d63b0712dad69baa182642fe28e5142ece Mon Sep 17 00:00:00 2001 From: Michael de Lang Date: Mon, 25 Sep 2023 09:35:57 +0200 Subject: [PATCH] Add Https support for host/client --- CMakeLists.txt | 6 +- Dockerfile | 2 +- Dockerfile-asan | 8 +- Dockerfile-musl | 18 +- Dockerfile-musl-aarch64 | 18 +- Dockerfile-tsan | 46 +++++ build.sh | 62 +++++-- docs/05-HttpsConnections.md | 171 ++++++++++++++++++ examples/realtime_example/TestService.h | 4 + .../network/http/HttpConnectionService.h | 4 + .../services/network/http/HttpHostService.h | 18 +- .../network/http/HttpConnectionService.cpp | 149 +++++++++++---- src/services/network/http/HttpHostService.cpp | 108 +++++++++-- test/AsyncAutoResetEventTests.cpp | 10 - test/HttpTests.cpp | 86 +++++++++ test/RedisTests.cpp | 2 +- test/TestServices/HttpThreadService.h | 1 + 17 files changed, 618 insertions(+), 95 deletions(-) create mode 100644 Dockerfile-tsan create mode 100644 docs/05-HttpsConnections.md delete mode 100644 test/AsyncAutoResetEventTests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bda4a673..6bf64ed5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,7 +146,10 @@ if(ICHOR_USE_BOOST_BEAST) endif() if(ICHOR_MUSL) target_compile_definitions(ichor PUBLIC ICHOR_MUSL) - target_link_options(ichor PUBLIC -static-libgcc -static-libstdc++ -static) + target_link_options(ichor PUBLIC -static-libgcc -static-libstdc++) +endif() +if(ICHOR_AARCH64) + target_compile_definitions(ichor PUBLIC ICHOR_AARCH64) endif() if(NOT WIN32) @@ -381,6 +384,7 @@ if(ICHOR_USE_BOOST_BEAST) #beast target_include_directories(ichor PUBLIC ${Boost_INCLUDE_DIRS}) target_link_directories(ichor PUBLIC ${Boost_LIBRARY_DIRS}) target_link_libraries(ichor PUBLIC ${Boost_LIBRARIES}) + target_link_libraries(ichor PUBLIC -lssl -lcrypto) #_SILENCE_ALL_CXX23_DEPRECATION_WARNINGS -> MSVC gives warnings on things like std::aligned_storage, which is still valid in C++20. target_compile_definitions(ichor PUBLIC BOOST_ASIO_NO_DEPRECATED _SILENCE_ALL_CXX23_DEPRECATION_WARNINGS) diff --git a/Dockerfile b/Dockerfile index 6493c6bf..db76ee88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ WORKDIR /opt/ichor/build ENTRYPOINT ["/bin/bash", "-c"] -CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=ON -DICHOR_USE_SPDLOG=ON /opt/ichor/src && ninja"] +CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 /opt/ichor/src && ninja"] diff --git a/Dockerfile-asan b/Dockerfile-asan index c88ad0be..2bb5cb87 100644 --- a/Dockerfile-asan +++ b/Dockerfile-asan @@ -7,6 +7,9 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60 RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 60 +ENV CFLAGS="-Og -fsanitize=address,undefined" +ENV CXXFLAGS="-Og -fsanitize=address,undefined" + WORKDIR /opt #Build boost with support for asan @@ -34,6 +37,9 @@ RUN mkdir -p /opt/ichor/build WORKDIR /opt/ichor/build +RUN unset CFLAGS +RUN unset CXXFLAGS + ENTRYPOINT ["/bin/bash", "-c"] -CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=ON -DICHOR_USE_SPDLOG=ON /opt/ichor/src && ninja"] +CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 /opt/ichor/src && ninja"] diff --git a/Dockerfile-musl b/Dockerfile-musl index db1254aa..3ac51558 100644 --- a/Dockerfile-musl +++ b/Dockerfile-musl @@ -1,17 +1,28 @@ FROM alpine:3.17 RUN apk update -RUN apk add gcc g++ build-base cmake openssl-dev git wget make nano sed linux-headers +RUN apk add gcc g++ build-base cmake git wget make nano sed linux-headers perl # Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads WORKDIR /opt RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2 RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz +RUN wget https://www.openssl.org/source/openssl-3.0.11.tar.gz WORKDIR /opt -#Build a new enough boost, apt only contains 1.74 which is too old. +ENV LDFLAGS="-static-libgcc -static-libstdc++ -static" +#Build openssl statically, alpine (and probably most distros) only provide shared libraries. Might be a security thing? +RUN tar xf openssl-3.0.11.tar.gz +WORKDIR /opt/openssl-3.0.11 +RUN ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared +RUN make -j +RUN make -j install + +WORKDIR /opt + +#Build boost statically RUN tar xf boost_1_81_0.tar.bz2 WORKDIR /opt/boost_1_81_0 @@ -27,7 +38,6 @@ RUN tar xf v1.2.0.tar.gz RUN mkdir /opt/hiredis-1.2.0/build WORKDIR /opt/hiredis-1.2.0/build -ENV LDFLAGS="-static-libgcc -static-libstdc++ -static" RUN cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 -DBUILD_SHARED_LIBS=0 .. RUN make -j && make install RUN mkdir -p /opt/ichor/build @@ -36,4 +46,4 @@ WORKDIR /opt/ichor/build ENTRYPOINT ["/bin/sh", "-c"] -CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=ON -DICHOR_MUSL=1 /opt/ichor/src && make -j"] +CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 -DICHOR_MUSL=1 /opt/ichor/src && make -j"] diff --git a/Dockerfile-musl-aarch64 b/Dockerfile-musl-aarch64 index bb95a682..04a05ee1 100644 --- a/Dockerfile-musl-aarch64 +++ b/Dockerfile-musl-aarch64 @@ -1,17 +1,28 @@ FROM arm64v8/alpine:3.17 RUN apk update -RUN apk add gcc g++ build-base cmake openssl-dev git wget make nano sed linux-headers +RUN apk add gcc g++ build-base cmake git wget make nano sed linux-headers perl # Run all downloads first to be able to use Docker's layers as cache and prevent excessive redownloads WORKDIR /opt RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2 RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz +RUN wget https://www.openssl.org/source/openssl-3.0.11.tar.gz WORKDIR /opt -#Build a new enough boost, apt only contains 1.74 which is too old. +ENV LDFLAGS="-static-libgcc -static-libstdc++ -static" +#Build openssl statically, alpine (and probably most distros) only provide shared libraries. Might be a security thing? +RUN tar xf openssl-3.0.11.tar.gz +WORKDIR /opt/openssl-3.0.11 +RUN ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared +RUN make -j +RUN make -j install + +WORKDIR /opt + +#Build boost statically RUN tar xf boost_1_81_0.tar.bz2 WORKDIR /opt/boost_1_81_0 @@ -27,7 +38,6 @@ RUN tar xf v1.2.0.tar.gz RUN mkdir /opt/hiredis-1.2.0/build WORKDIR /opt/hiredis-1.2.0/build -ENV LDFLAGS="-static-libgcc -static-libstdc++ -static" RUN cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 -DBUILD_SHARED_LIBS=0 .. RUN make -j && make install RUN mkdir -p /opt/ichor/build @@ -36,4 +46,4 @@ WORKDIR /opt/ichor/build ENTRYPOINT ["/bin/sh", "-c"] -CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=0 -DICHOR_USE_BOOST_BEAST=0 -DICHOR_USE_SPDLOG=ON -DICHOR_MUSL=1 -DICHOR_AARCH64=1 /opt/ichor/src && make -j"] +CMD ["cd /opt/ichor/build && cmake -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 -DICHOR_MUSL=1 -DICHOR_AARCH64=1 /opt/ichor/src && make -j"] diff --git a/Dockerfile-tsan b/Dockerfile-tsan new file mode 100644 index 00000000..c8b59077 --- /dev/null +++ b/Dockerfile-tsan @@ -0,0 +1,46 @@ +FROM ubuntu:jammy + +RUN apt update +RUN apt install -y g++-12 gcc-12 build-essential cmake libssl-dev pkg-config git wget ninja-build nano libzip-dev + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 60 +RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 60 + +ENV CFLAGS="-Og -fsanitize=thread" +ENV CXXFLAGS="-Og -fsanitize=thread" +ENV LDFLAGS="-fsanitize=thread -static-libtsan -static-libgcc -static-libstdc++" + +WORKDIR /opt + +#Build boost with support for asan +RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2 +RUN tar xf boost_1_81_0.tar.bz2 + +WORKDIR /opt/boost_1_81_0 + +RUN ./bootstrap.sh --prefix=/usr +RUN ./b2 cxxflags="-fsanitize=thread -Og -std=c++17 -DBOOST_USE_TSAN -DBOOST_USE_UCONTEXT" linkflags="-ltsan -static-libtsan -static-libgcc -static-libstdc++" variant=debug link=static threading=multi context-impl=ucontext +RUN ./b2 cxxflags="-fsanitize=thread -Og -std=c++17 -DBOOST_USE_TSAN -DBOOST_USE_UCONTEXT" linkflags="-ltsan -static-libtsan -static-libgcc -static-libstdc++" variant=debug link=static threading=multi context-impl=ucontext install + +WORKDIR /opt + +#Build latest hiredis containing sdevent support, not available yet in apt +RUN wget https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz +RUN tar xf v1.2.0.tar.gz +RUN mkdir /opt/hiredis-1.2.0/build + +WORKDIR /opt/hiredis-1.2.0/build +RUN cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDISABLE_TESTS=1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SSL=1 .. +RUN ninja && ninja install + +RUN mkdir -p /opt/ichor/build + +WORKDIR /opt/ichor/build + +RUN unset CFLAGS +RUN unset CXXFLAGS + +ENTRYPOINT ["/bin/bash", "-c"] + +CMD ["cd /opt/ichor/build && cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DICHOR_USE_SANITIZERS=0 -DICHOR_USE_THREAD_SANITIZER=1 -DICHOR_USE_HIREDIS=1 -DICHOR_USE_BOOST_BEAST=1 -DICHOR_USE_SPDLOG=1 /opt/ichor/src && ninja"] diff --git a/build.sh b/build.sh index 684e7e82..9b376122 100755 --- a/build.sh +++ b/build.sh @@ -8,6 +8,26 @@ cleanup () trap cleanup SIGINT SIGTERM +POSITIONAL_ARGS=() +DOCKER=1 + +while [[ $# -gt 0 ]]; do + case $1 in + --no-docker) + DOCKER=0 + shift # past value + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + ccompilers=("clang-14" "clang-16" "gcc-11" "gcc-12") cppcompilers=("clang++-14" "clang++-16" "g++-11" "g++-12") @@ -52,26 +72,29 @@ run_benchmarks () } -rm -rf ./* ../bin/* -docker build -f ../Dockerfile -t ichor . || exit 1 -docker run -v $(pwd)/../:/opt/ichor/src -it ichor || exit 1 -run_examples +if [[ $DOCKER -eq 1 ]]; then + rm -rf ./* ../bin/* + docker build -f ../Dockerfile -t ichor . || exit 1 + docker run -v $(pwd)/../:/opt/ichor/src -it ichor || exit 1 + run_examples -rm -rf ./* ../bin/* -docker build -f ../Dockerfile-musl -t ichor-musl . || exit 1 -docker run -v $(pwd)/../:/opt/ichor/src -it ichor-musl || exit 1 -run_examples + rm -rf ./* ../bin/* + docker build -f ../Dockerfile-musl -t ichor-musl . || exit 1 + docker run -v $(pwd)/../:/opt/ichor/src -it ichor-musl || exit 1 + run_examples -rm -rf ./* ../bin/* -docker build -f ../Dockerfile-asan -t ichor-asan . || exit 1 -docker run -v $(pwd)/../:/opt/ichor/src -it ichor-asan || exit 1 -run_examples + rm -rf ./* ../bin/* + docker build -f ../Dockerfile-asan -t ichor-asan . || exit 1 + docker run -v $(pwd)/../:/opt/ichor/src -it ichor-asan || exit 1 + run_examples + + # tsan is purposefully not run automatically, because it usually contains false positives. -rm -rf ./* ../bin/* -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || exit 1 -docker build -f ../Dockerfile-musl-aarch64 -t ichor-musl-aarch64 . || exit 1 -docker run -v $(pwd)/../:/opt/ichor/src -it ichor-musl-aarch64 || exit 1 -cat >> ../bin/run_aarch64_examples_and_tests.sh << EOF + rm -rf ./* ../bin/* + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || exit 1 + docker build -f ../Dockerfile-musl-aarch64 -t ichor-musl-aarch64 . || exit 1 + docker run -v $(pwd)/../:/opt/ichor/src -it ichor-musl-aarch64 || exit 1 + cat >> ../bin/run_aarch64_examples_and_tests.sh << EOF #!/bin/sh FILES=/opt/ichor/src/bin/* for f in \$FILES; do @@ -81,8 +104,9 @@ for f in \$FILES; do fi done EOF -chmod +x ../bin/run_aarch64_examples_and_tests.sh -docker run -v $(pwd)/../:/opt/ichor/src --privileged -it ichor-musl-aarch64 "sh -c 'ulimit -r unlimited && /opt/ichor/src/bin/run_aarch64_examples_and_tests.sh'" || exit 1 + chmod +x ../bin/run_aarch64_examples_and_tests.sh + docker run -v $(pwd)/../:/opt/ichor/src --privileged -it ichor-musl-aarch64 "sh -c 'ulimit -r unlimited && /opt/ichor/src/bin/run_aarch64_examples_and_tests.sh'" || exit 1 +fi for i in ${!ccompilers[@]}; do rm -rf ./* ../bin/* diff --git a/docs/05-HttpsConnections.md b/docs/05-HttpsConnections.md new file mode 100644 index 00000000..6055debd --- /dev/null +++ b/docs/05-HttpsConnections.md @@ -0,0 +1,171 @@ +# Https Connections + +Ichor supports setting up and connecting to servers using SSL. + +## Host + +### Create certificate + +First, let's create a certificate without a password: + +```shell +$ openssl req -newkey rsa:2048 -new -nodes -keyout key.pem -out csr.pem -x509 -days 365 -subj "/C=NL/L=Amsterdam/O=Ichor/CN=www.example.com" +$ cat csr.pem +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIUQEtUTmX2HbHwOpDPgFkYx66ei/swDQYJKoZIhvcNAQEL +BQAwSzELMAkGA1UEBhMCTkwxEjAQBgNVBAcMCUFtc3RlcmRhbTEOMAwGA1UECgwF +SWNob3IxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTAeFw0yMzEwMDIwODI1NTVa +Fw0yNDEwMDEwODI1NTVaMEsxCzAJBgNVBAYTAk5MMRIwEAYDVQQHDAlBbXN0ZXJk +YW0xDjAMBgNVBAoMBUljaG9yMRgwFgYDVQQDDA93d3cuZXhhbXBsZS5jb20wggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIisWpB9FSc/OIYsxznQJgioL2 +aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ+VNc63blCPAI +iL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFFelbLf7JTW80C +nNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi1QbWD3G1lkTQ +cCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGoswxe5IF+yB6f +WFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0yv+K0iUBAgMB +AAGjUzBRMB0GA1UdDgQWBBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAfBgNVHSMEGDAW +gBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQC1QKNwlqrvE5a7M5v151tA/JvIKNBfesIRCg/IG6tB3teSJjAq +PRsSajMcyvDtlDMkWL093B8Epsc8yt/vN1GgsCfQsMeIWRGvRYM6Qo2WoABaLTuC +TxH89YrXa9uwUnbesGwKSkpxzj8g6YuzUpLnAAV70LefBn8ZgBJktFVW/ANWwQqD +KiglduSDGX3NiLByn7nGKejg2dPw6kWpOTVtLsWtgG/5T4vw+INQzPJqy+pUvBeQ +OrV7cQyq12DJJPxboIRh1KqF8C25NAOnXVtCav985RixHCMaNaXyOXBJeEEASnXg +pdDrObCgMze03IaHQ1pj3LeMu0OvEMbsRrYW +-----END CERTIFICATE----- +$ cat key.pem +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDIisWpB9FSc/OI +YsxznQJgioL2aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ ++VNc63blCPAIiL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFF +elbLf7JTW80CnNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi +1QbWD3G1lkTQcCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGo +swxe5IF+yB6fWFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0 +yv+K0iUBAgMBAAECggEAB2lJgMOvMrLiTyoHkEY/Lj4wNNcNW8g0aQRWlmng7SdE +84mh1QEspJegaUe7eyNTsTY8TNwzET43jEFQmWCCVliMNmSHWWWF99sgmuL5W5aN +Ec+4LPnCWDR+pxAKcCEoN4yzhfLTKNzmsqCg0Ih5mKcN3ojZMLodpCfH3WczDkay +1KrJvIKIovIV8Yv8NJ4K9hDx7a+xGpHXqh5NLey9K6XNpDqbWwN97FygLLEoedhx +XDjPzBiGJuwIYQI/V2gYVXn3nPcglDFqHpZA8peX3+DwXojqDwzbNhf2CANpMgaD +5OQkGymb+FumKPh1UMEL+IfqtXWwwg2Ix7DtI6i/IQKBgQDasFuUC43F0NM61TFQ +hWuH4mRMp8B4YkhO9M+0/mitHI76cnU63EnJpMYvnp3CLRpVzs/sidEcbfC9gOma +Ui+xxtKers+RQDI4lrwgqyYIzNNpGO7ItlDYs3guTvX5Mvc5x/zIIHhcjhTudzHK +Utgho8PIXvrrF3fSHdL0QJb1tQKBgQDqwdGImGbF3lxsB8t4T/8fozIBwo2dWp4O +YgcVyDpq7Nl+J9Yy5fqTw/sIn6Fa6ZpqAzq0zPSoYJzU7l9+5CcpkkdPEulyER3D +maQaVMUv0cpnXUQSBRNDZJDdbC4eKocxxotyTuLjCVrUwYWuqYWo3aGx42VXWsWm +n4+9AwDBnQKBgBStumscUJKU9XRJtnkLtKhLsvpAnoWDnZzBr2ZI7DL6UVbDPeyL +6fpEN21HTVmQFD5q6ORP/9L1Xl888lniTZo816ujkgMFE/qf3jgkltscKx1z+xhF +jQ2AouuWEdI3jIMNMwzlbRwrXzVRVgbwoHlF1/x5ZraWKIFYyprIBL5FAoGAfXcM +12YsN0A6QPqBglGu1mfQCCTErv6JTsKRatDSd+cR7ly4HAfRvjuV5Ov7vqzu/A2x +yINplrvb1el4XEbvr0YgmmBPJ8mCENICZJg9sur6s/eis8bGntQWoGB63WB5VN76 +FCOZGyIay26KVekAKFobWwlfViqLTBwnJCuAsfkCgYAgrbUKCf+BGgz4epv3T3Ps +kJD2XX1eeQr/bw1IMqY7ZLKCjg7oxM1jjp2xyujY3ZNViaJsnj7ngCIbaOSJNa3t +YS8GDLTL4efNTmPEVzgxhnJdSeYVgMLDwu66M9EYhkw+0Y1PQsNS1+6SO89ySRzP +pFbool/8ZDecmB4ZSa03aw== +-----END PRIVATE KEY----- +``` + +Now we need to get these into the code. We can do this by reading a file using standard C++, but for convenience, this doc will just copy and paste them into code: + +```c++ +auto queue = std::make_unique(); +auto &dm = queue->createManager(); + +std::string key = "-----BEGIN PRIVATE KEY-----\n" + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDIisWpB9FSc/OI\n" + "YsxznQJgioL2aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ\n" + "+VNc63blCPAIiL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFF\n" + "elbLf7JTW80CnNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi\n" + "1QbWD3G1lkTQcCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGo\n" + "swxe5IF+yB6fWFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0\n" + "yv+K0iUBAgMBAAECggEAB2lJgMOvMrLiTyoHkEY/Lj4wNNcNW8g0aQRWlmng7SdE\n" + "84mh1QEspJegaUe7eyNTsTY8TNwzET43jEFQmWCCVliMNmSHWWWF99sgmuL5W5aN\n" + "Ec+4LPnCWDR+pxAKcCEoN4yzhfLTKNzmsqCg0Ih5mKcN3ojZMLodpCfH3WczDkay\n" + "1KrJvIKIovIV8Yv8NJ4K9hDx7a+xGpHXqh5NLey9K6XNpDqbWwN97FygLLEoedhx\n" + "XDjPzBiGJuwIYQI/V2gYVXn3nPcglDFqHpZA8peX3+DwXojqDwzbNhf2CANpMgaD\n" + "5OQkGymb+FumKPh1UMEL+IfqtXWwwg2Ix7DtI6i/IQKBgQDasFuUC43F0NM61TFQ\n" + "hWuH4mRMp8B4YkhO9M+0/mitHI76cnU63EnJpMYvnp3CLRpVzs/sidEcbfC9gOma\n" + "Ui+xxtKers+RQDI4lrwgqyYIzNNpGO7ItlDYs3guTvX5Mvc5x/zIIHhcjhTudzHK\n" + "Utgho8PIXvrrF3fSHdL0QJb1tQKBgQDqwdGImGbF3lxsB8t4T/8fozIBwo2dWp4O\n" + "YgcVyDpq7Nl+J9Yy5fqTw/sIn6Fa6ZpqAzq0zPSoYJzU7l9+5CcpkkdPEulyER3D\n" + "maQaVMUv0cpnXUQSBRNDZJDdbC4eKocxxotyTuLjCVrUwYWuqYWo3aGx42VXWsWm\n" + "n4+9AwDBnQKBgBStumscUJKU9XRJtnkLtKhLsvpAnoWDnZzBr2ZI7DL6UVbDPeyL\n" + "6fpEN21HTVmQFD5q6ORP/9L1Xl888lniTZo816ujkgMFE/qf3jgkltscKx1z+xhF\n" + "jQ2AouuWEdI3jIMNMwzlbRwrXzVRVgbwoHlF1/x5ZraWKIFYyprIBL5FAoGAfXcM\n" + "12YsN0A6QPqBglGu1mfQCCTErv6JTsKRatDSd+cR7ly4HAfRvjuV5Ov7vqzu/A2x\n" + "yINplrvb1el4XEbvr0YgmmBPJ8mCENICZJg9sur6s/eis8bGntQWoGB63WB5VN76\n" + "FCOZGyIay26KVekAKFobWwlfViqLTBwnJCuAsfkCgYAgrbUKCf+BGgz4epv3T3Ps\n" + "kJD2XX1eeQr/bw1IMqY7ZLKCjg7oxM1jjp2xyujY3ZNViaJsnj7ngCIbaOSJNa3t\n" + "YS8GDLTL4efNTmPEVzgxhnJdSeYVgMLDwu66M9EYhkw+0Y1PQsNS1+6SO89ySRzP\n" + "pFbool/8ZDecmB4ZSa03aw==\n" + "-----END PRIVATE KEY-----"; + +std::string cert = "-----BEGIN CERTIFICATE-----\n" + "MIIDdzCCAl+gAwIBAgIUQEtUTmX2HbHwOpDPgFkYx66ei/swDQYJKoZIhvcNAQEL\n" + "BQAwSzELMAkGA1UEBhMCTkwxEjAQBgNVBAcMCUFtc3RlcmRhbTEOMAwGA1UECgwF\n" + "SWNob3IxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTAeFw0yMzEwMDIwODI1NTVa\n" + "Fw0yNDEwMDEwODI1NTVaMEsxCzAJBgNVBAYTAk5MMRIwEAYDVQQHDAlBbXN0ZXJk\n" + "YW0xDjAMBgNVBAoMBUljaG9yMRgwFgYDVQQDDA93d3cuZXhhbXBsZS5jb20wggEi\n" + "MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIisWpB9FSc/OIYsxznQJgioL2\n" + "aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ+VNc63blCPAI\n" + "iL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFFelbLf7JTW80C\n" + "nNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi1QbWD3G1lkTQ\n" + "cCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGoswxe5IF+yB6f\n" + "WFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0yv+K0iUBAgMB\n" + "AAGjUzBRMB0GA1UdDgQWBBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAfBgNVHSMEGDAW\n" + "gBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3\n" + "DQEBCwUAA4IBAQC1QKNwlqrvE5a7M5v151tA/JvIKNBfesIRCg/IG6tB3teSJjAq\n" + "PRsSajMcyvDtlDMkWL093B8Epsc8yt/vN1GgsCfQsMeIWRGvRYM6Qo2WoABaLTuC\n" + "TxH89YrXa9uwUnbesGwKSkpxzj8g6YuzUpLnAAV70LefBn8ZgBJktFVW/ANWwQqD\n" + "KiglduSDGX3NiLByn7nGKejg2dPw6kWpOTVtLsWtgG/5T4vw+INQzPJqy+pUvBeQ\n" + "OrV7cQyq12DJJPxboIRh1KqF8C25NAOnXVtCav985RixHCMaNaXyOXBJeEEASnXg\n" + "pdDrObCgMze03IaHQ1pj3LeMu0OvEMbsRrYW\n" + "-----END CERTIFICATE-----"; + +dm.createServiceManager(); +dm.createServiceManager(Properties{{"Address", Ichor::make_any("127.0.0.1")}, {"Port", Ichor::make_any(static_cast(443))}, {"SslKey", Ichor::make_any(key)}, {"SslCert", Ichor::make_any(cert)}}); + +queue->start(CaptureSigInt); +``` + +This will create a Https-only Host on 127.0.0.1:443. Be mindful that ports below 1000 usually require some sort of root or privileged user to use it. If you want to support Http as well, simply create another Host Service: + +```c++ +dm.createServiceManager(Properties{{"Address", Ichor::make_any("127.0.0.1")}, {"Port", Ichor::make_any(static_cast(8000))}}); +``` + +## Client + +Connecting to a Https webserver is somewhat similar. Using the same certificate from above, we need to tell the client to trust it: + +```c++ +auto queue = std::make_unique(); +auto &dm = queue->createManager(); + +std::string cert = "-----BEGIN CERTIFICATE-----\n" + "MIIDdzCCAl+gAwIBAgIUQEtUTmX2HbHwOpDPgFkYx66ei/swDQYJKoZIhvcNAQEL\n" + "BQAwSzELMAkGA1UEBhMCTkwxEjAQBgNVBAcMCUFtc3RlcmRhbTEOMAwGA1UECgwF\n" + "SWNob3IxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTAeFw0yMzEwMDIwODI1NTVa\n" + "Fw0yNDEwMDEwODI1NTVaMEsxCzAJBgNVBAYTAk5MMRIwEAYDVQQHDAlBbXN0ZXJk\n" + "YW0xDjAMBgNVBAoMBUljaG9yMRgwFgYDVQQDDA93d3cuZXhhbXBsZS5jb20wggEi\n" + "MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIisWpB9FSc/OIYsxznQJgioL2\n" + "aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ+VNc63blCPAI\n" + "iL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFFelbLf7JTW80C\n" + "nNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi1QbWD3G1lkTQ\n" + "cCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGoswxe5IF+yB6f\n" + "WFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0yv+K0iUBAgMB\n" + "AAGjUzBRMB0GA1UdDgQWBBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAfBgNVHSMEGDAW\n" + "gBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3\n" + "DQEBCwUAA4IBAQC1QKNwlqrvE5a7M5v151tA/JvIKNBfesIRCg/IG6tB3teSJjAq\n" + "PRsSajMcyvDtlDMkWL093B8Epsc8yt/vN1GgsCfQsMeIWRGvRYM6Qo2WoABaLTuC\n" + "TxH89YrXa9uwUnbesGwKSkpxzj8g6YuzUpLnAAV70LefBn8ZgBJktFVW/ANWwQqD\n" + "KiglduSDGX3NiLByn7nGKejg2dPw6kWpOTVtLsWtgG/5T4vw+INQzPJqy+pUvBeQ\n" + "OrV7cQyq12DJJPxboIRh1KqF8C25NAOnXVtCav985RixHCMaNaXyOXBJeEEASnXg\n" + "pdDrObCgMze03IaHQ1pj3LeMu0OvEMbsRrYW\n" + "-----END CERTIFICATE-----"; + +dm.createServiceManager(); +dm.createServiceManager(Properties{{"Address", Ichor::make_any("127.0.0.1")}, {"Port", Ichor::make_any(static_cast(443))}, {"ConnectOverSsl", Ichor::make_any(true)}, {"RootCA", Ichor::make_any(cert)}}); + +queue->start(CaptureSigInt); +``` + +And you should be good to go. diff --git a/examples/realtime_example/TestService.h b/examples/realtime_example/TestService.h index f74b1b52..acb6a1b6 100644 --- a/examples/realtime_example/TestService.h +++ b/examples/realtime_example/TestService.h @@ -11,7 +11,11 @@ using namespace Ichor; #define ITERATIONS 20'000 +#ifdef ICHOR_AARCH64 +#define MAXIMUM_DURATION_USEC 1500 +#else #define MAXIMUM_DURATION_USEC 500 +#endif struct ExecuteTaskEvent final : public Event { ExecuteTaskEvent(uint64_t _id, uint64_t _originatingService, uint64_t _priority) noexcept : Event(TYPE, NAME, _id, _originatingService, _priority) {} diff --git a/include/ichor/services/network/http/HttpConnectionService.h b/include/ichor/services/network/http/HttpConnectionService.h index 8edeb710..9f40aca8 100644 --- a/include/ichor/services/network/http/HttpConnectionService.h +++ b/include/ichor/services/network/http/HttpConnectionService.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -55,11 +56,14 @@ namespace Ichor { friend DependencyRegister; std::unique_ptr _httpStream{}; + std::unique_ptr> _sslStream{}; + std::unique_ptr _sslContext{}; std::atomic _priority{INTERNAL_EVENT_PRIORITY}; std::atomic _quit{}; std::atomic _connecting{}; std::atomic _connected{}; std::atomic _tcpNoDelay{}; + std::atomic _useSsl{}; std::atomic _finishedListenAndRead{}; std::atomic _logger{}; IAsioContextService *_asioContextService{}; diff --git a/include/ichor/services/network/http/HttpHostService.h b/include/ichor/services/network/http/HttpHostService.h index 90cfde38..32d7523d 100644 --- a/include/ichor/services/network/http/HttpHostService.h +++ b/include/ichor/services/network/http/HttpHostService.h @@ -8,20 +8,25 @@ #include #include #include +#include #include #include namespace beast = boost::beast; // from namespace http = beast::http; // from namespace net = boost::asio; // from +namespace ssl = boost::asio::ssl; // from using tcp = boost::asio::ip::tcp; // from namespace Ichor { namespace Detail { using HostOutboxMessage = http::response, http::basic_fields>>; + + template struct Connection { Connection(tcp::socket &&_socket) : socket(std::move(_socket)) {} - beast::tcp_stream socket; + Connection(tcp::socket &&_socket, net::ssl::context &ctx) : socket(std::move(_socket), ctx) {} + SocketT socket; boost::circular_buffer outbox{10}; RealtimeMutex mutex{}; }; @@ -49,19 +54,26 @@ namespace Ichor { void fail(beast::error_code, char const* what, bool stopSelf); void listen(tcp::endpoint endpoint, net::yield_context yield); + + template void read(tcp::socket socket, net::yield_context yield); - void sendInternal(std::shared_ptr &connection, http::response, http::basic_fields>> &&res); + + template + void sendInternal(std::shared_ptr> &connection, http::response, http::basic_fields>> &&res); friend DependencyRegister; std::unique_ptr _httpAcceptor{}; - unordered_map> _httpStreams{}; + unordered_map>> _httpStreams{}; + unordered_map>>> _sslStreams{}; + std::unique_ptr _sslContext{}; RealtimeMutex _streamsMutex{}; std::atomic _priority{INTERNAL_EVENT_PRIORITY}; std::atomic _quit{}; std::atomic _goingToCleanupStream{}; std::atomic _finishedListenAndRead{}; std::atomic _tcpNoDelay{}; + std::atomic _useSsl{}; uint64_t _streamIdCounter{}; bool _sendServerHeader{true}; std::atomic _logger{}; diff --git a/src/services/network/http/HttpConnectionService.cpp b/src/services/network/http/HttpConnectionService.cpp index fa8e0822..b1d78d0f 100644 --- a/src/services/network/http/HttpConnectionService.cpp +++ b/src/services/network/http/HttpConnectionService.cpp @@ -14,13 +14,13 @@ Ichor::HttpConnectionService::HttpConnectionService(DependencyRegister ®, Pro Ichor::Task> Ichor::HttpConnectionService::start() { _queue = &GetThreadLocalEventQueue(); - if(!_asioContextService->fibersShouldStop() && !_connecting.load(std::memory_order_acquire) && !_connected.load(std::memory_order_acquire)) { + if (!_asioContextService->fibersShouldStop() && !_connecting.load(std::memory_order_acquire) && !_connected.load(std::memory_order_acquire)) { _quit.store(false, std::memory_order_release); - if (getProperties().contains("Priority")) { + if(getProperties().contains("Priority")) { _priority.store(Ichor::any_cast(getProperties()["Priority"]), std::memory_order_release); } - if (!getProperties().contains("Port") || !getProperties().contains("Address")) { + if(!getProperties().contains("Port") || !getProperties().contains("Address")) { ICHOR_LOG_ERROR_ATOMIC(_logger, "Missing port or address when starting HttpConnectionService"); co_return tl::unexpected(StartError::FAILED); } @@ -30,16 +30,26 @@ Ichor::Task> Ichor::HttpConnectionService: } // fmt::print("connecting to {}\n", Ichor::any_cast(getProperties()["Address"])); - auto address = net::ip::make_address(Ichor::any_cast(getProperties()["Address"])); + boost::system::error_code ec; + auto address = net::ip::make_address(Ichor::any_cast(getProperties()["Address"]), ec); auto port = Ichor::any_cast(getProperties()["Port"]); + if(ec) { + ICHOR_LOG_ERROR_ATOMIC(_logger, "Couldn't parse address \"{}\": {} {}", Ichor::any_cast(getProperties()["Address"]), ec.value(), ec.message()); + co_return tl::unexpected(StartError::FAILED); + } + + if (getProperties().contains("ConnectOverSsl")) { + _useSsl.store(Ichor::any_cast(getProperties()["ConnectOverSsl"]), std::memory_order_release); + } + _connecting.store(true, std::memory_order_release); net::spawn(*_asioContextService->getContext(), [this, address = std::move(address), port](net::yield_context yield) { connect(tcp::endpoint{address, port}, std::move(yield)); }ASIO_SPAWN_COMPLETION_TOKEN); } - if(_asioContextService->fibersShouldStop()) { + if (_asioContextService->fibersShouldStop()) { co_return tl::unexpected(StartError::FAILED); } @@ -150,11 +160,19 @@ Ichor::Task Ichor::HttpConnectionService::sendAsync(Ichor:: req.prepare_payload(); req.keep_alive(); - // Set the timeout for this operation. - // _httpStream should only be modified from the boost thread - _httpStream->expires_after(30s); - INTERNAL_DEBUG("http::write"); - http::async_write(*_httpStream, req, yield[ec]); + if(_useSsl.load(std::memory_order_acquire)) { + // Set the timeout for this operation. + // _sslStream should only be modified from the boost thread + beast::get_lowest_layer(*_sslStream).expires_after(30s); + INTERNAL_DEBUG("http::write"); + http::async_write(*_sslStream, req, yield[ec]); + } else { + // Set the timeout for this operation. + // _httpStream should only be modified from the boost thread + _httpStream->expires_after(30s); + INTERNAL_DEBUG("http::write"); + http::async_write(*_httpStream, req, yield[ec]); + } if (ec) { fail(ec, "HttpConnectionService::sendAsync write"); continue; @@ -172,8 +190,13 @@ Ichor::Task Ichor::HttpConnectionService::sendAsync(Ichor:: http::response, http::basic_fields>> res; INTERNAL_DEBUG("http::read"); + // Receive the HTTP response - http::async_read(*_httpStream, b, res, yield[ec]); + if(_useSsl.load(std::memory_order_acquire)) { + http::async_read(*_sslStream, b, res, yield[ec]); + } else { + http::async_read(*_httpStream, b, res, yield[ec]); + } if (ec) { fail(ec, "HttpConnectionService::sendAsync read"); continue; @@ -185,7 +208,11 @@ Ichor::Task Ichor::HttpConnectionService::sendAsync(Ichor:: INTERNAL_DEBUG("received HTTP response {}", std::string_view(reinterpret_cast(res.body().data()), res.body().size())); // unset the timeout for the next operation. - _httpStream->expires_never(); + if(_useSsl.load(std::memory_order_acquire)) { + beast::get_lowest_layer(*_sslStream).expires_never(); + } else { + _httpStream->expires_never(); + } next.response->status = (HttpStatus) (int) res.result(); next.response->headers.reserve(static_cast(std::distance(std::begin(res), std::end(res)))); @@ -204,14 +231,24 @@ Ichor::Task Ichor::HttpConnectionService::sendAsync(Ichor:: } Ichor::Task Ichor::HttpConnectionService::close() { - if(!_httpStream) { - co_return; + if(_useSsl.load(std::memory_order_acquire)) { + if(!_sslStream) { + co_return; + } + } else { + if(!_httpStream) { + co_return; + } } if(!_connecting.exchange(true, std::memory_order_acq_rel)) { net::spawn(*_asioContextService->getContext(), [this](net::yield_context) { - // _httpStream should only be modified from the boost thread - _httpStream->cancel(); + // _httpStream and _sslStream should only be modified from the boost thread + if(_useSsl.load(std::memory_order_acquire)) { + beast::get_lowest_layer(*_sslStream).cancel(); + } else { + _httpStream->cancel(); + } _queue->pushEvent(getServiceId(), [this]() { _startStopEvent.set(); @@ -229,6 +266,7 @@ Ichor::Task Ichor::HttpConnectionService::close() { _connected.store(false, std::memory_order_release); _connecting.store(false, std::memory_order_release); _httpStream = nullptr; + _sslStream = nullptr; co_return; } @@ -243,32 +281,75 @@ void Ichor::HttpConnectionService::connect(tcp::endpoint endpoint, net::yield_co beast::error_code ec; tcp::resolver resolver(*_asioContextService->getContext()); - // _httpStream should only be modified from the boost thread - _httpStream = std::make_unique(*_asioContextService->getContext()); auto const results = resolver.resolve(endpoint, ec); if(ec) { return fail(ec, "HttpConnectionService::connect resolve"); } - // Never expire until we actually have an operation. - _httpStream->expires_never(); - - // Make the connection on the IP address we get from a lookup - int attempts{}; - while(!_quit.load(std::memory_order_acquire) && !_asioContextService->fibersShouldStop() && attempts < 5) { - _httpStream->async_connect(results, yield[ec]); - if (ec) { - attempts++; - net::steady_timer t{*_asioContextService->getContext()}; - t.expires_after(250ms); - t.async_wait(yield); - } else { - break; + if (_useSsl.load(std::memory_order_acquire)) { + // _sslContext and _sslStream should only be modified from the boost thread + _sslContext = std::make_unique(net::ssl::context::tlsv12); + _sslContext->set_verify_mode(net::ssl::verify_peer); + + if(getProperties().contains("RootCA")) { + std::string &ca = Ichor::any_cast(getProperties()["RootCA"]); + _sslContext->add_certificate_authority(boost::asio::const_buffer(ca.c_str(), ca.size()), ec); } - } - _httpStream->socket().set_option(tcp::no_delay(_tcpNoDelay)); + _sslStream = std::make_unique>(*_asioContextService->getContext(), *_sslContext); + + if(!SSL_set_tlsext_host_name(_sslStream->native_handle(), endpoint.address().to_string().c_str())) { + ec.assign(static_cast(::ERR_get_error()), net::error::get_ssl_category()); + return fail(ec, "HttpConnectionService::connect ssl set host name"); + } + + // Never expire until we actually have an operation. + beast::get_lowest_layer(*_sslStream).expires_never(); + + // Make the connection on the IP address we get from a lookup + int attempts{}; + while(!_quit.load(std::memory_order_acquire) && !_asioContextService->fibersShouldStop() && attempts < 5) { + beast::get_lowest_layer(*_sslStream).async_connect(results, yield[ec]); + if (ec) { + attempts++; + net::steady_timer t{*_asioContextService->getContext()}; + t.expires_after(250ms); + t.async_wait(yield); + } else { + break; + } + } + + beast::get_lowest_layer(*_sslStream).socket().set_option(tcp::no_delay(_tcpNoDelay.load(std::memory_order_acquire))); + + _sslStream->async_handshake(net::ssl::stream_base::client, yield[ec]); + if(ec) { + return fail(ec, "HttpConnectionService::connect ssl handshake"); + } + } else { + // _httpStream should only be modified from the boost thread + _httpStream = std::make_unique(*_asioContextService->getContext()); + + // Never expire until we actually have an operation. + _httpStream->expires_never(); + + // Make the connection on the IP address we get from a lookup + int attempts{}; + while(!_quit.load(std::memory_order_acquire) && !_asioContextService->fibersShouldStop() && attempts < 5) { + _httpStream->async_connect(results, yield[ec]); + if (ec) { + attempts++; + net::steady_timer t{*_asioContextService->getContext()}; + t.expires_after(250ms); + t.async_wait(yield); + } else { + break; + } + } + + _httpStream->socket().set_option(tcp::no_delay(_tcpNoDelay.load(std::memory_order_acquire))); + } if(ec) { // see below for why _connecting has to be set last diff --git a/src/services/network/http/HttpHostService.cpp b/src/services/network/http/HttpHostService.cpp index fda0c818..a6bdf6ac 100644 --- a/src/services/network/http/HttpHostService.cpp +++ b/src/services/network/http/HttpHostService.cpp @@ -5,6 +5,8 @@ #include #include +// + Ichor::HttpHostService::HttpHostService(DependencyRegister ®, Properties props) : AdvancedService(std::move(props)) { reg.registerDependency(this, true); reg.registerDependency(this, true); @@ -28,12 +30,28 @@ Ichor::Task> Ichor::HttpHostService::start _tcpNoDelay.store(Ichor::any_cast(getProperties()["NoDelay"]), std::memory_order_release); } + if(getProperties().contains("SslCert")) { + _useSsl.store(true, std::memory_order_release); + } + + if((_useSsl.load(std::memory_order_acquire) && !getProperties().contains("SslKey")) || + (!_useSsl.load(std::memory_order_acquire) && getProperties().contains("SslKey"))) { + ICHOR_LOG_ERROR_ATOMIC(_logger, "Both SslCert and SslKey properties are required when using ssl"); + co_return tl::unexpected(StartError::FAILED); + } + _queue = &GetThreadLocalEventQueue(); - auto address = net::ip::make_address(Ichor::any_cast(getProperties()["Address"])); + boost::system::error_code ec; + auto address = net::ip::make_address(Ichor::any_cast(getProperties()["Address"]), ec); auto port = Ichor::any_cast(getProperties()["Port"]); - net::spawn(*_asioContextService->getContext(), [this, address = std::move(address), port](net::yield_context yield){ + if(ec) { + ICHOR_LOG_ERROR_ATOMIC(_logger, "Couldn't parse address \"{}\": {} {}", Ichor::any_cast(getProperties()["Address"]), ec.value(), ec.message()); + co_return tl::unexpected(StartError::FAILED); + } + + net::spawn(*_asioContextService->getContext(), [this, address = std::move(address), port](net::yield_context yield) { listen(tcp::endpoint{address, port}, std::move(yield)); }ASIO_SPAWN_COMPLETION_TOKEN); @@ -45,7 +63,7 @@ Ichor::Task Ichor::HttpHostService::stop() { _quit.store(true, std::memory_order_release); if(!_goingToCleanupStream.exchange(true, std::memory_order_acq_rel) && _httpAcceptor) { - if (_httpAcceptor->is_open()) { + if(_httpAcceptor->is_open()) { _httpAcceptor->close(); } net::spawn(*_asioContextService->getContext(), [this](net::yield_context _yield) { @@ -53,9 +71,11 @@ Ichor::Task Ichor::HttpHostService::stop() { { std::unique_lock lg{_streamsMutex}; for (auto &[id, stream]: _httpStreams) { -// stream->quit.store(true, std::memory_order_release); stream->socket.cancel(); } + for (auto &[id, stream]: _sslStreams) { + beast::get_lowest_layer(stream->socket).cancel(); + } } _httpAcceptor = nullptr; @@ -145,6 +165,29 @@ void Ichor::HttpHostService::listen(tcp::endpoint endpoint, net::yield_context y ScopeGuardAtomicCount guard{_finishedListenAndRead}; beast::error_code ec; + if(_useSsl.load(std::memory_order_acquire)) { + _sslContext = std::make_unique(net::ssl::context::tlsv12); + + if(getProperties().contains("SslPassword")) { + _sslContext->set_password_callback( + [password = Ichor::any_cast(getProperties()["SslPassword"])](std::size_t, boost::asio::ssl::context_base::password_purpose) { + return password; + }); + } + + _sslContext->set_options(boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_tlsv1 | + boost::asio::ssl::context::no_tlsv1_1 | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::no_sslv3); + + //pretty sure that Boost.Beast uses references to the data/size, so we should make sure to keep this in memory until we're done. + auto& cert = Ichor::any_cast(getProperties()["SslCert"]); + auto& key = Ichor::any_cast(getProperties()["SslKey"]); + _sslContext->use_certificate_chain(boost::asio::buffer(cert.data(), cert.size())); + _sslContext->use_private_key(boost::asio::buffer(key.data(), key.size()), boost::asio::ssl::context::file_format::pem); + } + _httpAcceptor = std::make_unique(*_asioContextService->getContext()); _httpAcceptor->open(endpoint.protocol(), ec); if(ec) { @@ -185,7 +228,11 @@ void Ichor::HttpHostService::listen(tcp::endpoint endpoint, net::yield_context y return; } - read(std::move(socket), std::move(_yield)); + if(_useSsl) { + read>(std::move(socket), std::move(_yield)); + } else { + read(std::move(socket), std::move(_yield)); + } }ASIO_SPAWN_COMPLETION_TOKEN); } @@ -195,16 +242,30 @@ void Ichor::HttpHostService::listen(tcp::endpoint endpoint, net::yield_context y } } +template void Ichor::HttpHostService::read(tcp::socket socket, net::yield_context yield) { ScopeGuardAtomicCount guard{_finishedListenAndRead}; beast::error_code ec; auto addr = socket.remote_endpoint().address().to_string(); - std::shared_ptr connection; + std::shared_ptr> connection; uint64_t streamId; { std::lock_guard lg(_streamsMutex); streamId = _streamIdCounter++; - connection = _httpStreams.emplace(streamId, std::make_shared(std::move(socket))).first->second; + if constexpr (std::is_same_v) { + connection = _httpStreams.emplace(streamId, std::make_shared>(std::move(socket))).first->second; + } else { + connection = _sslStreams.emplace(streamId, std::make_shared>(std::move(socket), *_sslContext)).first->second; + } + } + + if constexpr (!std::is_same_v) { + connection->socket.async_handshake(ssl::stream_base::server, yield[ec]); + } + + if(ec) { + fail(ec, "HttpHostService::read ssl handshake", false); + return; } // This buffer is required to persist across reads @@ -213,7 +274,11 @@ void Ichor::HttpHostService::read(tcp::socket socket, net::yield_context yield) while(!_quit.load(std::memory_order_acquire) && !_asioContextService->fibersShouldStop()) { // Set the timeout. - connection->socket.expires_after(30s); + if constexpr (std::is_same_v) { + connection->socket.expires_after(30s); + } else { + beast::get_lowest_layer(connection->socket).expires_after(30s); + } // Read a request http::request, http::basic_fields>> req; @@ -259,10 +324,10 @@ void Ichor::HttpHostService::read(tcp::socket socket, net::yield_context yield) co_return {}; } - if (routes != std::end(_handlers)) { + if(routes != std::end(_handlers)) { auto handler = routes->second.find(httpReq.route); - if (handler != std::end(routes->second)) { + if(handler != std::end(routes->second)) { // using reference here leads to heap use after free. Not sure why. HttpResponse httpRes = std::move(*co_await handler->second(httpReq).begin()); @@ -306,14 +371,19 @@ void Ichor::HttpHostService::read(tcp::socket socket, net::yield_context yield) { std::lock_guard lg(_streamsMutex); - _httpStreams.erase(streamId); + if constexpr (std::is_same_v) { + _httpStreams.erase(streamId); + } else { + _sslStreams.erase(streamId); + } } // At this point the connection is closed gracefully ICHOR_LOG_WARN_ATOMIC(_logger, "finished read() {} {}", _quit.load(std::memory_order_acquire), _asioContextService->fibersShouldStop()); } -void Ichor::HttpHostService::sendInternal(std::shared_ptr &connection, http::response, http::basic_fields>> &&res) { +template +void Ichor::HttpHostService::sendInternal(std::shared_ptr> &connection, http::response, http::basic_fields>> &&res) { static_assert(std::is_move_assignable_v, "HostOutboxMessage should be move assignable"); if(_quit.load(std::memory_order_acquire) || _asioContextService->fibersShouldStop()) { @@ -339,16 +409,20 @@ void Ichor::HttpHostService::sendInternal(std::shared_ptr &c // Move message, should be trivially copyable and prevents iterator invalidation auto next = std::move(connection->outbox.front()); lg.unlock(); - connection->socket.expires_after(30s); + if constexpr (std::is_same_v) { + connection->socket.expires_after(30s); + } else { + beast::get_lowest_layer(connection->socket).expires_after(30s); + } beast::error_code ec; http::async_write(connection->socket, next, yield[ec]); - if (ec == http::error::end_of_stream) { + if(ec == http::error::end_of_stream) { fail(ec, "HttpHostService::sendInternal end of stream", false); - } else if (ec == net::error::operation_aborted) { + } else if(ec == net::error::operation_aborted) { fail(ec, "HttpHostService::sendInternal operation aborted", false); - } else if (ec == net::error::bad_descriptor) { + } else if(ec == net::error::bad_descriptor) { fail(ec, "HttpHostService::sendInternal bad descriptor", false); - } else if (ec) { + } else if(ec) { fail(ec, "HttpHostService::sendInternal write", false); } lg.lock(); diff --git a/test/AsyncAutoResetEventTests.cpp b/test/AsyncAutoResetEventTests.cpp deleted file mode 100644 index d10426ef..00000000 --- a/test/AsyncAutoResetEventTests.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "Common.h" -#include - -TEST_CASE("AsyncAutoResetEventTests") { - - SECTION("default constructor") - { - Ichor::AsyncAutoResetEvent event; - } -} \ No newline at end of file diff --git a/test/HttpTests.cpp b/test/HttpTests.cpp index 11489b23..adb73515 100644 --- a/test/HttpTests.cpp +++ b/test/HttpTests.cpp @@ -58,6 +58,92 @@ TEST_CASE("HttpTests") { t.join(); } + SECTION("Https events on same thread") { + testThreadId = std::this_thread::get_id(); + _evt = std::make_unique(); + auto queue = std::make_unique(true); + auto &dm = queue->createManager(); + evtGate = false; + + std::thread t([&]() { + dmThreadId = std::this_thread::get_id(); + + std::string key = "-----BEGIN PRIVATE KEY-----\n" + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDIisWpB9FSc/OI\n" + "YsxznQJgioL2aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ\n" + "+VNc63blCPAIiL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFF\n" + "elbLf7JTW80CnNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi\n" + "1QbWD3G1lkTQcCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGo\n" + "swxe5IF+yB6fWFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0\n" + "yv+K0iUBAgMBAAECggEAB2lJgMOvMrLiTyoHkEY/Lj4wNNcNW8g0aQRWlmng7SdE\n" + "84mh1QEspJegaUe7eyNTsTY8TNwzET43jEFQmWCCVliMNmSHWWWF99sgmuL5W5aN\n" + "Ec+4LPnCWDR+pxAKcCEoN4yzhfLTKNzmsqCg0Ih5mKcN3ojZMLodpCfH3WczDkay\n" + "1KrJvIKIovIV8Yv8NJ4K9hDx7a+xGpHXqh5NLey9K6XNpDqbWwN97FygLLEoedhx\n" + "XDjPzBiGJuwIYQI/V2gYVXn3nPcglDFqHpZA8peX3+DwXojqDwzbNhf2CANpMgaD\n" + "5OQkGymb+FumKPh1UMEL+IfqtXWwwg2Ix7DtI6i/IQKBgQDasFuUC43F0NM61TFQ\n" + "hWuH4mRMp8B4YkhO9M+0/mitHI76cnU63EnJpMYvnp3CLRpVzs/sidEcbfC9gOma\n" + "Ui+xxtKers+RQDI4lrwgqyYIzNNpGO7ItlDYs3guTvX5Mvc5x/zIIHhcjhTudzHK\n" + "Utgho8PIXvrrF3fSHdL0QJb1tQKBgQDqwdGImGbF3lxsB8t4T/8fozIBwo2dWp4O\n" + "YgcVyDpq7Nl+J9Yy5fqTw/sIn6Fa6ZpqAzq0zPSoYJzU7l9+5CcpkkdPEulyER3D\n" + "maQaVMUv0cpnXUQSBRNDZJDdbC4eKocxxotyTuLjCVrUwYWuqYWo3aGx42VXWsWm\n" + "n4+9AwDBnQKBgBStumscUJKU9XRJtnkLtKhLsvpAnoWDnZzBr2ZI7DL6UVbDPeyL\n" + "6fpEN21HTVmQFD5q6ORP/9L1Xl888lniTZo816ujkgMFE/qf3jgkltscKx1z+xhF\n" + "jQ2AouuWEdI3jIMNMwzlbRwrXzVRVgbwoHlF1/x5ZraWKIFYyprIBL5FAoGAfXcM\n" + "12YsN0A6QPqBglGu1mfQCCTErv6JTsKRatDSd+cR7ly4HAfRvjuV5Ov7vqzu/A2x\n" + "yINplrvb1el4XEbvr0YgmmBPJ8mCENICZJg9sur6s/eis8bGntQWoGB63WB5VN76\n" + "FCOZGyIay26KVekAKFobWwlfViqLTBwnJCuAsfkCgYAgrbUKCf+BGgz4epv3T3Ps\n" + "kJD2XX1eeQr/bw1IMqY7ZLKCjg7oxM1jjp2xyujY3ZNViaJsnj7ngCIbaOSJNa3t\n" + "YS8GDLTL4efNTmPEVzgxhnJdSeYVgMLDwu66M9EYhkw+0Y1PQsNS1+6SO89ySRzP\n" + "pFbool/8ZDecmB4ZSa03aw==\n" + "-----END PRIVATE KEY-----"; + + std::string cert = "-----BEGIN CERTIFICATE-----\n" + "MIIDdzCCAl+gAwIBAgIUQEtUTmX2HbHwOpDPgFkYx66ei/swDQYJKoZIhvcNAQEL\n" + "BQAwSzELMAkGA1UEBhMCTkwxEjAQBgNVBAcMCUFtc3RlcmRhbTEOMAwGA1UECgwF\n" + "SWNob3IxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTAeFw0yMzEwMDIwODI1NTVa\n" + "Fw0yNDEwMDEwODI1NTVaMEsxCzAJBgNVBAYTAk5MMRIwEAYDVQQHDAlBbXN0ZXJk\n" + "YW0xDjAMBgNVBAoMBUljaG9yMRgwFgYDVQQDDA93d3cuZXhhbXBsZS5jb20wggEi\n" + "MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIisWpB9FSc/OIYsxznQJgioL2\n" + "aKlqf0fKzrP6IVSq1ApxlMF+hVpz340Yv5rVYizfb/zx46pTbPPJ+VNc63blCPAI\n" + "iL7h3NdYsMaAe+dTfdmQd2/m1kiBQFraceOw5TB7+Y9lFXfUCUFFelbLf7JTW80C\n" + "nNOP4xPmXCusMD8m4LL8z7e7BqKr4VFrrAFXB0EBqpmHA8HXbKAi1QbWD3G1lkTQ\n" + "cCrq8c7pw5oME0RNjfrevlGHATJ2a6zCAeUsIfsDGFpNHFIYqGGoswxe5IF+yB6f\n" + "WFWMjtbbRv93pZfViTFcOUWQuhLFsSN41bt9/XimlxM+ZtX5JDE0yv+K0iUBAgMB\n" + "AAGjUzBRMB0GA1UdDgQWBBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAfBgNVHSMEGDAW\n" + "gBRgTmWQZDeiFMXtL8tyQSdD/HRYPjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3\n" + "DQEBCwUAA4IBAQC1QKNwlqrvE5a7M5v151tA/JvIKNBfesIRCg/IG6tB3teSJjAq\n" + "PRsSajMcyvDtlDMkWL093B8Epsc8yt/vN1GgsCfQsMeIWRGvRYM6Qo2WoABaLTuC\n" + "TxH89YrXa9uwUnbesGwKSkpxzj8g6YuzUpLnAAV70LefBn8ZgBJktFVW/ANWwQqD\n" + "KiglduSDGX3NiLByn7nGKejg2dPw6kWpOTVtLsWtgG/5T4vw+INQzPJqy+pUvBeQ\n" + "OrV7cQyq12DJJPxboIRh1KqF8C25NAOnXVtCav985RixHCMaNaXyOXBJeEEASnXg\n" + "pdDrObCgMze03IaHQ1pj3LeMu0OvEMbsRrYW\n" + "-----END CERTIFICATE-----"; + + dm.createServiceManager({}, 10); + dm.createServiceManager, ILoggerFactory>(); + dm.createServiceManager>(); + dm.createServiceManager(); + dm.createServiceManager(Properties{{"Address", Ichor::make_any("127.0.0.1")}, {"Port", Ichor::make_any(static_cast(8001))}, {"SslKey", Ichor::make_any(key)}, {"SslCert", Ichor::make_any(cert)}}); + dm.createServiceManager>(); + dm.createServiceManager(Properties{{"Address", Ichor::make_any("127.0.0.1")}, {"Port", Ichor::make_any(static_cast(8001))}, {"ConnectOverSsl", Ichor::make_any(true)}, {"RootCA", Ichor::make_any(cert)}}); + + queue->start(CaptureSigInt); + }); + + while(!evtGate) { + std::this_thread::sleep_for(500us); + } + + queue->pushEvent(0, [&]() { + REQUIRE(Ichor::Detail::_local_dm == &dm); + REQUIRE(testThreadId != std::this_thread::get_id()); + REQUIRE(dmThreadId == std::this_thread::get_id()); + _evt->set(); + }); + + t.join(); + } + SECTION("Http events on 4 threads") { testThreadId = std::this_thread::get_id(); _evt = std::make_unique(); diff --git a/test/RedisTests.cpp b/test/RedisTests.cpp index 1b127e21..c18a9608 100644 --- a/test/RedisTests.cpp +++ b/test/RedisTests.cpp @@ -36,7 +36,7 @@ TEST_CASE("RedisTests") { TEST_CASE("RedisTests") { SECTION("Empty Test so that Catch2 exits with 0") { - + REQUIRE(true); } } diff --git a/test/TestServices/HttpThreadService.h b/test/TestServices/HttpThreadService.h index 4820666f..1a2daa86 100644 --- a/test/TestServices/HttpThreadService.h +++ b/test/TestServices/HttpThreadService.h @@ -116,6 +116,7 @@ class HttpThreadService final : public AdvancedService { if(response.status == HttpStatus::ok) { auto msg = _serializer->deserialize(std::move(response.body)); + fmt::print("Received {}:{}\n", msg->id, msg->val); } else { throw std::runtime_error("Status not ok"); }