From 32c47050dff1197b9ea9aff3b368957e9b0629ad Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 1 Apr 2024 16:55:40 -0400 Subject: [PATCH 01/55] add authentication signer --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 61 ++++ .../src/clp/aws/AwsAuthenticationSigner.hpp | 266 ++++++++++++++++++ components/core/src/clp/aws/Constants.hpp | 26 ++ components/core/src/clp/aws/hash_utils.hpp | 83 ++++++ components/core/src/clp/clp/CMakeLists.txt | 4 + 5 files changed, 440 insertions(+) create mode 100644 components/core/src/clp/aws/AwsAuthenticationSigner.cpp create mode 100644 components/core/src/clp/aws/AwsAuthenticationSigner.hpp create mode 100644 components/core/src/clp/aws/Constants.hpp create mode 100644 components/core/src/clp/aws/hash_utils.hpp diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp new file mode 100644 index 000000000..c020c27e2 --- /dev/null +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -0,0 +1,61 @@ +#include "AwsAuthenticationSigner.hpp" + +#include + +namespace clp::aws { +S3Url::S3Url(std::string const& url) { + // Regular expression to match virtual host-style HTTP URL format + std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); + // Regular expression to match path-style HTTP URL format + std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); + + std::smatch match; + if (std::regex_match(url, match, regex1)) { + m_bucket = match[1].str(); + m_region = match[3].str(); + m_path = match[4].str(); + } else if (std::regex_match(url, match, regex2)) { + m_region = match[2].str(); + m_bucket = match[3].str(); + m_path = match[4].str(); + } else { + throw std::invalid_argument("Invalid S3 HTTP URL format"); + } + + if (m_region.empty()) { + m_region = cDefaultRegion; + } + m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); +} + +std::string AwsAuthenticationSigner::generate_presigned_url(std::string const& url, HttpMethod method) { + if (HttpMethod::GET != method) { + throw std::runtime_error("Unsupported HTTP method!"); + } + + S3Url s3_url(url); + auto s3_region = s3_url.get_region(); + + // Gets current time + auto now = std::chrono::system_clock::now(); + auto date_string = get_date_string(now); + auto timestamp_string = get_timestamp_string(now); + + auto scope = get_scope(date_string, s3_region); + auto query_string = get_default_query_string(scope, timestamp_string); + auto canonical_request = get_canonical_request(method, s3_url, query_string); + auto string_to_sign = get_string_to_sign(scope, timestamp_string, canonical_request); + auto signature_key = get_signature_key(s3_region, date_string); + auto signature = get_signature(signature_key, string_to_sign); + + return fmt::format( + "https://{}{}?{}&{}={}", + s3_url.get_host(), + s3_url.get_path(), + query_string, + cXAmzSignature, + signature + ); +} + +} // namespace clp::aws diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp new file mode 100644 index 000000000..657897f72 --- /dev/null +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -0,0 +1,266 @@ +#ifndef CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP +#define CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP + +#include + +#include +#include + +#include "Constants.hpp" +#include "hash_utils.hpp" + +namespace clp::aws { +/** + * Class for parsing S3 HTTP URL + */ +class S3Url { +public: + // Constructor + explicit S3Url(std::string const& url); + + S3Url() = delete; + + // Methods + std::string& get_host() { return m_host; } + + std::string& get_bucket() { return m_bucket; } + + std::string& get_region() { return m_region; } + + std::string& get_path() { return m_path; } + +private: + std::string m_host; + std::string m_bucket; + std::string m_region; + std::string m_path; +}; + +/** + * Class for signing AWS requests + */ +class AwsAuthenticationSigner { +public: + using time_point = std::chrono::system_clock::time_point; + + // Default expire time of presigned URL in seconds + static constexpr int const cDefaultExpireTime = 86'400; // 24 hours + + // Types + enum class HttpMethod : uint8_t { + GET, + PUT, + POST, + DELETE + }; + + enum class AwsService : uint8_t { + S3 + }; + + // Constructors + AwsAuthenticationSigner( + std::string& access_key_id, + std::string& secret_access_key, + AwsService service = AwsService::S3 + ) + : m_access_key_id(access_key_id), + m_secret_access_key(secret_access_key), + m_service(service) { + if (AwsService::S3 != m_service) { + throw std::invalid_argument("Unsupported service"); + } + } + + AwsAuthenticationSigner() { + m_access_key_id = getenv("AWS_ACCESS_KEY_ID"); + m_secret_access_key = getenv("AWS_SECRET_ACCESS_KEY"); + + if (m_access_key_id.empty()) { + throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); + } + if (m_secret_access_key.empty()) { + throw std::invalid_argument("AWS_SECRET_ACCESS_KEY environment variable is not set"); + } + } + + // Methods + /** + * Generates a presigned URL using AWS Signature Version 4 + * @param url URL + * @param method HTTP method + * @return The generated presigned URL + */ + std::string generate_presigned_url(std::string const& url, HttpMethod method); + +private: + /** + * Converts an HttpMethod to a string + * @param method HTTP method + * @return The converted string + */ + static std::string get_method_string(HttpMethod method) { + switch (method) { + case HttpMethod::GET: + return "GET"; + case HttpMethod::PUT: + return "PUT"; + case HttpMethod::POST: + return "POST"; + case HttpMethod::DELETE: + return "DELETE"; + default: + throw std::runtime_error("Invalid HTTP method"); + } + } + + /** + * Gets the string to sign + * @param scope + * @param timestamp_string + * @param canonical_request + * @return String to sign + */ + static std::string get_string_to_sign( + std::string& scope, + std::string& timestamp_string, + std::string const& canonical_request + ) { + return fmt::format( + "{}\n{}\n{}\n{}", + cAws4HmacSha256, + timestamp_string, + scope, + get_sha256_hash(canonical_request) + ); + } + + /** + * Gets the canonical request string + * @param method HTTP method + * @param url S3 URL + * @param query_string Query string + * @return Canonical request + */ + static std::string + get_canonical_request(HttpMethod method, S3Url& url, std::string const& query_string) { + return fmt::format( + "{}\n{}\n{}\n{}:{}\n\n{}\n{}", + get_method_string(method), + get_encoded_uri(url.get_path(), false), + query_string, + cDefaultSignedHeaders, + url.get_host(), + cDefaultSignedHeaders, + cUnsignedPayload + ); + } + + /** + * Gets the encoded URI + * @param value + * @param encode_slash + * @return The encoded URI + */ + static std::string get_encoded_uri(std::string const& value, bool encode_slash = true) { + std::string encoded_uri; + + for (char c : value) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + encoded_uri += c; + continue; + } + + if (c == '/' && false == encode_slash) { + encoded_uri += c; + continue; + } + + encoded_uri += fmt::format("%{:02X}", static_cast(static_cast(c))); + } + + return encoded_uri; + } + + /** + * Gets the scope + * @param date_string + * @param region + * @return The scope + */ + static std::string get_scope(std::string& date_string, std::string const& region) { + return fmt::format("{}/{}/{}/{}", date_string, region, cS3Service, cAws4Request); + } + + /** + * Gets the timestamp string + * @param timestamp + * @return The timestamp string + */ + static std::string get_timestamp_string(time_point& timestamp) { + return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); + } + + /** + * Gets the date string + * @param timestamp + * @return The date string + */ + static std::string get_date_string(time_point& timestamp) { + return fmt::format("{:%Y%m%d}", timestamp); + } + + /** + * Gets the default query string + * @param scope + * @param timestamp_string + * @return + */ + std::string get_default_query_string(std::string& scope, std::string& timestamp_string) { + return fmt::format( + "{}={}&{}={}&{}={}&{}={}&{}={}", + cXAmzAlgorithm, + cAws4HmacSha256, + cXAmzCredential, + get_encoded_uri(m_access_key_id + "/" + scope), + cXAmzDate, + timestamp_string, + cXAmzExpires, + cDefaultExpireTime, + cXAmzSignedHeaders, + cDefaultSignedHeaders + ); + } + + /** + * Gets the signature key + * @param region + * @param date_string + * @return + */ + std::string get_signature_key(std::string const& region, std::string const& date_string) { + std::string date_key = get_hmac_sha256_hash(cAws4 + m_secret_access_key, date_string); + std::string date_region_key = get_hmac_sha256_hash(date_key, region); + std::string date_region_service_key = get_hmac_sha256_hash(date_region_key, cS3Service); + return get_hmac_sha256_hash(date_region_service_key, cAws4Request); + } + + /** + * Gets the signature + * @param signature_key + * @param string_to_sign + * @return + */ + static std::string + get_signature(std::string const& signature_key, std::string const& string_to_sign) { + return get_hmac_sha256_hash(signature_key, string_to_sign, true); + } + + // Variables + std::string m_access_key_id; + std::string m_secret_access_key; + AwsService m_service{AwsService::S3}; +}; +} // namespace clp::aws + +#endif // CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp new file mode 100644 index 000000000..874c1e531 --- /dev/null +++ b/components/core/src/clp/aws/Constants.hpp @@ -0,0 +1,26 @@ +#ifndef CLP_AWS_CONSTANTS_HPP +#define CLP_AWS_CONSTANTS_HPP + +namespace clp::aws { +// Query String Parameter Names +constexpr char const* const cXAmzAlgorithm = "X-Amz-Algorithm"; +constexpr char const* const cXAmzCredential = "X-Amz-Credential"; +constexpr char const* const cXAmzContentSha256 = "X-Amz-Content-Sha256"; +constexpr char const* const cXAmzDate = "X-Amz-Date"; +constexpr char const* const cXAmzExpires = "X-Amz-Expires"; +constexpr char const* const cXAmzSignature = "X-Amz-Signature"; +constexpr char const* const cXAmzSignedHeaders = "X-Amz-SignedHeaders"; + +// Other Constants +constexpr char const* const cAws4 = "AWS4"; +constexpr char const* const cAws4Request = "aws4_request"; +constexpr char const* const cAws4HmacSha256 = "AWS4-HMAC-SHA256"; +constexpr char const* const cDefaultSignedHeaders = "host"; +constexpr char const* const cDefaultRegion "us-east-1"; +constexpr char const* const cEmptyStringSha256 + = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +constexpr char const* const cS3Service = "s3"; +constexpr char const* const cUnsignedPayload = "UNSIGNED-PAYLOAD"; +} // namespace clp::aws + +#endif // CLP_AWS_CONSTANTS_HPP diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp new file mode 100644 index 000000000..8e3758d82 --- /dev/null +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -0,0 +1,83 @@ +#ifndef CLP_AWS_HASH_UTILS_HPP +#define CLP_AWS_HASH_UTILS_HPP + +#include +#include +#include + +#include + +/** + * Converts a char array to a string + * @param a + * @param size + * @return The converted string + */ +static std::string char_array_to_string(unsigned char const* a, size_t size) { + std::string hex_string; + for (size_t i = 0; i < size; i++) { + hex_string += fmt::format("{:02x}", static_cast(a[i])); + } + return hex_string; +} + +/** + * Gets the HMAC-SHA256 hash + * @param key + * @param value + * @param hex_output + * @return The HMAC SHA256 hash + */ +static std::string +get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_output = false) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + HMAC_CTX* hmac = HMAC_CTX_new(); + HMAC_Init_ex(hmac, key.c_str(), key.size(), EVP_sha256(), nullptr); + HMAC_Update(hmac, reinterpret_cast(value.c_str()), value.size()); + unsigned int len = SHA256_DIGEST_LENGTH; + HMAC_Final(hmac, hash, &len); + HMAC_CTX_free(hmac); + + if (hex_output) { + return char_array_to_string(hash, SHA256_DIGEST_LENGTH); + } else { + std::string result; + result.assign(reinterpret_cast(hash), SHA256_DIGEST_LENGTH); + return result; + } +} + +/** + * Gets the SHA256 hash + * @param input + * @return The SHA256 hash + */ +static std::string get_sha256_hash(std::string const& input) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, input.c_str(), input.size()); + SHA256_Final(hash, &sha256); + + return char_array_to_string(hash, SHA256_DIGEST_LENGTH); +} + +// Init SHA256 hash +void init_sha256_hash(SHA256_CTX* sha256) { + SHA256_Init(sha256); +} + +// Update SHA256 hash +void update_sha256_hash(SHA256_CTX* sha256, void* input, size_t length) { + SHA256_Update(sha256, input, length); +} + +// Finalize SHA256 hash +std::string finalize_sha256_hash(SHA256_CTX* sha256) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, sha256); + + return char_array_to_string(hash, SHA256_DIGEST_LENGTH); +} + +#endif // CLP_AWS_HASH_UTILS_HPP diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 0f18777d9..d3223fd62 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -1,6 +1,10 @@ set( CLP_SOURCES ../ArrayBackedPosIntSet.hpp + ../aws/AwsAuthenticationSigner.cpp + ../aws/AwsAuthenticationSigner.hpp + ../aws/Constants.hpp + ../aws/hash_utils.hpp ../BufferedFileReader.cpp ../BufferedFileReader.hpp ../BufferReader.cpp From 2231742126adac85109006470c11105a268b7b70 Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 5 Apr 2024 10:57:40 -0400 Subject: [PATCH 02/55] add s3 log ingestion support for clp --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 9 +- .../src/clp/aws/AwsAuthenticationSigner.hpp | 17 ++- components/core/src/clp/aws/Constants.hpp | 2 +- components/core/src/clp/aws/hash_utils.hpp | 35 +++-- components/core/src/clp/clp/CMakeLists.txt | 10 ++ .../core/src/clp/clp/CommandLineArguments.cpp | 15 +++ .../core/src/clp/clp/CommandLineArguments.hpp | 8 ++ .../core/src/clp/clp/FileCompressor.cpp | 123 +++++++++++------- .../core/src/clp/clp/FileCompressor.hpp | 25 +++- components/core/src/clp/clp/compression.cpp | 10 +- components/core/src/clp/clp/run.cpp | 56 ++++---- 11 files changed, 212 insertions(+), 98 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index c020c27e2..4afddabf2 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -5,9 +5,11 @@ namespace clp::aws { S3Url::S3Url(std::string const& url) { // Regular expression to match virtual host-style HTTP URL format - std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); +// std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); + std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); // Regular expression to match path-style HTTP URL format - std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); +// std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); + std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; if (std::regex_match(url, match, regex1)) { @@ -28,12 +30,11 @@ S3Url::S3Url(std::string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -std::string AwsAuthenticationSigner::generate_presigned_url(std::string const& url, HttpMethod method) { +std::string AwsAuthenticationSigner::generate_presigned_url(S3Url &s3_url, HttpMethod method) { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); } - S3Url s3_url(url); auto s3_region = s3_url.get_region(); // Gets current time diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 657897f72..20db0a4b2 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -41,7 +41,7 @@ class S3Url { */ class AwsAuthenticationSigner { public: - using time_point = std::chrono::system_clock::time_point; + using TimePoint = std::chrono::system_clock::time_point; // Default expire time of presigned URL in seconds static constexpr int const cDefaultExpireTime = 86'400; // 24 hours @@ -72,10 +72,9 @@ class AwsAuthenticationSigner { } } - AwsAuthenticationSigner() { - m_access_key_id = getenv("AWS_ACCESS_KEY_ID"); - m_secret_access_key = getenv("AWS_SECRET_ACCESS_KEY"); - + AwsAuthenticationSigner() + : m_access_key_id(getenv("AWS_ACCESS_KEY_ID")), + m_secret_access_key(getenv("AWS_SECRET_ACCESS_KEY")) { if (m_access_key_id.empty()) { throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); } @@ -87,11 +86,11 @@ class AwsAuthenticationSigner { // Methods /** * Generates a presigned URL using AWS Signature Version 4 - * @param url URL + * @param s3_url S3 URL * @param method HTTP method * @return The generated presigned URL */ - std::string generate_presigned_url(std::string const& url, HttpMethod method); + std::string generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET); private: /** @@ -197,7 +196,7 @@ class AwsAuthenticationSigner { * @param timestamp * @return The timestamp string */ - static std::string get_timestamp_string(time_point& timestamp) { + static std::string get_timestamp_string(TimePoint& timestamp) { return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); } @@ -206,7 +205,7 @@ class AwsAuthenticationSigner { * @param timestamp * @return The date string */ - static std::string get_date_string(time_point& timestamp) { + static std::string get_date_string(TimePoint& timestamp) { return fmt::format("{:%Y%m%d}", timestamp); } diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp index 874c1e531..a12b5a086 100644 --- a/components/core/src/clp/aws/Constants.hpp +++ b/components/core/src/clp/aws/Constants.hpp @@ -16,7 +16,7 @@ constexpr char const* const cAws4 = "AWS4"; constexpr char const* const cAws4Request = "aws4_request"; constexpr char const* const cAws4HmacSha256 = "AWS4-HMAC-SHA256"; constexpr char const* const cDefaultSignedHeaders = "host"; -constexpr char const* const cDefaultRegion "us-east-1"; +constexpr char const* const cDefaultRegion = "us-east-1"; constexpr char const* const cEmptyStringSha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; constexpr char const* const cS3Service = "s3"; diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp index 8e3758d82..9d1e64ab3 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -3,17 +3,19 @@ #include #include -#include #include +#include + +namespace clp::aws { /** * Converts a char array to a string * @param a * @param size * @return The converted string */ -static std::string char_array_to_string(unsigned char const* a, size_t size) { +inline std::string char_array_to_string(unsigned char const* a, size_t size) { std::string hex_string; for (size_t i = 0; i < size; i++) { hex_string += fmt::format("{:02x}", static_cast(a[i])); @@ -28,7 +30,7 @@ static std::string char_array_to_string(unsigned char const* a, size_t size) { * @param hex_output * @return The HMAC SHA256 hash */ -static std::string +inline std::string get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_output = false) { unsigned char hash[SHA256_DIGEST_LENGTH]; HMAC_CTX* hmac = HMAC_CTX_new(); @@ -52,7 +54,7 @@ get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_ * @param input * @return The SHA256 hash */ -static std::string get_sha256_hash(std::string const& input) { +inline std::string get_sha256_hash(std::string const& input) { unsigned char hash[SHA256_DIGEST_LENGTH]; SHA256_CTX sha256; SHA256_Init(&sha256); @@ -62,22 +64,35 @@ static std::string get_sha256_hash(std::string const& input) { return char_array_to_string(hash, SHA256_DIGEST_LENGTH); } -// Init SHA256 hash -void init_sha256_hash(SHA256_CTX* sha256) { +/** + * Initializes SHA256 hash + * @param sha256 + */ +inline void init_sha256_hash(SHA256_CTX* sha256) { SHA256_Init(sha256); } -// Update SHA256 hash -void update_sha256_hash(SHA256_CTX* sha256, void* input, size_t length) { +/** + * Updates SHA256 hash + * @param sha256 + * @param input + * @param length + */ +inline void update_sha256_hash(SHA256_CTX* sha256, void* input, size_t length) { SHA256_Update(sha256, input, length); } -// Finalize SHA256 hash -std::string finalize_sha256_hash(SHA256_CTX* sha256) { +/** + * Finalizes SHA256 hash + * @param sha256 + * @return The SHA256 hash + */ +inline std::string finalize_sha256_hash(SHA256_CTX* sha256) { unsigned char hash[SHA256_DIGEST_LENGTH]; SHA256_Final(hash, sha256); return char_array_to_string(hash, SHA256_DIGEST_LENGTH); } +} // namespace clp::aws #endif // CLP_AWS_HASH_UTILS_HPP diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index d3223fd62..9f9ffaff9 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -9,6 +9,11 @@ set( ../BufferedFileReader.hpp ../BufferReader.cpp ../BufferReader.hpp + ../CurlDownloadHandler.cpp + ../CurlDownloadHandler.hpp + ../CurlEasyHandle.hpp + ../CurlOperationFailed.hpp + ../CurlStringList.hpp ../database_utils.cpp ../database_utils.hpp ../Defs.h @@ -130,8 +135,12 @@ set( ../streaming_compression/zstd/Constants.hpp ../streaming_compression/zstd/Decompressor.cpp ../streaming_compression/zstd/Decompressor.hpp + ../NetworkReader.cpp + ../NetworkReader.hpp ../StringReader.cpp ../StringReader.hpp + ../Thread.cpp + ../Thread.hpp ../time_types.hpp ../TimestampPattern.cpp ../TimestampPattern.hpp @@ -174,6 +183,7 @@ target_include_directories(clp PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_link_libraries(clp PRIVATE Boost::filesystem Boost::iostreams Boost::program_options + ${CURL_LIBRARIES} fmt::fmt log_surgeon::log_surgeon spdlog::spdlog diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index ccdc99793..40e23faa0 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -332,7 +332,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // boost::program_options doesn't support boolean flags which can be set to false, so // we use a string argument to set the flag manually. string sort_input_files_str = "true"; + string input_source_str = "filesystem"; options_compression.add_options()( + "input-source", + po::value(&input_source_str) + ->value_name("SOURCE") + ->default_value(input_source_str), + "Source of input paths. 'filesystem' or 's3'." + )( "remove-path-prefix", po::value(&m_path_prefix_to_remove) ->value_name("DIR") @@ -420,6 +427,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { return ParsingResult::InfoCommand; } + if ("filesystem" == input_source_str) { + m_input_source = InputSource::Filesystem; + } else if ("s3" == input_source_str) { + m_input_source = InputSource::S3; + } else { + throw invalid_argument("Invalid input source specified."); + } + // Validate at least one input path should exist (we validate that the file isn't empty // later) if (m_input_paths.empty() && m_path_list_path.empty()) { diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index b9cf15740..b6ac0870c 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -19,6 +19,11 @@ class CommandLineArguments : public CommandLineArgumentsBase { ExtractIr = 'i' }; + enum class InputSource : uint8_t { + Filesystem, + S3 + }; + // Constructors explicit CommandLineArguments(std::string const& program_name) : CommandLineArgumentsBase(program_name), @@ -67,6 +72,8 @@ class CommandLineArguments : public CommandLineArgumentsBase { std::string const& get_archives_dir() const { return m_archives_dir; } + InputSource get_input_source() const { return m_input_source; } + std::vector const& get_input_paths() const { return m_input_paths; } std::string const& get_orig_file_id() const { return m_orig_file_id; } @@ -102,6 +109,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { int m_compression_level; Command m_command; std::string m_archives_dir; + InputSource m_input_source; std::vector m_input_paths; GlobalMetadataDBConfig m_metadata_db_config; }; diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 9898602cc..5beb242a1 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -116,38 +116,16 @@ bool FileCompressor::compress_file( streaming_archive::writer::Archive& archive_writer, bool use_heuristic ) { - std::string file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); - - PROFILER_SPDLOG_INFO("Start parsing {}", file_name) - Profiler::start_continuous_measurement(); - - m_file_reader.open(file_to_compress.get_path()); - - // Check that file is UTF-8 encoded - if (auto error_code = m_file_reader.try_refill_buffer_if_empty(); - ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) - { - if (ErrorCode_errno == error_code) { - SPDLOG_ERROR( - "Failed to read {} into buffer, errno={}", - file_to_compress.get_path(), - errno - ); - } else { - SPDLOG_ERROR( - "Failed to read {} into buffer, error={}", - file_to_compress.get_path(), - error_code - ); - } - return false; - } - char const* utf8_validation_buf{nullptr}; - size_t peek_size{0}; - m_file_reader.peek_buffered_data(utf8_validation_buf, peek_size); bool succeeded = true; - auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); - if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { + std::string file_name; + if (CommandLineArguments::InputSource::S3 == m_input_source) { + aws::S3Url s3_url(file_to_compress.get_path()); + file_name = s3_url.get_path().substr(1); + + PROFILER_SPDLOG_INFO("Start parsing {}", file_name) + Profiler::start_continuous_measurement( + ); + NetworkReader network_reader(m_aws_auth_signer.value().generate_presigned_url(s3_url)); if (use_heuristic) { parse_and_encode_with_heuristic( target_data_size_of_dicts, @@ -156,7 +134,7 @@ bool FileCompressor::compress_file( file_to_compress.get_path_for_compression(), file_to_compress.get_group_id(), archive_writer, - m_file_reader + network_reader ); } else { parse_and_encode_with_library( @@ -166,25 +144,78 @@ bool FileCompressor::compress_file( file_to_compress.get_path_for_compression(), file_to_compress.get_group_id(), archive_writer, - m_file_reader + network_reader ); } - } else { - if (false - == try_compressing_as_archive( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress, - archive_writer, - use_heuristic - )) + } else if (CommandLineArguments::InputSource::Filesystem == m_input_source) { + file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); + PROFILER_SPDLOG_INFO("Start parsing {}", file_name) + Profiler::start_continuous_measurement(); + + m_file_reader.open(file_to_compress.get_path()); + + // Check that file is UTF-8 encoded + if (auto error_code = m_file_reader.try_refill_buffer_if_empty(); + ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) { - succeeded = false; + if (ErrorCode_errno == error_code) { + SPDLOG_ERROR( + "Failed to read {} into buffer, errno={}", + file_to_compress.get_path(), + errno + ); + } else { + SPDLOG_ERROR( + "Failed to read {} into buffer, error={}", + file_to_compress.get_path(), + error_code + ); + } + return false; + } + char const* utf8_validation_buf{nullptr}; + size_t peek_size{0}; + m_file_reader.peek_buffered_data(utf8_validation_buf, peek_size); + auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); + if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { + if (use_heuristic) { + parse_and_encode_with_heuristic( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + m_file_reader + ); + } else { + parse_and_encode_with_library( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + m_file_reader + ); + } + } else { + if (false + == try_compressing_as_archive( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress, + archive_writer, + use_heuristic + )) + { + succeeded = false; + } } - } - m_file_reader.close(); + m_file_reader.close(); + } Profiler::stop_continuous_measurement(); LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::ParseLogFile) diff --git a/components/core/src/clp/clp/FileCompressor.hpp b/components/core/src/clp/clp/FileCompressor.hpp index b8b6c55fd..2d6b77891 100644 --- a/components/core/src/clp/clp/FileCompressor.hpp +++ b/components/core/src/clp/clp/FileCompressor.hpp @@ -14,6 +14,9 @@ #include "../MessageParser.hpp" #include "../ParsedMessage.hpp" #include "../streaming_archive/writer/Archive.hpp" +#include "../NetworkReader.hpp" +#include "aws/AwsAuthenticationSigner.hpp" +#include "CommandLineArguments.hpp" #include "FileToCompress.hpp" namespace clp::clp { @@ -24,11 +27,27 @@ class FileCompressor { public: // Constructors FileCompressor( + CommandLineArguments::InputSource input_source, boost::uuids::random_generator& uuid_generator, std::unique_ptr reader_parser ) - : m_uuid_generator(uuid_generator), - m_reader_parser(std::move(reader_parser)) {} + : m_input_source(input_source), + m_uuid_generator(uuid_generator), + m_reader_parser(std::move(reader_parser)) { + if (CommandLineArguments::InputSource::S3 == input_source) { + m_aws_auth_signer.emplace(); + if (ErrorCode_Success != NetworkReader::init()) { + SPDLOG_ERROR("Failed to initialize streaming reader"); + throw std::runtime_error("Failed to initialize streaming reader"); + } + } + } + + ~FileCompressor() { + if (CommandLineArguments::InputSource::S3 == m_input_source) { + NetworkReader::deinit(); + } + } // Methods /** @@ -149,10 +168,12 @@ class FileCompressor { ); // Variables + CommandLineArguments::InputSource m_input_source; boost::uuids::random_generator& m_uuid_generator; BufferedFileReader m_file_reader; LibarchiveReader m_libarchive_reader; LibarchiveFileReader m_libarchive_file_reader; + std::optional m_aws_auth_signer; MessageParser m_message_parser; ParsedMessage m_parsed_message; std::unique_ptr m_reader_parser; diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 1741557bc..3852afbd3 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -117,7 +117,11 @@ bool compress( archive_writer.add_empty_directories(empty_directory_paths); bool all_files_compressed_successfully = true; - FileCompressor file_compressor(uuid_generator, std::move(reader_parser)); + FileCompressor file_compressor( + command_line_args.get_input_source(), + uuid_generator, + std::move(reader_parser) + ); auto target_data_size_of_dictionaries = command_line_args.get_target_data_size_of_dictionaries(); @@ -127,7 +131,9 @@ bool compress( if (command_line_args.show_progress()) { num_files_to_compress = files_to_compress.size() + grouped_files_to_compress.size(); } - if (command_line_args.sort_input_files()) { + if (CommandLineArguments::InputSource::Filesystem == command_line_args.get_input_source() + && command_line_args.sort_input_files()) + { sort(files_to_compress.begin(), files_to_compress.end(), file_gt_last_write_time_comparator ); } diff --git a/components/core/src/clp/clp/run.cpp b/components/core/src/clp/clp/run.cpp index 5a3b0eb27..4d35b34b1 100644 --- a/components/core/src/clp/clp/run.cpp +++ b/components/core/src/clp/clp/run.cpp @@ -63,37 +63,45 @@ int run(int argc, char const* argv[]) { reader_parser = std::make_unique(schema_file_path); } - boost::filesystem::path path_prefix_to_remove(command_line_args.get_path_prefix_to_remove() - ); - - // Validate input paths exist - if (false == validate_paths_exist(input_paths)) { - return -1; - } - // Get paths of all files we need to compress vector files_to_compress; + vector grouped_files_to_compress; vector empty_directory_paths; - for (auto const& input_path : input_paths) { - if (false - == find_all_files_and_empty_directories( - path_prefix_to_remove, - input_path, - files_to_compress, - empty_directory_paths - )) - { + if (CommandLineArguments::InputSource::S3 == command_line_args.get_input_source()) { + for (auto const& input_path : input_paths) { + files_to_compress.emplace_back(input_path, input_path, 0); + } + } else if ((CommandLineArguments::InputSource::Filesystem + == command_line_args.get_input_source())) + { + boost::filesystem::path path_prefix_to_remove( + command_line_args.get_path_prefix_to_remove() + ); + + // Validate input paths exist + if (false == validate_paths_exist(input_paths)) { return -1; } - } - vector grouped_files_to_compress; + for (auto const& input_path : input_paths) { + if (false + == find_all_files_and_empty_directories( + path_prefix_to_remove, + input_path, + files_to_compress, + empty_directory_paths + )) + { + return -1; + } + } - if (files_to_compress.empty() && empty_directory_paths.empty() - && grouped_files_to_compress.empty()) - { - SPDLOG_ERROR("No files/directories to compress."); - return -1; + if (files_to_compress.empty() && empty_directory_paths.empty() + && grouped_files_to_compress.empty()) + { + SPDLOG_ERROR("No files/directories to compress."); + return -1; + } } bool compression_successful; From ae8bac6b9502af3c13ed8f13f38f819d14b69cba Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 5 Apr 2024 17:48:53 -0400 Subject: [PATCH 03/55] refactor the code --- components/core/.clang-format | 2 +- .../src/clp/aws/AwsAuthenticationSigner.cpp | 6 ++-- .../src/clp/aws/AwsAuthenticationSigner.hpp | 35 +++++++++++-------- components/core/src/clp/aws/hash_utils.hpp | 15 ++++---- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/components/core/.clang-format b/components/core/.clang-format index 18195cc31..76576ff14 100644 --- a/components/core/.clang-format +++ b/components/core/.clang-format @@ -75,7 +75,7 @@ IncludeCategories: # Library headers. Update when adding new libraries. # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec. - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|mariadb\ -|mongocxx|msgpack|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|zstd)" +|mongocxx|msgpack|openssl|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|zstd)" Priority: 3 # C system headers - Regex: "^<.+\\.h>" diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 4afddabf2..cabc4c38d 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -5,10 +5,10 @@ namespace clp::aws { S3Url::S3Url(std::string const& url) { // Regular expression to match virtual host-style HTTP URL format -// std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); + // std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); // Regular expression to match path-style HTTP URL format -// std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); + // std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; @@ -30,7 +30,7 @@ S3Url::S3Url(std::string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -std::string AwsAuthenticationSigner::generate_presigned_url(S3Url &s3_url, HttpMethod method) { +std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); } diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 20db0a4b2..9f47abe75 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -21,13 +21,13 @@ class S3Url { S3Url() = delete; // Methods - std::string& get_host() { return m_host; } + [[nodiscard]] std::string& get_host() { return m_host; } - std::string& get_bucket() { return m_bucket; } + [[nodiscard]] std::string& get_bucket() { return m_bucket; } - std::string& get_region() { return m_region; } + [[nodiscard]] std::string& get_region() { return m_region; } - std::string& get_path() { return m_path; } + [[nodiscard]] std::string& get_path() { return m_path; } private: std::string m_host; @@ -90,7 +90,8 @@ class AwsAuthenticationSigner { * @param method HTTP method * @return The generated presigned URL */ - std::string generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET); + [[nodiscard]] std::string + generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET); private: /** @@ -98,7 +99,7 @@ class AwsAuthenticationSigner { * @param method HTTP method * @return The converted string */ - static std::string get_method_string(HttpMethod method) { + [[nodiscard]] static std::string get_method_string(HttpMethod method) { switch (method) { case HttpMethod::GET: return "GET"; @@ -120,7 +121,7 @@ class AwsAuthenticationSigner { * @param canonical_request * @return String to sign */ - static std::string get_string_to_sign( + [[nodiscard]] static std::string get_string_to_sign( std::string& scope, std::string& timestamp_string, std::string const& canonical_request @@ -141,7 +142,7 @@ class AwsAuthenticationSigner { * @param query_string Query string * @return Canonical request */ - static std::string + [[nodiscard]] static std::string get_canonical_request(HttpMethod method, S3Url& url, std::string const& query_string) { return fmt::format( "{}\n{}\n{}\n{}:{}\n\n{}\n{}", @@ -161,7 +162,8 @@ class AwsAuthenticationSigner { * @param encode_slash * @return The encoded URI */ - static std::string get_encoded_uri(std::string const& value, bool encode_slash = true) { + [[nodiscard]] static std::string + get_encoded_uri(std::string const& value, bool encode_slash = true) { std::string encoded_uri; for (char c : value) { @@ -187,7 +189,8 @@ class AwsAuthenticationSigner { * @param region * @return The scope */ - static std::string get_scope(std::string& date_string, std::string const& region) { + [[nodiscard]] static std::string + get_scope(std::string& date_string, std::string const& region) { return fmt::format("{}/{}/{}/{}", date_string, region, cS3Service, cAws4Request); } @@ -196,7 +199,7 @@ class AwsAuthenticationSigner { * @param timestamp * @return The timestamp string */ - static std::string get_timestamp_string(TimePoint& timestamp) { + [[nodiscard]] static std::string get_timestamp_string(TimePoint& timestamp) { return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); } @@ -205,7 +208,7 @@ class AwsAuthenticationSigner { * @param timestamp * @return The date string */ - static std::string get_date_string(TimePoint& timestamp) { + [[nodiscard]] static std::string get_date_string(TimePoint& timestamp) { return fmt::format("{:%Y%m%d}", timestamp); } @@ -215,7 +218,8 @@ class AwsAuthenticationSigner { * @param timestamp_string * @return */ - std::string get_default_query_string(std::string& scope, std::string& timestamp_string) { + [[nodiscard]] std::string + get_default_query_string(std::string& scope, std::string& timestamp_string) { return fmt::format( "{}={}&{}={}&{}={}&{}={}&{}={}", cXAmzAlgorithm, @@ -237,7 +241,8 @@ class AwsAuthenticationSigner { * @param date_string * @return */ - std::string get_signature_key(std::string const& region, std::string const& date_string) { + [[nodiscard]] std::string + get_signature_key(std::string const& region, std::string const& date_string) { std::string date_key = get_hmac_sha256_hash(cAws4 + m_secret_access_key, date_string); std::string date_region_key = get_hmac_sha256_hash(date_key, region); std::string date_region_service_key = get_hmac_sha256_hash(date_region_key, cS3Service); @@ -250,7 +255,7 @@ class AwsAuthenticationSigner { * @param string_to_sign * @return */ - static std::string + [[nodiscard]] static std::string get_signature(std::string const& signature_key, std::string const& string_to_sign) { return get_hmac_sha256_hash(signature_key, string_to_sign, true); } diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp index 9d1e64ab3..448658b54 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -1,12 +1,11 @@ #ifndef CLP_AWS_HASH_UTILS_HPP #define CLP_AWS_HASH_UTILS_HPP -#include -#include - #include #include +#include +#include namespace clp::aws { /** @@ -27,7 +26,7 @@ inline std::string char_array_to_string(unsigned char const* a, size_t size) { * Gets the HMAC-SHA256 hash * @param key * @param value - * @param hex_output + * @param hex_output Whether to output the hash as a hex string * @return The HMAC SHA256 hash */ inline std::string @@ -42,11 +41,11 @@ get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_ if (hex_output) { return char_array_to_string(hash, SHA256_DIGEST_LENGTH); - } else { - std::string result; - result.assign(reinterpret_cast(hash), SHA256_DIGEST_LENGTH); - return result; } + + std::string result; + result.assign(reinterpret_cast(hash), SHA256_DIGEST_LENGTH); + return result; } /** From b789ecb89519515dc4b4887c0a58f5dcc101037f Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:23:06 -0400 Subject: [PATCH 04/55] Update Openssl apis to get rid of warnings --- components/core/src/clp/aws/hash_utils.hpp | 77 ++++++++++------------ 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp index 448658b54..86b1322e7 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -7,6 +7,10 @@ #include #include +#include "../type_utils.hpp" + +using clp::size_checked_pointer_cast; + namespace clp::aws { /** * Converts a char array to a string @@ -30,21 +34,31 @@ inline std::string char_array_to_string(unsigned char const* a, size_t size) { * @return The HMAC SHA256 hash */ inline std::string -get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_output = false) { - unsigned char hash[SHA256_DIGEST_LENGTH]; - HMAC_CTX* hmac = HMAC_CTX_new(); - HMAC_Init_ex(hmac, key.c_str(), key.size(), EVP_sha256(), nullptr); - HMAC_Update(hmac, reinterpret_cast(value.c_str()), value.size()); - unsigned int len = SHA256_DIGEST_LENGTH; - HMAC_Final(hmac, hash, &len); - HMAC_CTX_free(hmac); +get_hmac_sha256_hash(std::string const& key, std::string const& input, bool hex_output = false) { + + std::array hash{}; + unsigned int hash_length {0}; + int key_length{0}; + if(key.size() > INT32_MAX) { - if (hex_output) { - return char_array_to_string(hash, SHA256_DIGEST_LENGTH); } + auto* res = HMAC( + EVP_sha256(), + key.c_str(), + key.size(), + size_checked_pointer_cast(input.c_str()), + input.size(), + hash.data(), + &hash_length + ); std::string result; - result.assign(reinterpret_cast(hash), SHA256_DIGEST_LENGTH); + result.assign(size_checked_pointer_cast(hash.data()), SHA256_DIGEST_LENGTH); + + if (hex_output) { + return char_array_to_string(hash.data(), SHA256_DIGEST_LENGTH); + } + return result; } @@ -54,43 +68,20 @@ get_hmac_sha256_hash(std::string const& key, std::string const& value, bool hex_ * @return The SHA256 hash */ inline std::string get_sha256_hash(std::string const& input) { - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256_CTX sha256; - SHA256_Init(&sha256); - SHA256_Update(&sha256, input.c_str(), input.size()); - SHA256_Final(hash, &sha256); - return char_array_to_string(hash, SHA256_DIGEST_LENGTH); -} + EVP_MD_CTX * mdctx = EVP_MD_CTX_new(); -/** - * Initializes SHA256 hash - * @param sha256 - */ -inline void init_sha256_hash(SHA256_CTX* sha256) { - SHA256_Init(sha256); -} + EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); -/** - * Updates SHA256 hash - * @param sha256 - * @param input - * @param length - */ -inline void update_sha256_hash(SHA256_CTX* sha256, void* input, size_t length) { - SHA256_Update(sha256, input, length); -} + EVP_DigestUpdate(mdctx, input.c_str(), input.size()); -/** - * Finalizes SHA256 hash - * @param sha256 - * @return The SHA256 hash - */ -inline std::string finalize_sha256_hash(SHA256_CTX* sha256) { - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256_Final(hash, sha256); + std::array hash{}; + unsigned int digest_len{0}; + EVP_DigestFinal_ex(mdctx, hash.data(), &digest_len); + + EVP_MD_CTX_free(mdctx); - return char_array_to_string(hash, SHA256_DIGEST_LENGTH); + return char_array_to_string(hash.data(), digest_len); } } // namespace clp::aws From 86e9d5cdd20ded90a34dd9a23f911012c0098000 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 18 Jul 2024 23:02:18 -0400 Subject: [PATCH 05/55] Refactor checkpoint --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 71 ++++++++++++++++++- .../src/clp/aws/AwsAuthenticationSigner.hpp | 25 ++----- components/core/src/clp/aws/Constants.hpp | 6 +- components/core/src/clp/aws/hash_utils.hpp | 65 ++--------------- components/core/src/clp/clp/CMakeLists.txt | 1 + 5 files changed, 85 insertions(+), 83 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index cabc4c38d..643b6d762 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -1,7 +1,15 @@ #include "AwsAuthenticationSigner.hpp" +#include "../type_utils.hpp" + +using clp::size_checked_pointer_cast; #include +#include +#include + +using std::span; +using std::vector; namespace clp::aws { S3Url::S3Url(std::string const& url) { // Regular expression to match virtual host-style HTTP URL format @@ -46,8 +54,16 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM auto query_string = get_default_query_string(scope, timestamp_string); auto canonical_request = get_canonical_request(method, s3_url, query_string); auto string_to_sign = get_string_to_sign(scope, timestamp_string, canonical_request); - auto signature_key = get_signature_key(s3_region, date_string); - auto signature = get_signature(signature_key, string_to_sign); + vector signature_key {}; + vector signature {}; + auto error_code = get_signature_key(s3_region, date_string, signature_key); + error_code = get_signature( + {size_checked_pointer_cast(signature_key.data()), signature_key.size()}, + {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size()}, + signature + ); + + auto const signature_str = char_array_to_string({signature.data(), signature.size()}); return fmt::format( "https://{}{}?{}&{}={}", @@ -55,8 +71,57 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM s3_url.get_path(), query_string, cXAmzSignature, - signature + signature_str + ); +} + +[[nodiscard]] std::string AwsAuthenticationSigner::get_string_to_sign( + std::string& scope, + std::string& timestamp_string, + std::string const& canonical_request +) { + std::vector signed_canonical_request; + auto error_code = get_sha256_hash(canonical_request, signed_canonical_request); + auto const signed_canonical_request_str = char_array_to_string({signed_canonical_request.data(), signed_canonical_request.size()}); + return fmt::format( + "{}\n{}\n{}\n{}", + cAws4HmacSha256, + timestamp_string, + scope, + signed_canonical_request_str ); } +[[nodiscard]] ErrorCode +AwsAuthenticationSigner::get_signature_key(std::string const& region, std::string const& date_string, std::vector& signature_key) { + auto const input_key = cAws4 + m_secret_access_key; + + std::vector date_key{}; + std::vector date_region_key{}; + std::vector date_region_service_key{}; + auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(input_key.data()), input_key.size()}, + {size_checked_pointer_cast(date_string.data()), date_string.size()}, + date_key); + + error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_key.data()), date_key.size()}, + {size_checked_pointer_cast(region.data()), region.size()}, + date_region_key); + + error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_region_key.data()), date_region_key.size()}, + {size_checked_pointer_cast(cS3Service.data()), cS3Service.size()}, + date_region_service_key); + + error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_region_service_key.data()), date_region_service_key.size()}, + {size_checked_pointer_cast(cAws4Request.data()), cAws4Request.size()}, + signature_key + ); + + return ErrorCode_Success; +} + + } // namespace clp::aws diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 9f47abe75..d4320e859 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -125,15 +125,7 @@ class AwsAuthenticationSigner { std::string& scope, std::string& timestamp_string, std::string const& canonical_request - ) { - return fmt::format( - "{}\n{}\n{}\n{}", - cAws4HmacSha256, - timestamp_string, - scope, - get_sha256_hash(canonical_request) - ); - } + ); /** * Gets the canonical request string @@ -241,13 +233,8 @@ class AwsAuthenticationSigner { * @param date_string * @return */ - [[nodiscard]] std::string - get_signature_key(std::string const& region, std::string const& date_string) { - std::string date_key = get_hmac_sha256_hash(cAws4 + m_secret_access_key, date_string); - std::string date_region_key = get_hmac_sha256_hash(date_key, region); - std::string date_region_service_key = get_hmac_sha256_hash(date_region_key, cS3Service); - return get_hmac_sha256_hash(date_region_service_key, cAws4Request); - } + [[nodiscard]] ErrorCode + get_signature_key(std::string const& region, std::string const& date_string, std::vector& signature_key); /** * Gets the signature @@ -255,9 +242,9 @@ class AwsAuthenticationSigner { * @param string_to_sign * @return */ - [[nodiscard]] static std::string - get_signature(std::string const& signature_key, std::string const& string_to_sign) { - return get_hmac_sha256_hash(signature_key, string_to_sign, true); + [[nodiscard]] static ErrorCode + get_signature(std::span signature_key, std::span string_to_sign, std::vector& signature) { + return get_hmac_sha256_hash(signature_key, string_to_sign, signature); } // Variables diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp index a12b5a086..f7b06777b 100644 --- a/components/core/src/clp/aws/Constants.hpp +++ b/components/core/src/clp/aws/Constants.hpp @@ -1,6 +1,8 @@ #ifndef CLP_AWS_CONSTANTS_HPP #define CLP_AWS_CONSTANTS_HPP +#include + namespace clp::aws { // Query String Parameter Names constexpr char const* const cXAmzAlgorithm = "X-Amz-Algorithm"; @@ -13,13 +15,13 @@ constexpr char const* const cXAmzSignedHeaders = "X-Amz-SignedHeaders"; // Other Constants constexpr char const* const cAws4 = "AWS4"; -constexpr char const* const cAws4Request = "aws4_request"; +constexpr std::string_view cAws4Request {"aws4_request"}; constexpr char const* const cAws4HmacSha256 = "AWS4-HMAC-SHA256"; constexpr char const* const cDefaultSignedHeaders = "host"; constexpr char const* const cDefaultRegion = "us-east-1"; constexpr char const* const cEmptyStringSha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; -constexpr char const* const cS3Service = "s3"; +constexpr std::string_view cS3Service {"s3"}; constexpr char const* const cUnsignedPayload = "UNSIGNED-PAYLOAD"; } // namespace clp::aws diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp index 86b1322e7..fb35250ed 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -2,14 +2,10 @@ #define CLP_AWS_HASH_UTILS_HPP #include +#include "../ErrorCode.hpp" -#include -#include -#include - -#include "../type_utils.hpp" - -using clp::size_checked_pointer_cast; +#include +#include namespace clp::aws { /** @@ -18,13 +14,7 @@ namespace clp::aws { * @param size * @return The converted string */ -inline std::string char_array_to_string(unsigned char const* a, size_t size) { - std::string hex_string; - for (size_t i = 0; i < size; i++) { - hex_string += fmt::format("{:02x}", static_cast(a[i])); - } - return hex_string; -} +std::string char_array_to_string(std::span input); /** * Gets the HMAC-SHA256 hash @@ -33,56 +23,13 @@ inline std::string char_array_to_string(unsigned char const* a, size_t size) { * @param hex_output Whether to output the hash as a hex string * @return The HMAC SHA256 hash */ -inline std::string -get_hmac_sha256_hash(std::string const& key, std::string const& input, bool hex_output = false) { - - std::array hash{}; - unsigned int hash_length {0}; - int key_length{0}; - if(key.size() > INT32_MAX) { - - } - auto* res = HMAC( - EVP_sha256(), - key.c_str(), - key.size(), - size_checked_pointer_cast(input.c_str()), - input.size(), - hash.data(), - &hash_length - ); - - std::string result; - result.assign(size_checked_pointer_cast(hash.data()), SHA256_DIGEST_LENGTH); - - if (hex_output) { - return char_array_to_string(hash.data(), SHA256_DIGEST_LENGTH); - } - - return result; -} +ErrorCode get_hmac_sha256_hash(std::span key, std::span input, std::vector& hash); /** * Gets the SHA256 hash * @param input * @return The SHA256 hash */ -inline std::string get_sha256_hash(std::string const& input) { - - EVP_MD_CTX * mdctx = EVP_MD_CTX_new(); - - EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); - - EVP_DigestUpdate(mdctx, input.c_str(), input.size()); - - std::array hash{}; - unsigned int digest_len{0}; - EVP_DigestFinal_ex(mdctx, hash.data(), &digest_len); - - EVP_MD_CTX_free(mdctx); - - return char_array_to_string(hash.data(), digest_len); -} - +ErrorCode get_sha256_hash(std::string const& input, std::vector& hash); } // namespace clp::aws #endif // CLP_AWS_HASH_UTILS_HPP diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 9f9ffaff9..4dc74a0be 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -4,6 +4,7 @@ set( ../aws/AwsAuthenticationSigner.cpp ../aws/AwsAuthenticationSigner.hpp ../aws/Constants.hpp + ../aws/hash_utils.cpp ../aws/hash_utils.hpp ../BufferedFileReader.cpp ../BufferedFileReader.hpp From 3b4815cd67ff4951654861feb0ebc427e8cbf797 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 18 Jul 2024 23:39:35 -0400 Subject: [PATCH 06/55] Replace constants with string_view --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 3 ++- components/core/src/clp/aws/Constants.hpp | 26 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 643b6d762..fb1dc4442 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -94,7 +94,8 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM [[nodiscard]] ErrorCode AwsAuthenticationSigner::get_signature_key(std::string const& region, std::string const& date_string, std::vector& signature_key) { - auto const input_key = cAws4 + m_secret_access_key; + std::string input_key {cAws4}; + input_key += m_secret_access_key; std::vector date_key{}; std::vector date_region_key{}; diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp index f7b06777b..d1aae6098 100644 --- a/components/core/src/clp/aws/Constants.hpp +++ b/components/core/src/clp/aws/Constants.hpp @@ -5,24 +5,22 @@ namespace clp::aws { // Query String Parameter Names -constexpr char const* const cXAmzAlgorithm = "X-Amz-Algorithm"; -constexpr char const* const cXAmzCredential = "X-Amz-Credential"; -constexpr char const* const cXAmzContentSha256 = "X-Amz-Content-Sha256"; -constexpr char const* const cXAmzDate = "X-Amz-Date"; -constexpr char const* const cXAmzExpires = "X-Amz-Expires"; -constexpr char const* const cXAmzSignature = "X-Amz-Signature"; -constexpr char const* const cXAmzSignedHeaders = "X-Amz-SignedHeaders"; +constexpr std::string_view cXAmzAlgorithm {"X-Amz-Algorithm"}; +constexpr std::string_view cXAmzCredential = "X-Amz-Credential"; +constexpr std::string_view cXAmzContentSha256 = "X-Amz-Content-Sha256"; +constexpr std::string_view cXAmzDate = "X-Amz-Date"; +constexpr std::string_view cXAmzExpires = "X-Amz-Expires"; +constexpr std::string_view cXAmzSignature = "X-Amz-Signature"; +constexpr std::string_view cXAmzSignedHeaders = "X-Amz-SignedHeaders"; // Other Constants -constexpr char const* const cAws4 = "AWS4"; +constexpr std::string_view cAws4 {"AWS4"}; constexpr std::string_view cAws4Request {"aws4_request"}; -constexpr char const* const cAws4HmacSha256 = "AWS4-HMAC-SHA256"; -constexpr char const* const cDefaultSignedHeaders = "host"; -constexpr char const* const cDefaultRegion = "us-east-1"; -constexpr char const* const cEmptyStringSha256 - = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +constexpr std::string_view cAws4HmacSha256 = "AWS4-HMAC-SHA256"; +constexpr std::string_view cDefaultSignedHeaders = "host"; +constexpr std::string_view cDefaultRegion = "us-east-1"; constexpr std::string_view cS3Service {"s3"}; -constexpr char const* const cUnsignedPayload = "UNSIGNED-PAYLOAD"; +constexpr std::string_view cUnsignedPayload {"UNSIGNED-PAYLOAD"}; } // namespace clp::aws #endif // CLP_AWS_CONSTANTS_HPP From 423934f5aca6c742ce1e2d07f4bbfd0e9aef9a69 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:42:37 -0400 Subject: [PATCH 07/55] Change more string to sting_view and move some static functions out from the signer class --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 169 +++++++++++++++--- .../src/clp/aws/AwsAuthenticationSigner.hpp | 151 +--------------- components/core/src/clp/aws/hash_utils.cpp | 91 ++++++++++ components/core/src/clp/aws/hash_utils.hpp | 5 +- 4 files changed, 250 insertions(+), 166 deletions(-) create mode 100644 components/core/src/clp/aws/hash_utils.cpp diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index fb1dc4442..adfe40480 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -10,6 +10,136 @@ using clp::size_checked_pointer_cast; using std::span; using std::vector; + +namespace { +/** + * Gets the timestamp string + * @param timestamp + * @return The timestamp string + */ +[[nodiscard]] std::string get_timestamp_string(std::chrono::system_clock::time_point const& timestamp) { + return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); +} + +/** + * Gets the date string + * @param timestamp + * @return The date string + */ +[[nodiscard]] std::string get_date_string(std::chrono::system_clock::time_point const& timestamp) { + return fmt::format("{:%Y%m%d}", timestamp); +} + +/** + * Converts an HttpMethod to a string + * @param method HTTP method + * @return The converted string + */ +[[nodiscard]] static std::string get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method) { + switch (method) { + case clp::aws::AwsAuthenticationSigner::HttpMethod::GET: + return "GET"; + case clp::aws::AwsAuthenticationSigner::HttpMethod::PUT: + return "PUT"; + case clp::aws::AwsAuthenticationSigner::HttpMethod::POST: + return "POST"; + case clp::aws::AwsAuthenticationSigner::HttpMethod::DELETE: + return "DELETE"; + default: + throw std::runtime_error("Invalid HTTP method"); + } +} + +/** + * Gets the string to sign + * @param scope + * @param timestamp_string + * @param canonical_request + * @return String to sign + */ +[[nodiscard]] std::string get_string_to_sign( + std::string& scope, + std::string& timestamp_string, + std::string const& canonical_request +) { + std::vector signed_canonical_request; + auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); + auto const signed_canonical_request_str = clp::aws::char_array_to_string({signed_canonical_request.data(), signed_canonical_request.size()}); + return fmt::format( + "{}\n{}\n{}\n{}", + clp::aws::cAws4HmacSha256, + timestamp_string, + scope, + signed_canonical_request_str + ); +} +/** + * Gets the encoded URI + * @param value + * @param encode_slash + * @return The encoded URI + */ +[[nodiscard]] std::string +get_encoded_uri(std::string_view value, bool encode_slash = true) { + std::string encoded_uri; + + for (auto const c : value) { + if ((std::isalnum(c) != 0) || c == '-' || c == '_' || c == '.' || c == '~') { + encoded_uri += c; + } else if (c == '/' && false == encode_slash) { + encoded_uri += c; + } else { + encoded_uri += fmt::format("%{:02X}", static_cast(c)); + } + } + + return encoded_uri; +} + +/** + * Gets the scope + * @param date_string + * @param region + * @return The scope + */ +[[nodiscard]] std::string +get_scope(std::string& date_string, std::string_view region) { + return fmt::format("{}/{}/{}/{}", date_string, region, clp::aws::cS3Service, clp::aws::cAws4Request); +} + +/** + * Gets the canonical request string + * @param method HTTP method + * @param url S3 URL + * @param query_string Query string + * @return Canonical request + */ +[[nodiscard]] std::string +get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp::aws::S3Url& url, std::string const& query_string) { + return fmt::format( + "{}\n{}\n{}\n{}:{}\n\n{}\n{}", + get_method_string(method), + get_encoded_uri(url.get_path(), false), + query_string, + clp::aws::cDefaultSignedHeaders, + url.get_host(), + clp::aws::cDefaultSignedHeaders, + clp::aws::cUnsignedPayload + ); +} + +/** + * Gets the signature + * @param signature_key + * @param string_to_sign + * @return + */ +[[nodiscard]] clp::ErrorCode +get_signature(std::span signature_key, std::span string_to_sign, std::vector& signature) { + return clp::aws::get_hmac_sha256_hash(signature_key, string_to_sign, signature); +} +} + namespace clp::aws { S3Url::S3Url(std::string const& url) { // Regular expression to match virtual host-style HTTP URL format @@ -43,10 +173,10 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM throw std::runtime_error("Unsupported HTTP method!"); } - auto s3_region = s3_url.get_region(); + auto const s3_region = s3_url.get_region(); // Gets current time - auto now = std::chrono::system_clock::now(); + auto const now = std::chrono::system_clock::now(); auto date_string = get_date_string(now); auto timestamp_string = get_timestamp_string(now); @@ -75,25 +205,8 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM ); } -[[nodiscard]] std::string AwsAuthenticationSigner::get_string_to_sign( - std::string& scope, - std::string& timestamp_string, - std::string const& canonical_request -) { - std::vector signed_canonical_request; - auto error_code = get_sha256_hash(canonical_request, signed_canonical_request); - auto const signed_canonical_request_str = char_array_to_string({signed_canonical_request.data(), signed_canonical_request.size()}); - return fmt::format( - "{}\n{}\n{}\n{}", - cAws4HmacSha256, - timestamp_string, - scope, - signed_canonical_request_str - ); -} - [[nodiscard]] ErrorCode -AwsAuthenticationSigner::get_signature_key(std::string const& region, std::string const& date_string, std::vector& signature_key) { +AwsAuthenticationSigner::get_signature_key(std::string_view region, std::string_view date_string, std::vector& signature_key) { std::string input_key {cAws4}; input_key += m_secret_access_key; @@ -124,5 +237,21 @@ AwsAuthenticationSigner::get_signature_key(std::string const& region, std::strin return ErrorCode_Success; } +std::string +AwsAuthenticationSigner::get_default_query_string(std::string& scope, std::string& timestamp_string) { + return fmt::format( + "{}={}&{}={}&{}={}&{}={}&{}={}", + cXAmzAlgorithm, + cAws4HmacSha256, + cXAmzCredential, + get_encoded_uri(m_access_key_id + "/" + scope), + cXAmzDate, + timestamp_string, + cXAmzExpires, + cDefaultExpireTime, + cXAmzSignedHeaders, + cDefaultSignedHeaders + ); +} } // namespace clp::aws diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index d4320e859..a51e7f1a1 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -21,13 +21,13 @@ class S3Url { S3Url() = delete; // Methods - [[nodiscard]] std::string& get_host() { return m_host; } + [[nodiscard]] std::string_view get_host() { return m_host; } - [[nodiscard]] std::string& get_bucket() { return m_bucket; } + [[nodiscard]] std::string_view get_bucket() { return m_bucket; } - [[nodiscard]] std::string& get_region() { return m_region; } + [[nodiscard]] std::string_view get_region() { return m_region; } - [[nodiscard]] std::string& get_path() { return m_path; } + [[nodiscard]] std::string_view get_path() { return m_path; } private: std::string m_host; @@ -41,10 +41,8 @@ class S3Url { */ class AwsAuthenticationSigner { public: - using TimePoint = std::chrono::system_clock::time_point; - // Default expire time of presigned URL in seconds - static constexpr int const cDefaultExpireTime = 86'400; // 24 hours + static constexpr int cDefaultExpireTime = 86'400; // 24 hours // Types enum class HttpMethod : uint8_t { @@ -94,116 +92,6 @@ class AwsAuthenticationSigner { generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET); private: - /** - * Converts an HttpMethod to a string - * @param method HTTP method - * @return The converted string - */ - [[nodiscard]] static std::string get_method_string(HttpMethod method) { - switch (method) { - case HttpMethod::GET: - return "GET"; - case HttpMethod::PUT: - return "PUT"; - case HttpMethod::POST: - return "POST"; - case HttpMethod::DELETE: - return "DELETE"; - default: - throw std::runtime_error("Invalid HTTP method"); - } - } - - /** - * Gets the string to sign - * @param scope - * @param timestamp_string - * @param canonical_request - * @return String to sign - */ - [[nodiscard]] static std::string get_string_to_sign( - std::string& scope, - std::string& timestamp_string, - std::string const& canonical_request - ); - - /** - * Gets the canonical request string - * @param method HTTP method - * @param url S3 URL - * @param query_string Query string - * @return Canonical request - */ - [[nodiscard]] static std::string - get_canonical_request(HttpMethod method, S3Url& url, std::string const& query_string) { - return fmt::format( - "{}\n{}\n{}\n{}:{}\n\n{}\n{}", - get_method_string(method), - get_encoded_uri(url.get_path(), false), - query_string, - cDefaultSignedHeaders, - url.get_host(), - cDefaultSignedHeaders, - cUnsignedPayload - ); - } - - /** - * Gets the encoded URI - * @param value - * @param encode_slash - * @return The encoded URI - */ - [[nodiscard]] static std::string - get_encoded_uri(std::string const& value, bool encode_slash = true) { - std::string encoded_uri; - - for (char c : value) { - if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { - encoded_uri += c; - continue; - } - - if (c == '/' && false == encode_slash) { - encoded_uri += c; - continue; - } - - encoded_uri += fmt::format("%{:02X}", static_cast(static_cast(c))); - } - - return encoded_uri; - } - - /** - * Gets the scope - * @param date_string - * @param region - * @return The scope - */ - [[nodiscard]] static std::string - get_scope(std::string& date_string, std::string const& region) { - return fmt::format("{}/{}/{}/{}", date_string, region, cS3Service, cAws4Request); - } - - /** - * Gets the timestamp string - * @param timestamp - * @return The timestamp string - */ - [[nodiscard]] static std::string get_timestamp_string(TimePoint& timestamp) { - return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); - } - - /** - * Gets the date string - * @param timestamp - * @return The date string - */ - [[nodiscard]] static std::string get_date_string(TimePoint& timestamp) { - return fmt::format("{:%Y%m%d}", timestamp); - } - /** * Gets the default query string * @param scope @@ -211,21 +99,7 @@ class AwsAuthenticationSigner { * @return */ [[nodiscard]] std::string - get_default_query_string(std::string& scope, std::string& timestamp_string) { - return fmt::format( - "{}={}&{}={}&{}={}&{}={}&{}={}", - cXAmzAlgorithm, - cAws4HmacSha256, - cXAmzCredential, - get_encoded_uri(m_access_key_id + "/" + scope), - cXAmzDate, - timestamp_string, - cXAmzExpires, - cDefaultExpireTime, - cXAmzSignedHeaders, - cDefaultSignedHeaders - ); - } + get_default_query_string(std::string& scope, std::string& timestamp_string); /** * Gets the signature key @@ -234,18 +108,7 @@ class AwsAuthenticationSigner { * @return */ [[nodiscard]] ErrorCode - get_signature_key(std::string const& region, std::string const& date_string, std::vector& signature_key); - - /** - * Gets the signature - * @param signature_key - * @param string_to_sign - * @return - */ - [[nodiscard]] static ErrorCode - get_signature(std::span signature_key, std::span string_to_sign, std::vector& signature) { - return get_hmac_sha256_hash(signature_key, string_to_sign, signature); - } + get_signature_key(std::string_view region, std::string_view date_string, std::vector& signature_key); // Variables std::string m_access_key_id; diff --git a/components/core/src/clp/aws/hash_utils.cpp b/components/core/src/clp/aws/hash_utils.cpp new file mode 100644 index 000000000..4babd7914 --- /dev/null +++ b/components/core/src/clp/aws/hash_utils.cpp @@ -0,0 +1,91 @@ +#include "hash_utils.hpp" + +#include +#include +#include + +#include +#include +#include +#include + +#include "../type_utils.hpp" + +#include "../ErrorCode.hpp" +#include "../spdlog_with_specializations.hpp" + +using clp::size_checked_pointer_cast; +using std::string; +using std::vector; +using std::span; +using std::string_view; + +namespace clp::aws { +/** + * Converts a char array to a string + * @param a + * @param size + * @return The converted string + */ +std::string char_array_to_string(span input) { + std::string hex_string; + for (auto const i : input) { + hex_string += fmt::format("{:02x}", static_cast(i)); + } + return hex_string; +} + +ErrorCode get_hmac_sha256_hash(span key, span input, vector& hash) { + + hash.resize(SHA256_DIGEST_LENGTH); + unsigned int hash_length {0}; + + if (key.size() > INT32_MAX) { + SPDLOG_ERROR("Key too long"); + return ErrorCode_BadParam; + } + int key_length{static_cast(key.size())}; + auto* res = HMAC( + EVP_sha256(), + key.data(), + key_length, + input.data(), + input.size(), + hash.data(), + &hash_length + ); + + if (nullptr == res) { + SPDLOG_ERROR("Failed to get HMAC hashes"); + return ErrorCode_Failure; + } + + if (hash_length != SHA256_DIGEST_LENGTH) { + SPDLOG_ERROR("Unexpected hash length"); + return ErrorCode_Failure; + } + + return ErrorCode_Success; +} + +/** + * Gets the SHA256 hash + * @param input + * @return The SHA256 hash + */ +ErrorCode get_sha256_hash(string_view input, std::vector& hash) { + + EVP_MD_CTX * mdctx = EVP_MD_CTX_new(); + + EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); + + EVP_DigestUpdate(mdctx, input.data(), input.size()); + + unsigned int digest_len{0}; + hash.resize(SHA256_DIGEST_LENGTH); + EVP_DigestFinal_ex(mdctx, hash.data(), &digest_len); + EVP_MD_CTX_free(mdctx); + + return ErrorCode_Success; +} +} \ No newline at end of file diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/aws/hash_utils.hpp index fb35250ed..1b48a2b37 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/aws/hash_utils.hpp @@ -1,11 +1,12 @@ #ifndef CLP_AWS_HASH_UTILS_HPP #define CLP_AWS_HASH_UTILS_HPP -#include #include "../ErrorCode.hpp" #include #include +#include +#include namespace clp::aws { /** @@ -30,6 +31,6 @@ ErrorCode get_hmac_sha256_hash(std::span key, std::span& hash); +ErrorCode get_sha256_hash(std::string_view, std::vector& hash); } // namespace clp::aws #endif // CLP_AWS_HASH_UTILS_HPP From 4bf400544dbcd9bc083284462aed31efeaf7462a Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:45:58 -0400 Subject: [PATCH 08/55] refactoring --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 66 ++++++++++--------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 2 +- components/core/src/clp/aws/hash_utils.cpp | 4 +- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index adfe40480..7c58cc9da 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -10,6 +10,8 @@ using clp::size_checked_pointer_cast; using std::span; using std::vector; +using std::string; +using std::string_view; namespace { /** @@ -17,7 +19,7 @@ namespace { * @param timestamp * @return The timestamp string */ -[[nodiscard]] std::string get_timestamp_string(std::chrono::system_clock::time_point const& timestamp) { +[[nodiscard]] string get_timestamp_string(std::chrono::system_clock::time_point const& timestamp) { return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); } @@ -26,7 +28,7 @@ namespace { * @param timestamp * @return The date string */ -[[nodiscard]] std::string get_date_string(std::chrono::system_clock::time_point const& timestamp) { +[[nodiscard]] string get_date_string(std::chrono::system_clock::time_point const& timestamp) { return fmt::format("{:%Y%m%d}", timestamp); } @@ -35,7 +37,7 @@ namespace { * @param method HTTP method * @return The converted string */ -[[nodiscard]] static std::string get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method) { +[[nodiscard]] string get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method) { switch (method) { case clp::aws::AwsAuthenticationSigner::HttpMethod::GET: return "GET"; @@ -57,12 +59,12 @@ namespace { * @param canonical_request * @return String to sign */ -[[nodiscard]] std::string get_string_to_sign( - std::string& scope, - std::string& timestamp_string, - std::string const& canonical_request +[[nodiscard]] string get_string_to_sign( + string_view scope, + string_view timestamp_string, + string_view canonical_request ) { - std::vector signed_canonical_request; + vector signed_canonical_request; auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); auto const signed_canonical_request_str = clp::aws::char_array_to_string({signed_canonical_request.data(), signed_canonical_request.size()}); return fmt::format( @@ -79,9 +81,9 @@ namespace { * @param encode_slash * @return The encoded URI */ -[[nodiscard]] std::string -get_encoded_uri(std::string_view value, bool encode_slash = true) { - std::string encoded_uri; +[[nodiscard]] string +get_encoded_uri(string_view value, bool encode_slash = true) { + string encoded_uri; for (auto const c : value) { if ((std::isalnum(c) != 0) || c == '-' || c == '_' || c == '.' || c == '~') { @@ -89,7 +91,7 @@ get_encoded_uri(std::string_view value, bool encode_slash = true) { } else if (c == '/' && false == encode_slash) { encoded_uri += c; } else { - encoded_uri += fmt::format("%{:02X}", static_cast(c)); + encoded_uri += fmt::format("%{:02X}", c); } } @@ -102,8 +104,8 @@ get_encoded_uri(std::string_view value, bool encode_slash = true) { * @param region * @return The scope */ -[[nodiscard]] std::string -get_scope(std::string& date_string, std::string_view region) { +[[nodiscard]] string +get_scope(string_view date_string, string_view region) { return fmt::format("{}/{}/{}/{}", date_string, region, clp::aws::cS3Service, clp::aws::cAws4Request); } @@ -114,8 +116,8 @@ get_scope(std::string& date_string, std::string_view region) { * @param query_string Query string * @return Canonical request */ -[[nodiscard]] std::string -get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp::aws::S3Url& url, std::string const& query_string) { +[[nodiscard]] string +get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp::aws::S3Url& url, string_view query_string) { return fmt::format( "{}\n{}\n{}\n{}:{}\n\n{}\n{}", get_method_string(method), @@ -135,19 +137,19 @@ get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp: * @return */ [[nodiscard]] clp::ErrorCode -get_signature(std::span signature_key, std::span string_to_sign, std::vector& signature) { +get_signature(span signature_key, span string_to_sign, vector& signature) { return clp::aws::get_hmac_sha256_hash(signature_key, string_to_sign, signature); } } namespace clp::aws { -S3Url::S3Url(std::string const& url) { +S3Url::S3Url(string const& url) { // Regular expression to match virtual host-style HTTP URL format // std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); - std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); + std::regex const regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); // Regular expression to match path-style HTTP URL format // std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); - std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); + std::regex const regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; if (std::regex_match(url, match, regex1)) { @@ -168,7 +170,7 @@ S3Url::S3Url(std::string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) { +string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); } @@ -177,8 +179,8 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM // Gets current time auto const now = std::chrono::system_clock::now(); - auto date_string = get_date_string(now); - auto timestamp_string = get_timestamp_string(now); + auto const date_string = get_date_string(now); + auto const timestamp_string = get_timestamp_string(now); auto scope = get_scope(date_string, s3_region); auto query_string = get_default_query_string(scope, timestamp_string); @@ -206,13 +208,13 @@ std::string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpM } [[nodiscard]] ErrorCode -AwsAuthenticationSigner::get_signature_key(std::string_view region, std::string_view date_string, std::vector& signature_key) { - std::string input_key {cAws4}; +AwsAuthenticationSigner::get_signature_key(string_view region, string_view date_string, vector& signature_key) { + string input_key {cAws4}; input_key += m_secret_access_key; - std::vector date_key{}; - std::vector date_region_key{}; - std::vector date_region_service_key{}; + vector date_key{}; + vector date_region_key{}; + vector date_region_service_key{}; auto error_code = get_hmac_sha256_hash( {size_checked_pointer_cast(input_key.data()), input_key.size()}, {size_checked_pointer_cast(date_string.data()), date_string.size()}, @@ -237,14 +239,16 @@ AwsAuthenticationSigner::get_signature_key(std::string_view region, std::string_ return ErrorCode_Success; } -std::string -AwsAuthenticationSigner::get_default_query_string(std::string& scope, std::string& timestamp_string) { +string +AwsAuthenticationSigner::get_default_query_string(string_view scope, string_view timestamp_string) { + string uri {m_access_key_id + "/"}; + uri += scope; return fmt::format( "{}={}&{}={}&{}={}&{}={}&{}={}", cXAmzAlgorithm, cAws4HmacSha256, cXAmzCredential, - get_encoded_uri(m_access_key_id + "/" + scope), + get_encoded_uri(uri), cXAmzDate, timestamp_string, cXAmzExpires, diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index a51e7f1a1..9c9c9e4e0 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -99,7 +99,7 @@ class AwsAuthenticationSigner { * @return */ [[nodiscard]] std::string - get_default_query_string(std::string& scope, std::string& timestamp_string); + get_default_query_string(std::string_view scope, std::string_view timestamp_string); /** * Gets the signature key diff --git a/components/core/src/clp/aws/hash_utils.cpp b/components/core/src/clp/aws/hash_utils.cpp index 4babd7914..d110c724f 100644 --- a/components/core/src/clp/aws/hash_utils.cpp +++ b/components/core/src/clp/aws/hash_utils.cpp @@ -29,8 +29,8 @@ namespace clp::aws { */ std::string char_array_to_string(span input) { std::string hex_string; - for (auto const i : input) { - hex_string += fmt::format("{:02x}", static_cast(i)); + for (auto const c : input) { + hex_string += fmt::format("{:02x}", c); } return hex_string; } From fb8a46fd50fa4e8b990dccfee7331611d3612033 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:48:03 -0400 Subject: [PATCH 09/55] Move hash utils in to clp/ and refactor constructors. --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 14 ++------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 26 +++-------------- components/core/src/clp/clp/CMakeLists.txt | 4 +-- .../core/src/clp/clp/FileCompressor.cpp | 29 +++++++++++++++++++ .../core/src/clp/clp/FileCompressor.hpp | 19 ++---------- .../core/src/clp/{aws => }/hash_utils.cpp | 6 ++-- .../core/src/clp/{aws => }/hash_utils.hpp | 2 +- 7 files changed, 43 insertions(+), 57 deletions(-) rename components/core/src/clp/{aws => }/hash_utils.cpp (95%) rename components/core/src/clp/{aws => }/hash_utils.hpp (96%) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 7c58cc9da..bfa823643 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -1,5 +1,6 @@ #include "AwsAuthenticationSigner.hpp" #include "../type_utils.hpp" +#include "../hash_utils.hpp" using clp::size_checked_pointer_cast; @@ -129,17 +130,6 @@ get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp: clp::aws::cUnsignedPayload ); } - -/** - * Gets the signature - * @param signature_key - * @param string_to_sign - * @return - */ -[[nodiscard]] clp::ErrorCode -get_signature(span signature_key, span string_to_sign, vector& signature) { - return clp::aws::get_hmac_sha256_hash(signature_key, string_to_sign, signature); -} } namespace clp::aws { @@ -189,7 +179,7 @@ string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod vector signature_key {}; vector signature {}; auto error_code = get_signature_key(s3_region, date_string, signature_key); - error_code = get_signature( + error_code = get_hmac_sha256_hash( {size_checked_pointer_cast(signature_key.data()), signature_key.size()}, {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size()}, signature diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 9c9c9e4e0..ff493f0b4 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -7,7 +7,7 @@ #include #include "Constants.hpp" -#include "hash_utils.hpp" +#include "../ErrorCode.hpp" namespace clp::aws { /** @@ -58,28 +58,11 @@ class AwsAuthenticationSigner { // Constructors AwsAuthenticationSigner( - std::string& access_key_id, - std::string& secret_access_key, - AwsService service = AwsService::S3 + std::string_view access_key_id, + std::string_view secret_access_key ) : m_access_key_id(access_key_id), - m_secret_access_key(secret_access_key), - m_service(service) { - if (AwsService::S3 != m_service) { - throw std::invalid_argument("Unsupported service"); - } - } - - AwsAuthenticationSigner() - : m_access_key_id(getenv("AWS_ACCESS_KEY_ID")), - m_secret_access_key(getenv("AWS_SECRET_ACCESS_KEY")) { - if (m_access_key_id.empty()) { - throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); - } - if (m_secret_access_key.empty()) { - throw std::invalid_argument("AWS_SECRET_ACCESS_KEY environment variable is not set"); - } - } + m_secret_access_key(secret_access_key) {} // Methods /** @@ -113,7 +96,6 @@ class AwsAuthenticationSigner { // Variables std::string m_access_key_id; std::string m_secret_access_key; - AwsService m_service{AwsService::S3}; }; } // namespace clp::aws diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 4dc74a0be..58c98bd47 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -4,8 +4,6 @@ set( ../aws/AwsAuthenticationSigner.cpp ../aws/AwsAuthenticationSigner.hpp ../aws/Constants.hpp - ../aws/hash_utils.cpp - ../aws/hash_utils.hpp ../BufferedFileReader.cpp ../BufferedFileReader.hpp ../BufferReader.cpp @@ -50,6 +48,8 @@ set( ../GlobalMySQLMetadataDB.hpp ../GlobalSQLiteMetadataDB.cpp ../GlobalSQLiteMetadataDB.hpp + ../hash_utils.cpp + ../hash_utils.hpp ../ir/constants.hpp ../ir/LogEvent.hpp ../ir/LogEventDeserializer.cpp diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 5beb242a1..630881ca1 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -108,6 +108,35 @@ static void write_message_to_encoded_file( } namespace clp::clp { +FileCompressor::FileCompressor( + CommandLineArguments::InputSource input_source, + boost::uuids::random_generator& uuid_generator, + std::unique_ptr reader_parser +) : m_input_source(input_source), m_uuid_generator(uuid_generator), m_reader_parser(std::move(reader_parser)) +{ + if (CommandLineArguments::InputSource::S3 == input_source) { + if (ErrorCode_Success != NetworkReader::init()) { + SPDLOG_ERROR("Failed to initialize streaming reader"); + throw std::runtime_error("Failed to initialize streaming reader"); + } + string const access_key_id {getenv("AWS_ACCESS_KEY_ID")}; + string const secret_access_key {getenv("AWS_SECRET_ACCESS_KEY")}; + if (access_key_id.empty()) { + throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); + } + if (secret_access_key.empty()) { + throw std::invalid_argument("AWS_SECRET_ACCESS_KEY environment variable is not set"); + } + m_aws_auth_signer.emplace(access_key_id, secret_access_key); + } +} + +FileCompressor::~FileCompressor () { + if (CommandLineArguments::InputSource::S3 == m_input_source) { + NetworkReader::deinit(); + } +} + bool FileCompressor::compress_file( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, diff --git a/components/core/src/clp/clp/FileCompressor.hpp b/components/core/src/clp/clp/FileCompressor.hpp index 2d6b77891..de4d697fd 100644 --- a/components/core/src/clp/clp/FileCompressor.hpp +++ b/components/core/src/clp/clp/FileCompressor.hpp @@ -30,24 +30,9 @@ class FileCompressor { CommandLineArguments::InputSource input_source, boost::uuids::random_generator& uuid_generator, std::unique_ptr reader_parser - ) - : m_input_source(input_source), - m_uuid_generator(uuid_generator), - m_reader_parser(std::move(reader_parser)) { - if (CommandLineArguments::InputSource::S3 == input_source) { - m_aws_auth_signer.emplace(); - if (ErrorCode_Success != NetworkReader::init()) { - SPDLOG_ERROR("Failed to initialize streaming reader"); - throw std::runtime_error("Failed to initialize streaming reader"); - } - } - } + ); - ~FileCompressor() { - if (CommandLineArguments::InputSource::S3 == m_input_source) { - NetworkReader::deinit(); - } - } + ~FileCompressor(); // Methods /** diff --git a/components/core/src/clp/aws/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp similarity index 95% rename from components/core/src/clp/aws/hash_utils.cpp rename to components/core/src/clp/hash_utils.cpp index d110c724f..c426e9247 100644 --- a/components/core/src/clp/aws/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -9,10 +9,10 @@ #include #include -#include "../type_utils.hpp" +#include "type_utils.hpp" -#include "../ErrorCode.hpp" -#include "../spdlog_with_specializations.hpp" +#include "ErrorCode.hpp" +#include "spdlog_with_specializations.hpp" using clp::size_checked_pointer_cast; using std::string; diff --git a/components/core/src/clp/aws/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp similarity index 96% rename from components/core/src/clp/aws/hash_utils.hpp rename to components/core/src/clp/hash_utils.hpp index 1b48a2b37..5168f9ec5 100644 --- a/components/core/src/clp/aws/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -1,7 +1,7 @@ #ifndef CLP_AWS_HASH_UTILS_HPP #define CLP_AWS_HASH_UTILS_HPP -#include "../ErrorCode.hpp" +#include "ErrorCode.hpp" #include #include From 303aaaf0f47d8aab420a6e5ba1f18440b10e45cc Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:11:57 -0400 Subject: [PATCH 10/55] simple refactoring to parse and encode --- .../core/src/clp/clp/FileCompressor.cpp | 128 +++++++++--------- .../core/src/clp/clp/FileCompressor.hpp | 10 ++ 2 files changed, 74 insertions(+), 64 deletions(-) diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 630881ca1..7f50b2400 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -114,7 +114,7 @@ FileCompressor::FileCompressor( std::unique_ptr reader_parser ) : m_input_source(input_source), m_uuid_generator(uuid_generator), m_reader_parser(std::move(reader_parser)) { - if (CommandLineArguments::InputSource::S3 == input_source) { + if (CommandLineArguments::InputSource::S3 == m_input_source) { if (ErrorCode_Success != NetworkReader::init()) { SPDLOG_ERROR("Failed to initialize streaming reader"); throw std::runtime_error("Failed to initialize streaming reader"); @@ -155,27 +155,16 @@ bool FileCompressor::compress_file( Profiler::start_continuous_measurement( ); NetworkReader network_reader(m_aws_auth_signer.value().generate_presigned_url(s3_url)); - if (use_heuristic) { - parse_and_encode_with_heuristic( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - network_reader - ); - } else { - parse_and_encode_with_library( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - network_reader - ); - } + parse_and_encode( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + network_reader, + use_heuristic + ); } else if (CommandLineArguments::InputSource::Filesystem == m_input_source) { file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); PROFILER_SPDLOG_INFO("Start parsing {}", file_name) @@ -207,27 +196,16 @@ bool FileCompressor::compress_file( m_file_reader.peek_buffered_data(utf8_validation_buf, peek_size); auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { - if (use_heuristic) { - parse_and_encode_with_heuristic( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - m_file_reader - ); - } else { - parse_and_encode_with_library( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - m_file_reader - ); - } + parse_and_encode( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + m_file_reader, + use_heuristic + ); } else { if (false == try_compressing_as_archive( @@ -253,6 +231,39 @@ bool FileCompressor::compress_file( return succeeded; } +auto FileCompressor::parse_and_encode ( + size_t target_data_size_of_dicts, + streaming_archive::writer::Archive::UserConfig& archive_user_config, + size_t target_encoded_file_size, + string const& path_for_compression, + group_id_t group_id, + streaming_archive::writer::Archive& archive_writer, + ReaderInterface& reader, + bool use_heuristic +) -> void { + if (use_heuristic) { + parse_and_encode_with_heuristic( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + path_for_compression, + group_id, + archive_writer, + reader + ); + } else { + parse_and_encode_with_library( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + path_for_compression, + group_id, + archive_writer, + reader + ); + } +} + void FileCompressor::parse_and_encode_with_library( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, @@ -423,27 +434,16 @@ bool FileCompressor::try_compressing_as_archive( auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { auto boost_path_for_compression = parent_boost_path / file_path; - if (use_heuristic) { - parse_and_encode_with_heuristic( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - boost_path_for_compression.string(), - file_to_compress.get_group_id(), - archive_writer, - m_libarchive_file_reader - ); - } else { - parse_and_encode_with_library( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - boost_path_for_compression.string(), - file_to_compress.get_group_id(), - archive_writer, - m_libarchive_file_reader - ); - } + parse_and_encode( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + boost_path_for_compression.string(), + file_to_compress.get_group_id(), + archive_writer, + m_libarchive_file_reader, + use_heuristic + ); } else if (has_ir_stream_magic_number({utf8_validation_buf, peek_size})) { // Remove .clp suffix if found static constexpr char cIrStreamExtension[] = ".clp"; diff --git a/components/core/src/clp/clp/FileCompressor.hpp b/components/core/src/clp/clp/FileCompressor.hpp index de4d697fd..4e0b1fd76 100644 --- a/components/core/src/clp/clp/FileCompressor.hpp +++ b/components/core/src/clp/clp/FileCompressor.hpp @@ -58,6 +58,16 @@ class FileCompressor { static constexpr size_t cUtfMaxValidationLen = 4096; // Methods + auto parse_and_encode( + size_t target_data_size_of_dicts, + streaming_archive::writer::Archive::UserConfig& archive_user_config, + size_t target_encoded_file_size, + std::string const& path_for_compression, + group_id_t group_id, + streaming_archive::writer::Archive& archive_writer, + ReaderInterface& reader, + bool use_heuristic + ) -> void; /** * Parses and encodes content from the given reader into the given archive_writer * @param target_data_size_of_dicts From ad868f0f9f3912f4ec5ec4d0feb8ec1da1a0c967 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:56:29 -0400 Subject: [PATCH 11/55] Update constructor --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 25 ++++++++++++++----- .../src/clp/aws/AwsAuthenticationSigner.hpp | 13 +++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index bfa823643..82863c898 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -135,18 +135,16 @@ get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp: namespace clp::aws { S3Url::S3Url(string const& url) { // Regular expression to match virtual host-style HTTP URL format - // std::regex regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/.*?)$)"); - std::regex const regex1(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); + std::regex const host_style_url_regex(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); // Regular expression to match path-style HTTP URL format - // std::regex regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/.*?)$)"); - std::regex const regex2(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); + std::regex const path_style_url_regex(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; - if (std::regex_match(url, match, regex1)) { + if (std::regex_match(url, match, host_style_url_regex)) { m_bucket = match[1].str(); m_region = match[3].str(); m_path = match[4].str(); - } else if (std::regex_match(url, match, regex2)) { + } else if (std::regex_match(url, match, path_style_url_regex)) { m_region = match[2].str(); m_bucket = match[3].str(); m_path = match[4].str(); @@ -160,6 +158,21 @@ S3Url::S3Url(string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } +S3Url::S3Url(string const& s3_uri, string_view region): m_region{region} { + // Regular expression to match S3 URI format. But it does not include region. + std::regex const s3_uri_regex(R"(s3://([a-z0-9.-]+)(/[^?]+).*)"); + + std::smatch match; + if (std::regex_match(s3_uri, match, s3_uri_regex)) { + m_bucket = match[1].str(); + m_path = match[2].str(); + } else { + throw std::invalid_argument("S3 URI format"); + } + + m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); +} + string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index ff493f0b4..10bd40885 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -16,18 +16,17 @@ namespace clp::aws { class S3Url { public: // Constructor - explicit S3Url(std::string const& url); - - S3Url() = delete; + S3Url(std::string const& url); + S3Url(std::string const& s3_uri, std::string_view region); // Methods - [[nodiscard]] std::string_view get_host() { return m_host; } + [[nodiscard]] auto get_host() -> std::string_view { return m_host; } - [[nodiscard]] std::string_view get_bucket() { return m_bucket; } + [[nodiscard]] auto get_bucket() -> std::string_view { return m_bucket; } - [[nodiscard]] std::string_view get_region() { return m_region; } + [[nodiscard]] auto get_region() -> std::string_view { return m_region; } - [[nodiscard]] std::string_view get_path() { return m_path; } + [[nodiscard]] auto get_path() -> std::string_view { return m_path; } private: std::string m_host; From e26a0109094eede82fbc0704d7616a55fee765f2 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:13:38 -0400 Subject: [PATCH 12/55] Apply linter --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 96 ++++++++++++------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 14 +-- components/core/src/clp/aws/Constants.hpp | 10 +- components/core/src/clp/hash_utils.cpp | 48 +++++----- components/core/src/clp/hash_utils.hpp | 12 ++- 5 files changed, 103 insertions(+), 77 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 82863c898..5e04463dd 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -1,18 +1,18 @@ #include "AwsAuthenticationSigner.hpp" -#include "../type_utils.hpp" + #include "../hash_utils.hpp" +#include "../type_utils.hpp" using clp::size_checked_pointer_cast; #include - #include #include using std::span; -using std::vector; using std::string; using std::string_view; +using std::vector; namespace { /** @@ -60,14 +60,13 @@ namespace { * @param canonical_request * @return String to sign */ -[[nodiscard]] string get_string_to_sign( - string_view scope, - string_view timestamp_string, - string_view canonical_request -) { +[[nodiscard]] string +get_string_to_sign(string_view scope, string_view timestamp_string, string_view canonical_request) { vector signed_canonical_request; auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); - auto const signed_canonical_request_str = clp::aws::char_array_to_string({signed_canonical_request.data(), signed_canonical_request.size()}); + auto const signed_canonical_request_str = clp::aws::char_array_to_string( + {signed_canonical_request.data(), signed_canonical_request.size()} + ); return fmt::format( "{}\n{}\n{}\n{}", clp::aws::cAws4HmacSha256, @@ -76,14 +75,14 @@ namespace { signed_canonical_request_str ); } + /** * Gets the encoded URI * @param value * @param encode_slash * @return The encoded URI */ -[[nodiscard]] string -get_encoded_uri(string_view value, bool encode_slash = true) { +[[nodiscard]] string get_encoded_uri(string_view value, bool encode_slash = true) { string encoded_uri; for (auto const c : value) { @@ -105,9 +104,14 @@ get_encoded_uri(string_view value, bool encode_slash = true) { * @param region * @return The scope */ -[[nodiscard]] string -get_scope(string_view date_string, string_view region) { - return fmt::format("{}/{}/{}/{}", date_string, region, clp::aws::cS3Service, clp::aws::cAws4Request); +[[nodiscard]] string get_scope(string_view date_string, string_view region) { + return fmt::format( + "{}/{}/{}/{}", + date_string, + region, + clp::aws::cS3Service, + clp::aws::cAws4Request + ); } /** @@ -117,8 +121,11 @@ get_scope(string_view date_string, string_view region) { * @param query_string Query string * @return Canonical request */ -[[nodiscard]] string -get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp::aws::S3Url& url, string_view query_string) { +[[nodiscard]] string get_canonical_request( + clp::aws::AwsAuthenticationSigner::HttpMethod method, + clp::aws::S3Url& url, + string_view query_string +) { return fmt::format( "{}\n{}\n{}\n{}:{}\n\n{}\n{}", get_method_string(method), @@ -130,14 +137,18 @@ get_canonical_request(clp::aws::AwsAuthenticationSigner::HttpMethod method, clp: clp::aws::cUnsignedPayload ); } -} +} // namespace namespace clp::aws { S3Url::S3Url(string const& url) { // Regular expression to match virtual host-style HTTP URL format - std::regex const host_style_url_regex(R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)"); + std::regex const host_style_url_regex( + R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)" + ); // Regular expression to match path-style HTTP URL format - std::regex const path_style_url_regex(R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)"); + std::regex const path_style_url_regex( + R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)" + ); std::smatch match; if (std::regex_match(url, match, host_style_url_regex)) { @@ -158,7 +169,7 @@ S3Url::S3Url(string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -S3Url::S3Url(string const& s3_uri, string_view region): m_region{region} { +S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { // Regular expression to match S3 URI format. But it does not include region. std::regex const s3_uri_regex(R"(s3://([a-z0-9.-]+)(/[^?]+).*)"); @@ -189,13 +200,14 @@ string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod auto query_string = get_default_query_string(scope, timestamp_string); auto canonical_request = get_canonical_request(method, s3_url, query_string); auto string_to_sign = get_string_to_sign(scope, timestamp_string, canonical_request); - vector signature_key {}; - vector signature {}; + vector signature_key{}; + vector signature{}; auto error_code = get_signature_key(s3_region, date_string, signature_key); error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(signature_key.data()), signature_key.size()}, - {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size()}, - signature + {size_checked_pointer_cast(signature_key.data()), signature_key.size()}, + {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size() + }, + signature ); auto const signature_str = char_array_to_string({signature.data(), signature.size()}); @@ -210,9 +222,12 @@ string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod ); } -[[nodiscard]] ErrorCode -AwsAuthenticationSigner::get_signature_key(string_view region, string_view date_string, vector& signature_key) { - string input_key {cAws4}; +[[nodiscard]] ErrorCode AwsAuthenticationSigner::get_signature_key( + string_view region, + string_view date_string, + vector& signature_key +) { + string input_key{cAws4}; input_key += m_secret_access_key; vector date_key{}; @@ -220,31 +235,38 @@ AwsAuthenticationSigner::get_signature_key(string_view region, string_view date_ vector date_region_service_key{}; auto error_code = get_hmac_sha256_hash( {size_checked_pointer_cast(input_key.data()), input_key.size()}, - {size_checked_pointer_cast(date_string.data()), date_string.size()}, - date_key); + {size_checked_pointer_cast(date_string.data()), date_string.size() + }, + date_key + ); error_code = get_hmac_sha256_hash( {size_checked_pointer_cast(date_key.data()), date_key.size()}, {size_checked_pointer_cast(region.data()), region.size()}, - date_region_key); + date_region_key + ); error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_key.data()), date_region_key.size()}, + {size_checked_pointer_cast(date_region_key.data()), + date_region_key.size()}, {size_checked_pointer_cast(cS3Service.data()), cS3Service.size()}, - date_region_service_key); + date_region_service_key + ); error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_service_key.data()), date_region_service_key.size()}, - {size_checked_pointer_cast(cAws4Request.data()), cAws4Request.size()}, + {size_checked_pointer_cast(date_region_service_key.data()), + date_region_service_key.size()}, + {size_checked_pointer_cast(cAws4Request.data()), + cAws4Request.size()}, signature_key - ); + ); return ErrorCode_Success; } string AwsAuthenticationSigner::get_default_query_string(string_view scope, string_view timestamp_string) { - string uri {m_access_key_id + "/"}; + string uri{m_access_key_id + "/"}; uri += scope; return fmt::format( "{}={}&{}={}&{}={}&{}={}&{}={}", diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 10bd40885..1934eb75f 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -6,8 +6,8 @@ #include #include -#include "Constants.hpp" #include "../ErrorCode.hpp" +#include "Constants.hpp" namespace clp::aws { /** @@ -56,10 +56,7 @@ class AwsAuthenticationSigner { }; // Constructors - AwsAuthenticationSigner( - std::string_view access_key_id, - std::string_view secret_access_key - ) + AwsAuthenticationSigner(std::string_view access_key_id, std::string_view secret_access_key) : m_access_key_id(access_key_id), m_secret_access_key(secret_access_key) {} @@ -89,8 +86,11 @@ class AwsAuthenticationSigner { * @param date_string * @return */ - [[nodiscard]] ErrorCode - get_signature_key(std::string_view region, std::string_view date_string, std::vector& signature_key); + [[nodiscard]] ErrorCode get_signature_key( + std::string_view region, + std::string_view date_string, + std::vector& signature_key + ); // Variables std::string m_access_key_id; diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp index d1aae6098..320432822 100644 --- a/components/core/src/clp/aws/Constants.hpp +++ b/components/core/src/clp/aws/Constants.hpp @@ -5,7 +5,7 @@ namespace clp::aws { // Query String Parameter Names -constexpr std::string_view cXAmzAlgorithm {"X-Amz-Algorithm"}; +constexpr std::string_view cXAmzAlgorithm{"X-Amz-Algorithm"}; constexpr std::string_view cXAmzCredential = "X-Amz-Credential"; constexpr std::string_view cXAmzContentSha256 = "X-Amz-Content-Sha256"; constexpr std::string_view cXAmzDate = "X-Amz-Date"; @@ -14,13 +14,13 @@ constexpr std::string_view cXAmzSignature = "X-Amz-Signature"; constexpr std::string_view cXAmzSignedHeaders = "X-Amz-SignedHeaders"; // Other Constants -constexpr std::string_view cAws4 {"AWS4"}; -constexpr std::string_view cAws4Request {"aws4_request"}; +constexpr std::string_view cAws4{"AWS4"}; +constexpr std::string_view cAws4Request{"aws4_request"}; constexpr std::string_view cAws4HmacSha256 = "AWS4-HMAC-SHA256"; constexpr std::string_view cDefaultSignedHeaders = "host"; constexpr std::string_view cDefaultRegion = "us-east-1"; -constexpr std::string_view cS3Service {"s3"}; -constexpr std::string_view cUnsignedPayload {"UNSIGNED-PAYLOAD"}; +constexpr std::string_view cS3Service{"s3"}; +constexpr std::string_view cUnsignedPayload{"UNSIGNED-PAYLOAD"}; } // namespace clp::aws #endif // CLP_AWS_CONSTANTS_HPP diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index c426e9247..83f743f2c 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -1,24 +1,23 @@ #include "hash_utils.hpp" -#include -#include -#include - -#include -#include #include +#include #include +#include -#include "type_utils.hpp" +#include +#include +#include #include "ErrorCode.hpp" #include "spdlog_with_specializations.hpp" +#include "type_utils.hpp" using clp::size_checked_pointer_cast; -using std::string; -using std::vector; using std::span; +using std::string; using std::string_view; +using std::vector; namespace clp::aws { /** @@ -35,25 +34,27 @@ std::string char_array_to_string(span input) { return hex_string; } -ErrorCode get_hmac_sha256_hash(span key, span input, vector& hash) { - +ErrorCode get_hmac_sha256_hash( + span key, + span input, + vector& hash +) { hash.resize(SHA256_DIGEST_LENGTH); - unsigned int hash_length {0}; + unsigned int hash_length{0}; if (key.size() > INT32_MAX) { SPDLOG_ERROR("Key too long"); return ErrorCode_BadParam; } int key_length{static_cast(key.size())}; - auto* res = HMAC( - EVP_sha256(), - key.data(), - key_length, - input.data(), - input.size(), - hash.data(), - &hash_length - ); + auto* res + = HMAC(EVP_sha256(), + key.data(), + key_length, + input.data(), + input.size(), + hash.data(), + &hash_length); if (nullptr == res) { SPDLOG_ERROR("Failed to get HMAC hashes"); @@ -74,8 +75,7 @@ ErrorCode get_hmac_sha256_hash(span key, span& hash) { - - EVP_MD_CTX * mdctx = EVP_MD_CTX_new(); + EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); @@ -88,4 +88,4 @@ ErrorCode get_sha256_hash(string_view input, std::vector& hash) { return ErrorCode_Success; } -} \ No newline at end of file +} // namespace clp::aws diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 5168f9ec5..42e27d3cb 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -1,12 +1,12 @@ #ifndef CLP_AWS_HASH_UTILS_HPP #define CLP_AWS_HASH_UTILS_HPP -#include "ErrorCode.hpp" - -#include #include #include #include +#include + +#include "ErrorCode.hpp" namespace clp::aws { /** @@ -24,7 +24,11 @@ std::string char_array_to_string(std::span input); * @param hex_output Whether to output the hash as a hex string * @return The HMAC SHA256 hash */ -ErrorCode get_hmac_sha256_hash(std::span key, std::span input, std::vector& hash); +ErrorCode get_hmac_sha256_hash( + std::span key, + std::span input, + std::vector& hash +); /** * Gets the SHA256 hash From 2be15a9bb5c842b78ad5245e2af55585963124b4 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:02:01 -0400 Subject: [PATCH 13/55] More refactoring --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 138 ++++++++++++------ .../src/clp/aws/AwsAuthenticationSigner.hpp | 28 ++-- components/core/src/clp/hash_utils.cpp | 10 +- components/core/src/clp/hash_utils.hpp | 2 +- 4 files changed, 113 insertions(+), 65 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 5e04463dd..7fc38ff10 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -64,7 +64,7 @@ namespace { get_string_to_sign(string_view scope, string_view timestamp_string, string_view canonical_request) { vector signed_canonical_request; auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); - auto const signed_canonical_request_str = clp::aws::char_array_to_string( + auto const signed_canonical_request_str = clp::aws::convert_hash_to_string( {signed_canonical_request.data(), signed_canonical_request.size()} ); return fmt::format( @@ -82,7 +82,7 @@ get_string_to_sign(string_view scope, string_view timestamp_string, string_view * @param encode_slash * @return The encoded URI */ -[[nodiscard]] string get_encoded_uri(string_view value, bool encode_slash = true) { +[[nodiscard]] string encode_uri(string_view value, bool encode_slash = true) { string encoded_uri; for (auto const c : value) { @@ -129,7 +129,7 @@ get_string_to_sign(string_view scope, string_view timestamp_string, string_view return fmt::format( "{}\n{}\n{}\n{}:{}\n\n{}\n{}", get_method_string(method), - get_encoded_uri(url.get_path(), false), + encode_uri(url.get_path(), false), query_string, clp::aws::cDefaultSignedHeaders, url.get_host(), @@ -197,75 +197,119 @@ string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod auto const timestamp_string = get_timestamp_string(now); auto scope = get_scope(date_string, s3_region); - auto query_string = get_default_query_string(scope, timestamp_string); - auto canonical_request = get_canonical_request(method, s3_url, query_string); + auto canonical_query_string = get_canonical_query_string(scope, timestamp_string); + + auto canonical_request = get_canonical_request(method, s3_url, canonical_query_string); auto string_to_sign = get_string_to_sign(scope, timestamp_string, canonical_request); - vector signature_key{}; - vector signature{}; - auto error_code = get_signature_key(s3_region, date_string, signature_key); - error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(signature_key.data()), signature_key.size()}, - {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size() - }, - signature - ); - auto const signature_str = char_array_to_string({signature.data(), signature.size()}); + vector signature{}; + if (auto error_code = get_signature(s3_region, date_string, string_to_sign, signature); + ErrorCode_Success != error_code) + { + throw std::runtime_error("Unexpected error"); + } + auto const signature_str = convert_hash_to_string({signature.data(), signature.size()}); return fmt::format( "https://{}{}?{}&{}={}", s3_url.get_host(), s3_url.get_path(), - query_string, + canonical_query_string, cXAmzSignature, signature_str ); } -[[nodiscard]] ErrorCode AwsAuthenticationSigner::get_signature_key( +ErrorCode AwsAuthenticationSigner::get_signature( + string_view region, + string_view date_string, + string_view string_to_sign, + vector& signature +) { + vector signing_key{}; + if (auto error_code = get_signing_key(region, date_string, signing_key); + ErrorCode_Success != error_code) + { + return error_code; + } + + if (auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(signing_key.data()), + signing_key.size()}, + {size_checked_pointer_cast(string_to_sign.data()), + string_to_sign.size()}, + signature + ); + ErrorCode_Success != error_code) + { + return error_code; + } + return ErrorCode_Success; +} + +ErrorCode AwsAuthenticationSigner::get_signing_key( string_view region, string_view date_string, - vector& signature_key + vector& signing_key ) { - string input_key{cAws4}; - input_key += m_secret_access_key; + string key{cAws4}; + key += m_secret_access_key; vector date_key{}; vector date_region_key{}; vector date_region_service_key{}; - auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(input_key.data()), input_key.size()}, - {size_checked_pointer_cast(date_string.data()), date_string.size() - }, - date_key - ); + if (auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(key.data()), key.size()}, + {size_checked_pointer_cast(date_string.data()), + date_string.size()}, + date_key + ); + error_code != ErrorCode_Success) + { + return error_code; + } - error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_key.data()), date_key.size()}, - {size_checked_pointer_cast(region.data()), region.size()}, - date_region_key - ); + if (auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_key.data()), date_key.size()}, + {size_checked_pointer_cast(region.data()), region.size()}, + date_region_key + ); + error_code != ErrorCode_Success) + { + return error_code; + } - error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_key.data()), - date_region_key.size()}, - {size_checked_pointer_cast(cS3Service.data()), cS3Service.size()}, - date_region_service_key - ); + if (auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_region_key.data()), + date_region_key.size()}, + {size_checked_pointer_cast(cS3Service.data()), + cS3Service.size()}, + date_region_service_key + ); + error_code != ErrorCode_Success) + { + return error_code; + } - error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_service_key.data()), - date_region_service_key.size()}, - {size_checked_pointer_cast(cAws4Request.data()), - cAws4Request.size()}, - signature_key - ); + if (auto error_code = get_hmac_sha256_hash( + {size_checked_pointer_cast(date_region_service_key.data()), + date_region_service_key.size()}, + {size_checked_pointer_cast(cAws4Request.data()), + cAws4Request.size()}, + signing_key + ); + error_code != ErrorCode_Success) + { + return error_code; + } return ErrorCode_Success; } -string -AwsAuthenticationSigner::get_default_query_string(string_view scope, string_view timestamp_string) { +string AwsAuthenticationSigner::get_canonical_query_string( + string_view scope, + string_view timestamp_string +) { string uri{m_access_key_id + "/"}; uri += scope; return fmt::format( @@ -273,7 +317,7 @@ AwsAuthenticationSigner::get_default_query_string(string_view scope, string_view cXAmzAlgorithm, cAws4HmacSha256, cXAmzCredential, - get_encoded_uri(uri), + encode_uri(uri), cXAmzDate, timestamp_string, cXAmzExpires, diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 1934eb75f..1a49ef769 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -22,8 +22,6 @@ class S3Url { // Methods [[nodiscard]] auto get_host() -> std::string_view { return m_host; } - [[nodiscard]] auto get_bucket() -> std::string_view { return m_bucket; } - [[nodiscard]] auto get_region() -> std::string_view { return m_region; } [[nodiscard]] auto get_path() -> std::string_view { return m_path; } @@ -51,10 +49,6 @@ class AwsAuthenticationSigner { DELETE }; - enum class AwsService : uint8_t { - S3 - }; - // Constructors AwsAuthenticationSigner(std::string_view access_key_id, std::string_view secret_access_key) : m_access_key_id(access_key_id), @@ -78,7 +72,7 @@ class AwsAuthenticationSigner { * @return */ [[nodiscard]] std::string - get_default_query_string(std::string_view scope, std::string_view timestamp_string); + get_canonical_query_string(std::string_view scope, std::string_view timestamp_string); /** * Gets the signature key @@ -86,10 +80,26 @@ class AwsAuthenticationSigner { * @param date_string * @return */ - [[nodiscard]] ErrorCode get_signature_key( + [[nodiscard]] ErrorCode get_signing_key( + std::string_view region, + std::string_view date_string, + std::vector& signing_key + ); + + /** + * + */ + /** + * Gets the signature key + * @param region + * @param date_string + * @return + */ + [[nodiscard]] ErrorCode get_signature( std::string_view region, std::string_view date_string, - std::vector& signature_key + std::string_view string_to_sign, + std::vector& signature ); // Variables diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 83f743f2c..699cdf50c 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -20,13 +20,7 @@ using std::string_view; using std::vector; namespace clp::aws { -/** - * Converts a char array to a string - * @param a - * @param size - * @return The converted string - */ -std::string char_array_to_string(span input) { +std::string convert_hash_to_string(span input) { std::string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); @@ -46,7 +40,7 @@ ErrorCode get_hmac_sha256_hash( SPDLOG_ERROR("Key too long"); return ErrorCode_BadParam; } - int key_length{static_cast(key.size())}; + int const key_length{static_cast(key.size())}; auto* res = HMAC(EVP_sha256(), key.data(), diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 42e27d3cb..761e19a7f 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -15,7 +15,7 @@ namespace clp::aws { * @param size * @return The converted string */ -std::string char_array_to_string(std::span input); +std::string convert_hash_to_string(std::span input); /** * Gets the HMAC-SHA256 hash From 836f9b3009a27eae80965f808be34193bba1a3d7 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:37:35 -0400 Subject: [PATCH 14/55] Use trailing return type --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 38 +++++++++++-------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 17 ++++++--- components/core/src/clp/hash_utils.cpp | 10 ++--- components/core/src/clp/hash_utils.hpp | 8 ++-- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 7fc38ff10..b1d5c47f9 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -20,7 +20,8 @@ namespace { * @param timestamp * @return The timestamp string */ -[[nodiscard]] string get_timestamp_string(std::chrono::system_clock::time_point const& timestamp) { +[[nodiscard]] auto get_timestamp_string(std::chrono::system_clock::time_point const& timestamp +) -> string { return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); } @@ -29,7 +30,8 @@ namespace { * @param timestamp * @return The date string */ -[[nodiscard]] string get_date_string(std::chrono::system_clock::time_point const& timestamp) { +[[nodiscard]] auto get_date_string(std::chrono::system_clock::time_point const& timestamp +) -> string { return fmt::format("{:%Y%m%d}", timestamp); } @@ -38,7 +40,8 @@ namespace { * @param method HTTP method * @return The converted string */ -[[nodiscard]] string get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method) { +[[nodiscard]] auto get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method +) -> string { switch (method) { case clp::aws::AwsAuthenticationSigner::HttpMethod::GET: return "GET"; @@ -60,8 +63,11 @@ namespace { * @param canonical_request * @return String to sign */ -[[nodiscard]] string -get_string_to_sign(string_view scope, string_view timestamp_string, string_view canonical_request) { +[[nodiscard]] auto get_string_to_sign( + string_view scope, + string_view timestamp_string, + string_view canonical_request +) -> string { vector signed_canonical_request; auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); auto const signed_canonical_request_str = clp::aws::convert_hash_to_string( @@ -82,7 +88,7 @@ get_string_to_sign(string_view scope, string_view timestamp_string, string_view * @param encode_slash * @return The encoded URI */ -[[nodiscard]] string encode_uri(string_view value, bool encode_slash = true) { +[[nodiscard]] auto encode_uri(string_view value, bool encode_slash = true) -> string { string encoded_uri; for (auto const c : value) { @@ -104,7 +110,7 @@ get_string_to_sign(string_view scope, string_view timestamp_string, string_view * @param region * @return The scope */ -[[nodiscard]] string get_scope(string_view date_string, string_view region) { +[[nodiscard]] auto get_scope(string_view date_string, string_view region) -> string { return fmt::format( "{}/{}/{}/{}", date_string, @@ -121,11 +127,11 @@ get_string_to_sign(string_view scope, string_view timestamp_string, string_view * @param query_string Query string * @return Canonical request */ -[[nodiscard]] string get_canonical_request( +[[nodiscard]] auto get_canonical_request( clp::aws::AwsAuthenticationSigner::HttpMethod method, clp::aws::S3Url& url, string_view query_string -) { +) -> string { return fmt::format( "{}\n{}\n{}\n{}:{}\n\n{}\n{}", get_method_string(method), @@ -184,7 +190,7 @@ S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) { +auto AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) -> string { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); } @@ -220,12 +226,12 @@ string AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod ); } -ErrorCode AwsAuthenticationSigner::get_signature( +auto AwsAuthenticationSigner::get_signature( string_view region, string_view date_string, string_view string_to_sign, vector& signature -) { +) -> ErrorCode { vector signing_key{}; if (auto error_code = get_signing_key(region, date_string, signing_key); ErrorCode_Success != error_code) @@ -247,11 +253,11 @@ ErrorCode AwsAuthenticationSigner::get_signature( return ErrorCode_Success; } -ErrorCode AwsAuthenticationSigner::get_signing_key( +auto AwsAuthenticationSigner::get_signing_key( string_view region, string_view date_string, vector& signing_key -) { +) -> ErrorCode { string key{cAws4}; key += m_secret_access_key; @@ -306,10 +312,10 @@ ErrorCode AwsAuthenticationSigner::get_signing_key( return ErrorCode_Success; } -string AwsAuthenticationSigner::get_canonical_query_string( +auto AwsAuthenticationSigner::get_canonical_query_string( string_view scope, string_view timestamp_string -) { +) -> string { string uri{m_access_key_id + "/"}; uri += scope; return fmt::format( diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 1a49ef769..405ac4edc 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -2,6 +2,9 @@ #define CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP #include +#include +#include +#include #include #include @@ -61,8 +64,8 @@ class AwsAuthenticationSigner { * @param method HTTP method * @return The generated presigned URL */ - [[nodiscard]] std::string - generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET); + [[nodiscard]] auto + generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET) -> std::string; private: /** @@ -71,8 +74,10 @@ class AwsAuthenticationSigner { * @param timestamp_string * @return */ - [[nodiscard]] std::string - get_canonical_query_string(std::string_view scope, std::string_view timestamp_string); + [[nodiscard]] auto get_canonical_query_string( + std::string_view scope, + std::string_view timestamp_string + ) -> std::string; /** * Gets the signature key @@ -80,11 +85,11 @@ class AwsAuthenticationSigner { * @param date_string * @return */ - [[nodiscard]] ErrorCode get_signing_key( + [[nodiscard]] auto get_signing_key( std::string_view region, std::string_view date_string, std::vector& signing_key - ); + ) -> ErrorCode; /** * diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 699cdf50c..752719482 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -20,19 +20,19 @@ using std::string_view; using std::vector; namespace clp::aws { -std::string convert_hash_to_string(span input) { - std::string hex_string; +auto convert_hash_to_string(span input) -> string { + string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); } return hex_string; } -ErrorCode get_hmac_sha256_hash( +auto get_hmac_sha256_hash( span key, span input, vector& hash -) { +) -> ErrorCode { hash.resize(SHA256_DIGEST_LENGTH); unsigned int hash_length{0}; @@ -68,7 +68,7 @@ ErrorCode get_hmac_sha256_hash( * @param input * @return The SHA256 hash */ -ErrorCode get_sha256_hash(string_view input, std::vector& hash) { +auto get_sha256_hash(string_view input, vector& hash) -> ErrorCode { EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 761e19a7f..9dde6aea2 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -15,7 +15,7 @@ namespace clp::aws { * @param size * @return The converted string */ -std::string convert_hash_to_string(std::span input); +auto convert_hash_to_string(std::span input) -> std::string; /** * Gets the HMAC-SHA256 hash @@ -24,17 +24,17 @@ std::string convert_hash_to_string(std::span input); * @param hex_output Whether to output the hash as a hex string * @return The HMAC SHA256 hash */ -ErrorCode get_hmac_sha256_hash( +auto get_hmac_sha256_hash( std::span key, std::span input, std::vector& hash -); +) -> ErrorCode; /** * Gets the SHA256 hash * @param input * @return The SHA256 hash */ -ErrorCode get_sha256_hash(std::string_view, std::vector& hash); +auto get_sha256_hash(std::string_view, std::vector& hash) -> ErrorCode; } // namespace clp::aws #endif // CLP_AWS_HASH_UTILS_HPP From 94ec93e367b28ed45a8cf7eed6da4cfce46cc74b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:51:22 -0400 Subject: [PATCH 15/55] Update function to return Errorcode --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 21 ++++++++++++------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 7 +++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index b1d5c47f9..482aae478 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -157,13 +157,14 @@ S3Url::S3Url(string const& url) { ); std::smatch match; + string bucket{}; if (std::regex_match(url, match, host_style_url_regex)) { - m_bucket = match[1].str(); + bucket = match[1].str(); m_region = match[3].str(); m_path = match[4].str(); } else if (std::regex_match(url, match, path_style_url_regex)) { m_region = match[2].str(); - m_bucket = match[3].str(); + bucket = match[3].str(); m_path = match[4].str(); } else { throw std::invalid_argument("Invalid S3 HTTP URL format"); @@ -172,7 +173,7 @@ S3Url::S3Url(string const& url) { if (m_region.empty()) { m_region = cDefaultRegion; } - m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); + m_host = fmt::format("{}.s3.{}.amazonaws.com", bucket, m_region); } S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { @@ -180,17 +181,22 @@ S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { std::regex const s3_uri_regex(R"(s3://([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; + string bucket{}; if (std::regex_match(s3_uri, match, s3_uri_regex)) { - m_bucket = match[1].str(); + bucket = match[1].str(); m_path = match[2].str(); } else { throw std::invalid_argument("S3 URI format"); } - m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); + m_host = fmt::format("{}.s3.{}.amazonaws.com", bucket, m_region); } -auto AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod method) -> string { +auto AwsAuthenticationSigner::generate_presigned_url( + S3Url& s3_url, + string& presigned_url, + HttpMethod method +) -> ErrorCode { if (HttpMethod::GET != method) { throw std::runtime_error("Unsupported HTTP method!"); } @@ -216,7 +222,7 @@ auto AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod m } auto const signature_str = convert_hash_to_string({signature.data(), signature.size()}); - return fmt::format( + presigned_url = fmt::format( "https://{}{}?{}&{}={}", s3_url.get_host(), s3_url.get_path(), @@ -224,6 +230,7 @@ auto AwsAuthenticationSigner::generate_presigned_url(S3Url& s3_url, HttpMethod m cXAmzSignature, signature_str ); + return ErrorCode_Success; } auto AwsAuthenticationSigner::get_signature( diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 405ac4edc..611145ec3 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -64,8 +64,11 @@ class AwsAuthenticationSigner { * @param method HTTP method * @return The generated presigned URL */ - [[nodiscard]] auto - generate_presigned_url(S3Url& s3_url, HttpMethod method = HttpMethod::GET) -> std::string; + [[nodiscard]] auto generate_presigned_url( + S3Url& s3_url, + std::string& presigned_url, + HttpMethod method = HttpMethod::GET + ) -> ErrorCode; private: /** From 80cb96100b5793e629a5b85af5bb565aa96c2d6b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:14:39 -0400 Subject: [PATCH 16/55] Further updates --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 45 +++++---- components/core/src/clp/hash_utils.cpp | 29 ++++-- components/core/src/clp/hash_utils.hpp | 92 +++++++++++++++---- 3 files changed, 122 insertions(+), 44 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 482aae478..0d549dc25 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -66,20 +66,26 @@ namespace { [[nodiscard]] auto get_string_to_sign( string_view scope, string_view timestamp_string, - string_view canonical_request -) -> string { + string_view canonical_request, + string& string_to_sign +) -> clp::ErrorCode { vector signed_canonical_request; - auto error_code = clp::aws::get_sha256_hash(canonical_request, signed_canonical_request); - auto const signed_canonical_request_str = clp::aws::convert_hash_to_string( + if (auto error_code = clp::get_sha256_hash(canonical_request, signed_canonical_request); + clp::ErrorCode_Success != error_code) + { + return error_code; + } + auto const signed_canonical_request_str = clp::convert_hash_to_hex_string( {signed_canonical_request.data(), signed_canonical_request.size()} ); - return fmt::format( + string_to_sign = fmt::format( "{}\n{}\n{}\n{}", clp::aws::cAws4HmacSha256, timestamp_string, scope, signed_canonical_request_str ); + return clp::ErrorCode_Success; } /** @@ -212,15 +218,22 @@ auto AwsAuthenticationSigner::generate_presigned_url( auto canonical_query_string = get_canonical_query_string(scope, timestamp_string); auto canonical_request = get_canonical_request(method, s3_url, canonical_query_string); - auto string_to_sign = get_string_to_sign(scope, timestamp_string, canonical_request); + + string string_to_sign{}; + if (auto error_code + = get_string_to_sign(scope, timestamp_string, canonical_request, string_to_sign); + ErrorCode_Success != error_code) + { + return error_code; + } vector signature{}; if (auto error_code = get_signature(s3_region, date_string, string_to_sign, signature); ErrorCode_Success != error_code) { - throw std::runtime_error("Unexpected error"); + return error_code; } - auto const signature_str = convert_hash_to_string({signature.data(), signature.size()}); + auto const signature_str = convert_hash_to_hex_string({signature.data(), signature.size()}); presigned_url = fmt::format( "https://{}{}?{}&{}={}", @@ -247,10 +260,10 @@ auto AwsAuthenticationSigner::get_signature( } if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(signing_key.data()), - signing_key.size()}, {size_checked_pointer_cast(string_to_sign.data()), string_to_sign.size()}, + {size_checked_pointer_cast(signing_key.data()), + signing_key.size()}, signature ); ErrorCode_Success != error_code) @@ -272,9 +285,9 @@ auto AwsAuthenticationSigner::get_signing_key( vector date_region_key{}; vector date_region_service_key{}; if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(key.data()), key.size()}, {size_checked_pointer_cast(date_string.data()), date_string.size()}, + {size_checked_pointer_cast(key.data()), key.size()}, date_key ); error_code != ErrorCode_Success) @@ -283,8 +296,8 @@ auto AwsAuthenticationSigner::get_signing_key( } if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_key.data()), date_key.size()}, {size_checked_pointer_cast(region.data()), region.size()}, + {size_checked_pointer_cast(date_key.data()), date_key.size()}, date_region_key ); error_code != ErrorCode_Success) @@ -293,10 +306,10 @@ auto AwsAuthenticationSigner::get_signing_key( } if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_key.data()), - date_region_key.size()}, {size_checked_pointer_cast(cS3Service.data()), cS3Service.size()}, + {size_checked_pointer_cast(date_region_key.data()), + date_region_key.size()}, date_region_service_key ); error_code != ErrorCode_Success) @@ -305,10 +318,10 @@ auto AwsAuthenticationSigner::get_signing_key( } if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_region_service_key.data()), - date_region_service_key.size()}, {size_checked_pointer_cast(cAws4Request.data()), cAws4Request.size()}, + {size_checked_pointer_cast(date_region_service_key.data()), + date_region_service_key.size()}, signing_key ); error_code != ErrorCode_Success) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 752719482..ad9f9322f 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -19,8 +19,8 @@ using std::string; using std::string_view; using std::vector; -namespace clp::aws { -auto convert_hash_to_string(span input) -> string { +namespace clp { +auto convert_hash_to_hex_string(std::span input) -> string { string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); @@ -29,15 +29,15 @@ auto convert_hash_to_string(span input) -> string { } auto get_hmac_sha256_hash( - span key, span input, + span key, vector& hash ) -> ErrorCode { hash.resize(SHA256_DIGEST_LENGTH); unsigned int hash_length{0}; if (key.size() > INT32_MAX) { - SPDLOG_ERROR("Key too long"); + SPDLOG_ERROR("Input key exceeds maximum length"); return ErrorCode_BadParam; } int const key_length{static_cast(key.size())}; @@ -69,17 +69,26 @@ auto get_hmac_sha256_hash( * @return The SHA256 hash */ auto get_sha256_hash(string_view input, vector& hash) -> ErrorCode { - EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); + EvpCtxManager evp_ctx_manager{}; - EVP_DigestInit_ex(mdctx, EVP_sha256(), nullptr); + if (1 != evp_ctx_manager.DigestInitEx(EVP_sha256(), nullptr)) { + SPDLOG_ERROR("Failed to initialize ctx manager"); + return ErrorCode_Failure; + } - EVP_DigestUpdate(mdctx, input.data(), input.size()); + if (1 != evp_ctx_manager.DigestUpdate(input.data(), input.size())) { + SPDLOG_ERROR("Failed to digest input"); + return ErrorCode_Failure; + } unsigned int digest_len{0}; hash.resize(SHA256_DIGEST_LENGTH); - EVP_DigestFinal_ex(mdctx, hash.data(), &digest_len); - EVP_MD_CTX_free(mdctx); + + if (1 != evp_ctx_manager.DigestFinalEx(hash.data(), &digest_len)) { + SPDLOG_ERROR("Failed to Finalize digest"); + return ErrorCode_Failure; + } return ErrorCode_Success; } -} // namespace clp::aws +} // namespace clp diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 9dde6aea2..a69242bf9 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -1,40 +1,96 @@ -#ifndef CLP_AWS_HASH_UTILS_HPP -#define CLP_AWS_HASH_UTILS_HPP +#ifndef CLP_HASH_UTILS_HPP +#define CLP_HASH_UTILS_HPP +#include #include #include #include #include +#include + #include "ErrorCode.hpp" +#include "TraceableException.hpp" -namespace clp::aws { +namespace clp { /** - * Converts a char array to a string - * @param a - * @param size - * @return The converted string + * A class the wraps around openssl EVP_MD_CTX to manage the life cycle of + * dynamically allocated memory. */ -auto convert_hash_to_string(std::span input) -> std::string; +class EvpCtxManager { +public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + + // Methods + [[nodiscard]] auto what() const noexcept -> char const* override { + return "EvpCtxManager operation failed"; + } + }; + + /** + * Constructor + * @throw EvpCtxManager::OperationFailed if a new evp_md_ctx can not be created + */ + EvpCtxManager() : m_md_ctx{EVP_MD_CTX_new()} { + if (nullptr == m_md_ctx) { + throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); + } + } + + auto DigestInitEx(const EVP_MD* type, ENGINE* impl) -> int { + return EVP_DigestInit_ex(m_md_ctx, type, impl); + } + + auto DigestUpdate(void const* d, size_t cnt) -> int { + return EVP_DigestUpdate(m_md_ctx, d, cnt); + } + + auto DigestFinalEx(unsigned char* md, unsigned int* s) -> int { + return EVP_DigestFinal_ex(m_md_ctx, md, s); + } + + ~EvpCtxManager() { EVP_MD_CTX_free(m_md_ctx); } + +private: + EVP_MD_CTX* m_md_ctx{nullptr}; +}; /** - * Gets the HMAC-SHA256 hash + * Converts input hash into a hex string + * @param input + * @return input hash as a hex string + */ +auto convert_hash_to_hex_string(std::span input) -> std::string; + +/** + * Gets the HMAC-SHA256 hash of input with key + * @param input * @param key - * @param value - * @param hex_output Whether to output the hash as a hex string - * @return The HMAC SHA256 hash + * @param hash Returns the result hash by reference. + * @return ErrorCode_BadParam if input key exceeds maximum length. + * ErrorCode_Failure if hash generation fails + * ErrorCode_Success on success */ auto get_hmac_sha256_hash( - std::span key, std::span input, + std::span key, std::vector& hash ) -> ErrorCode; /** - * Gets the SHA256 hash + * Gets the SHA256 hash of input * @param input - * @return The SHA256 hash + * @param hash Returns the result hash by reference. + * @return ErrorCode_BadParam if input key exceeds maximum length. + * ErrorCode_Failure if hash generation fails + * ErrorCode_Success on success + * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initalized */ -auto get_sha256_hash(std::string_view, std::vector& hash) -> ErrorCode; -} // namespace clp::aws -#endif // CLP_AWS_HASH_UTILS_HPP +auto get_sha256_hash(std::string_view input, std::vector& hash) -> ErrorCode; +} // namespace clp +#endif // CLP_HASH_UTILS_HPP From 93662b50ebfb3150c82254264d7dbfae50d52b59 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:31:51 -0400 Subject: [PATCH 17/55] Add preliminary docstrings --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 29 ++++++++------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 37 +++++++++++-------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 0d549dc25..7b4a16a22 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -16,7 +16,7 @@ using std::vector; namespace { /** - * Gets the timestamp string + * Gets the timestamp string specified by AWS Signature Version 4 format * @param timestamp * @return The timestamp string */ @@ -26,7 +26,7 @@ namespace { } /** - * Gets the date string + * Gets the date string specified by AWS Signature Version 4 format * @param timestamp * @return The date string */ @@ -57,11 +57,13 @@ namespace { } /** - * Gets the string to sign + * Gets the string to sign specified by AWS Signature Version 4 format * @param scope * @param timestamp_string * @param canonical_request - * @return String to sign + * @param string_to_sign Returns the string to sign + * @return clp::ErrorCode_Success on success + * On failure, same as clp::get_sha256_hash */ [[nodiscard]] auto get_string_to_sign( string_view scope, @@ -89,15 +91,15 @@ namespace { } /** - * Gets the encoded URI - * @param value + * Encode the uri as specified by AWS Signature Version 4's UriEncode() + * @param uri * @param encode_slash * @return The encoded URI */ -[[nodiscard]] auto encode_uri(string_view value, bool encode_slash = true) -> string { +[[nodiscard]] auto encode_uri(string_view uri, bool encode_slash = true) -> string { string encoded_uri; - for (auto const c : value) { + for (auto const c : uri) { if ((std::isalnum(c) != 0) || c == '-' || c == '_' || c == '.' || c == '~') { encoded_uri += c; } else if (c == '/' && false == encode_slash) { @@ -111,10 +113,10 @@ namespace { } /** - * Gets the scope + * Gets the scope specified by AWS Signature Version 4 format * @param date_string * @param region - * @return The scope + * @return The scope as a string */ [[nodiscard]] auto get_scope(string_view date_string, string_view region) -> string { return fmt::format( @@ -183,7 +185,8 @@ S3Url::S3Url(string const& url) { } S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { - // Regular expression to match S3 URI format. But it does not include region. + // Regular expression to match S3 URI format. + // Note it does not include region. std::regex const s3_uri_regex(R"(s3://([a-z0-9.-]+)(/[^?]+).*)"); std::smatch match; @@ -215,7 +218,7 @@ auto AwsAuthenticationSigner::generate_presigned_url( auto const timestamp_string = get_timestamp_string(now); auto scope = get_scope(date_string, s3_region); - auto canonical_query_string = get_canonical_query_string(scope, timestamp_string); + auto canonical_query_string = generate_canonical_query_string(scope, timestamp_string); auto canonical_request = get_canonical_request(method, s3_url, canonical_query_string); @@ -332,7 +335,7 @@ auto AwsAuthenticationSigner::get_signing_key( return ErrorCode_Success; } -auto AwsAuthenticationSigner::get_canonical_query_string( +auto AwsAuthenticationSigner::generate_canonical_query_string( string_view scope, string_view timestamp_string ) -> string { diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 611145ec3..5c1a0f94e 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -14,7 +14,7 @@ namespace clp::aws { /** - * Class for parsing S3 HTTP URL + * Class for parsing S3 URL */ class S3Url { public: @@ -37,7 +37,9 @@ class S3Url { }; /** - * Class for signing AWS requests + * Class for signing AWS requests. The class is based on AWS Signature Version 4. + * A detailed explanation on how the signature is generated can be found at + * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html */ class AwsAuthenticationSigner { public: @@ -54,15 +56,18 @@ class AwsAuthenticationSigner { // Constructors AwsAuthenticationSigner(std::string_view access_key_id, std::string_view secret_access_key) - : m_access_key_id(access_key_id), - m_secret_access_key(secret_access_key) {} + : m_access_key_id{access_key_id}, + m_secret_access_key{secret_access_key} {} // Methods /** * Generates a presigned URL using AWS Signature Version 4 * @param s3_url S3 URL + * @param presigned_url Returns the generated presigned url * @param method HTTP method - * @return The generated presigned URL + * @return ErrorCode_Success on success. + * On failure, same as get_string_to_sign and AwsAuthenticationSigner::get_signature + * */ [[nodiscard]] auto generate_presigned_url( S3Url& s3_url, @@ -72,21 +77,23 @@ class AwsAuthenticationSigner { private: /** - * Gets the default query string + * Generates the canonical query string * @param scope * @param timestamp_string - * @return + * @return the canonical query string */ - [[nodiscard]] auto get_canonical_query_string( + [[nodiscard]] auto generate_canonical_query_string( std::string_view scope, std::string_view timestamp_string ) -> std::string; /** - * Gets the signature key + * Gets the signature signing key for the request * @param region * @param date_string - * @return + * @param signing_key Returns the signing key + * @return ErrorCode_Success on success. + * Same as get_hmac_sha256_hash on Failure. */ [[nodiscard]] auto get_signing_key( std::string_view region, @@ -95,13 +102,13 @@ class AwsAuthenticationSigner { ) -> ErrorCode; /** - * - */ - /** - * Gets the signature key + * Signs the string_to_sign and returns the request signature by reference * @param region * @param date_string - * @return + * @param string_to_sign StringToSign specified by the AWS Signature Version 4 + * @param signature Returns the signature + * @return ErrorCode_Success on success. + * Same as get_hmac_sha256_hash on Failure. */ [[nodiscard]] ErrorCode get_signature( std::string_view region, From 7e359b9ea32454b8be0f924daca7028fe9547a5f Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:39:24 -0400 Subject: [PATCH 18/55] disable unused constructors --- components/core/src/clp/hash_utils.cpp | 6 +++--- components/core/src/clp/hash_utils.hpp | 22 ++++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index ad9f9322f..02105e4ad 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -71,12 +71,12 @@ auto get_hmac_sha256_hash( auto get_sha256_hash(string_view input, vector& hash) -> ErrorCode { EvpCtxManager evp_ctx_manager{}; - if (1 != evp_ctx_manager.DigestInitEx(EVP_sha256(), nullptr)) { + if (1 != evp_ctx_manager.digest_init_ex(EVP_sha256(), nullptr)) { SPDLOG_ERROR("Failed to initialize ctx manager"); return ErrorCode_Failure; } - if (1 != evp_ctx_manager.DigestUpdate(input.data(), input.size())) { + if (1 != evp_ctx_manager.digest_update(input.data(), input.size())) { SPDLOG_ERROR("Failed to digest input"); return ErrorCode_Failure; } @@ -84,7 +84,7 @@ auto get_sha256_hash(string_view input, vector& hash) -> ErrorCod unsigned int digest_len{0}; hash.resize(SHA256_DIGEST_LENGTH); - if (1 != evp_ctx_manager.DigestFinalEx(hash.data(), &digest_len)) { + if (1 != evp_ctx_manager.digest_final_ex(hash.data(), &digest_len)) { SPDLOG_ERROR("Failed to Finalize digest"); return ErrorCode_Failure; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index a69242bf9..4d277af9e 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -15,7 +15,7 @@ namespace clp { /** * A class the wraps around openssl EVP_MD_CTX to manage the life cycle of - * dynamically allocated memory. + * dynamically allocated EVP_MD_CTX object. */ class EvpCtxManager { public: @@ -32,30 +32,32 @@ class EvpCtxManager { } }; - /** - * Constructor - * @throw EvpCtxManager::OperationFailed if a new evp_md_ctx can not be created - */ + // Constructors EvpCtxManager() : m_md_ctx{EVP_MD_CTX_new()} { if (nullptr == m_md_ctx) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } } - auto DigestInitEx(const EVP_MD* type, ENGINE* impl) -> int { + // Disable copy and move constructor/assignment + EvpCtxManager(EvpCtxManager const&) = delete; + auto operator=(EvpCtxManager const&) -> EvpCtxManager& = delete; + + // Destructor + ~EvpCtxManager() { EVP_MD_CTX_free(m_md_ctx); } + + auto digest_init_ex(const EVP_MD* type, ENGINE* impl) -> int { return EVP_DigestInit_ex(m_md_ctx, type, impl); } - auto DigestUpdate(void const* d, size_t cnt) -> int { + auto digest_update(void const* d, size_t cnt) -> int { return EVP_DigestUpdate(m_md_ctx, d, cnt); } - auto DigestFinalEx(unsigned char* md, unsigned int* s) -> int { + auto digest_final_ex(unsigned char* md, unsigned int* s) -> int { return EVP_DigestFinal_ex(m_md_ctx, md, s); } - ~EvpCtxManager() { EVP_MD_CTX_free(m_md_ctx); } - private: EVP_MD_CTX* m_md_ctx{nullptr}; }; From 1a7b394db01c5b617385deb59b5792fc9e0fc899 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:17:08 -0400 Subject: [PATCH 19/55] caller code --- .../core/src/clp/clp/FileCompressor.cpp | 57 +++++++++++-------- .../core/src/clp/clp/FileCompressor.hpp | 2 +- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index 7f50b2400..cc97b2ea2 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -112,15 +112,17 @@ FileCompressor::FileCompressor( CommandLineArguments::InputSource input_source, boost::uuids::random_generator& uuid_generator, std::unique_ptr reader_parser -) : m_input_source(input_source), m_uuid_generator(uuid_generator), m_reader_parser(std::move(reader_parser)) -{ +) + : m_input_source(input_source), + m_uuid_generator(uuid_generator), + m_reader_parser(std::move(reader_parser)) { if (CommandLineArguments::InputSource::S3 == m_input_source) { if (ErrorCode_Success != NetworkReader::init()) { SPDLOG_ERROR("Failed to initialize streaming reader"); throw std::runtime_error("Failed to initialize streaming reader"); } - string const access_key_id {getenv("AWS_ACCESS_KEY_ID")}; - string const secret_access_key {getenv("AWS_SECRET_ACCESS_KEY")}; + string const access_key_id{getenv("AWS_ACCESS_KEY_ID")}; + string const secret_access_key{getenv("AWS_SECRET_ACCESS_KEY")}; if (access_key_id.empty()) { throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); } @@ -131,7 +133,7 @@ FileCompressor::FileCompressor( } } -FileCompressor::~FileCompressor () { +FileCompressor::~FileCompressor() { if (CommandLineArguments::InputSource::S3 == m_input_source) { NetworkReader::deinit(); } @@ -149,32 +151,39 @@ bool FileCompressor::compress_file( std::string file_name; if (CommandLineArguments::InputSource::S3 == m_input_source) { aws::S3Url s3_url(file_to_compress.get_path()); - file_name = s3_url.get_path().substr(1); - - PROFILER_SPDLOG_INFO("Start parsing {}", file_name) Profiler::start_continuous_measurement( ); - NetworkReader network_reader(m_aws_auth_signer.value().generate_presigned_url(s3_url)); - parse_and_encode( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - network_reader, - use_heuristic - ); + string presigned_url; + if (auto error_code + = m_aws_auth_signer.value().generate_presigned_url(s3_url, presigned_url); + ErrorCode_Success != error_code) + { + SPDLOG_ERROR( + "Failed to generate presigned url for {}, errno={}", + s3_url.get_path(), + error_code + ); + } + NetworkReader network_reader(presigned_url); + parse_and_encode( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + network_reader, + use_heuristic + ); } else if (CommandLineArguments::InputSource::Filesystem == m_input_source) { - file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); - PROFILER_SPDLOG_INFO("Start parsing {}", file_name) - Profiler::start_continuous_measurement(); + Profiler::start_continuous_measurement( + ); m_file_reader.open(file_to_compress.get_path()); // Check that file is UTF-8 encoded if (auto error_code = m_file_reader.try_refill_buffer_if_empty(); - ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) + ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) { if (ErrorCode_errno == error_code) { SPDLOG_ERROR( @@ -231,7 +240,7 @@ bool FileCompressor::compress_file( return succeeded; } -auto FileCompressor::parse_and_encode ( +auto FileCompressor::parse_and_encode( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, size_t target_encoded_file_size, diff --git a/components/core/src/clp/clp/FileCompressor.hpp b/components/core/src/clp/clp/FileCompressor.hpp index 4e0b1fd76..0a4a583c4 100644 --- a/components/core/src/clp/clp/FileCompressor.hpp +++ b/components/core/src/clp/clp/FileCompressor.hpp @@ -12,9 +12,9 @@ #include "../LibarchiveFileReader.hpp" #include "../LibarchiveReader.hpp" #include "../MessageParser.hpp" +#include "../NetworkReader.hpp" #include "../ParsedMessage.hpp" #include "../streaming_archive/writer/Archive.hpp" -#include "../NetworkReader.hpp" #include "aws/AwsAuthenticationSigner.hpp" #include "CommandLineArguments.hpp" #include "FileToCompress.hpp" From edd65e08f9933f7f57dae19dc6f106ed1046f10c Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:47:32 -0400 Subject: [PATCH 20/55] Temporarily revert the changes that are not for this PR --- components/core/src/clp/clp/CMakeLists.txt | 15 -- .../core/src/clp/clp/CommandLineArguments.cpp | 15 -- .../core/src/clp/clp/CommandLineArguments.hpp | 8 - .../core/src/clp/clp/FileCompressor.cpp | 221 ++++++------------ .../core/src/clp/clp/FileCompressor.hpp | 22 +- components/core/src/clp/clp/compression.cpp | 10 +- components/core/src/clp/clp/run.cpp | 56 ++--- 7 files changed, 105 insertions(+), 242 deletions(-) diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 58c98bd47..0f18777d9 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -1,18 +1,10 @@ set( CLP_SOURCES ../ArrayBackedPosIntSet.hpp - ../aws/AwsAuthenticationSigner.cpp - ../aws/AwsAuthenticationSigner.hpp - ../aws/Constants.hpp ../BufferedFileReader.cpp ../BufferedFileReader.hpp ../BufferReader.cpp ../BufferReader.hpp - ../CurlDownloadHandler.cpp - ../CurlDownloadHandler.hpp - ../CurlEasyHandle.hpp - ../CurlOperationFailed.hpp - ../CurlStringList.hpp ../database_utils.cpp ../database_utils.hpp ../Defs.h @@ -48,8 +40,6 @@ set( ../GlobalMySQLMetadataDB.hpp ../GlobalSQLiteMetadataDB.cpp ../GlobalSQLiteMetadataDB.hpp - ../hash_utils.cpp - ../hash_utils.hpp ../ir/constants.hpp ../ir/LogEvent.hpp ../ir/LogEventDeserializer.cpp @@ -136,12 +126,8 @@ set( ../streaming_compression/zstd/Constants.hpp ../streaming_compression/zstd/Decompressor.cpp ../streaming_compression/zstd/Decompressor.hpp - ../NetworkReader.cpp - ../NetworkReader.hpp ../StringReader.cpp ../StringReader.hpp - ../Thread.cpp - ../Thread.hpp ../time_types.hpp ../TimestampPattern.cpp ../TimestampPattern.hpp @@ -184,7 +170,6 @@ target_include_directories(clp PRIVATE "${PROJECT_SOURCE_DIR}/submodules") target_link_libraries(clp PRIVATE Boost::filesystem Boost::iostreams Boost::program_options - ${CURL_LIBRARIES} fmt::fmt log_surgeon::log_surgeon spdlog::spdlog diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index 40e23faa0..ccdc99793 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -332,14 +332,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // boost::program_options doesn't support boolean flags which can be set to false, so // we use a string argument to set the flag manually. string sort_input_files_str = "true"; - string input_source_str = "filesystem"; options_compression.add_options()( - "input-source", - po::value(&input_source_str) - ->value_name("SOURCE") - ->default_value(input_source_str), - "Source of input paths. 'filesystem' or 's3'." - )( "remove-path-prefix", po::value(&m_path_prefix_to_remove) ->value_name("DIR") @@ -427,14 +420,6 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { return ParsingResult::InfoCommand; } - if ("filesystem" == input_source_str) { - m_input_source = InputSource::Filesystem; - } else if ("s3" == input_source_str) { - m_input_source = InputSource::S3; - } else { - throw invalid_argument("Invalid input source specified."); - } - // Validate at least one input path should exist (we validate that the file isn't empty // later) if (m_input_paths.empty() && m_path_list_path.empty()) { diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index b6ac0870c..b9cf15740 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -19,11 +19,6 @@ class CommandLineArguments : public CommandLineArgumentsBase { ExtractIr = 'i' }; - enum class InputSource : uint8_t { - Filesystem, - S3 - }; - // Constructors explicit CommandLineArguments(std::string const& program_name) : CommandLineArgumentsBase(program_name), @@ -72,8 +67,6 @@ class CommandLineArguments : public CommandLineArgumentsBase { std::string const& get_archives_dir() const { return m_archives_dir; } - InputSource get_input_source() const { return m_input_source; } - std::vector const& get_input_paths() const { return m_input_paths; } std::string const& get_orig_file_id() const { return m_orig_file_id; } @@ -109,7 +102,6 @@ class CommandLineArguments : public CommandLineArgumentsBase { int m_compression_level; Command m_command; std::string m_archives_dir; - InputSource m_input_source; std::vector m_input_paths; GlobalMetadataDBConfig m_metadata_db_config; }; diff --git a/components/core/src/clp/clp/FileCompressor.cpp b/components/core/src/clp/clp/FileCompressor.cpp index cc97b2ea2..9898602cc 100644 --- a/components/core/src/clp/clp/FileCompressor.cpp +++ b/components/core/src/clp/clp/FileCompressor.cpp @@ -108,37 +108,6 @@ static void write_message_to_encoded_file( } namespace clp::clp { -FileCompressor::FileCompressor( - CommandLineArguments::InputSource input_source, - boost::uuids::random_generator& uuid_generator, - std::unique_ptr reader_parser -) - : m_input_source(input_source), - m_uuid_generator(uuid_generator), - m_reader_parser(std::move(reader_parser)) { - if (CommandLineArguments::InputSource::S3 == m_input_source) { - if (ErrorCode_Success != NetworkReader::init()) { - SPDLOG_ERROR("Failed to initialize streaming reader"); - throw std::runtime_error("Failed to initialize streaming reader"); - } - string const access_key_id{getenv("AWS_ACCESS_KEY_ID")}; - string const secret_access_key{getenv("AWS_SECRET_ACCESS_KEY")}; - if (access_key_id.empty()) { - throw std::invalid_argument("AWS_ACCESS_KEY_ID environment variable is not set"); - } - if (secret_access_key.empty()) { - throw std::invalid_argument("AWS_SECRET_ACCESS_KEY environment variable is not set"); - } - m_aws_auth_signer.emplace(access_key_id, secret_access_key); - } -} - -FileCompressor::~FileCompressor() { - if (CommandLineArguments::InputSource::S3 == m_input_source) { - NetworkReader::deinit(); - } -} - bool FileCompressor::compress_file( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, @@ -147,92 +116,76 @@ bool FileCompressor::compress_file( streaming_archive::writer::Archive& archive_writer, bool use_heuristic ) { - bool succeeded = true; - std::string file_name; - if (CommandLineArguments::InputSource::S3 == m_input_source) { - aws::S3Url s3_url(file_to_compress.get_path()); - Profiler::start_continuous_measurement( - ); - string presigned_url; - if (auto error_code - = m_aws_auth_signer.value().generate_presigned_url(s3_url, presigned_url); - ErrorCode_Success != error_code) - { + std::string file_name = std::filesystem::canonical(file_to_compress.get_path()).string(); + + PROFILER_SPDLOG_INFO("Start parsing {}", file_name) + Profiler::start_continuous_measurement(); + + m_file_reader.open(file_to_compress.get_path()); + + // Check that file is UTF-8 encoded + if (auto error_code = m_file_reader.try_refill_buffer_if_empty(); + ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) + { + if (ErrorCode_errno == error_code) { + SPDLOG_ERROR( + "Failed to read {} into buffer, errno={}", + file_to_compress.get_path(), + errno + ); + } else { SPDLOG_ERROR( - "Failed to generate presigned url for {}, errno={}", - s3_url.get_path(), + "Failed to read {} into buffer, error={}", + file_to_compress.get_path(), error_code ); } - NetworkReader network_reader(presigned_url); - parse_and_encode( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress.get_path_for_compression(), - file_to_compress.get_group_id(), - archive_writer, - network_reader, - use_heuristic - ); - } else if (CommandLineArguments::InputSource::Filesystem == m_input_source) { - Profiler::start_continuous_measurement( - ); - - m_file_reader.open(file_to_compress.get_path()); - - // Check that file is UTF-8 encoded - if (auto error_code = m_file_reader.try_refill_buffer_if_empty(); - ErrorCode_Success != error_code && ErrorCode_EndOfFile != error_code) - { - if (ErrorCode_errno == error_code) { - SPDLOG_ERROR( - "Failed to read {} into buffer, errno={}", - file_to_compress.get_path(), - errno - ); - } else { - SPDLOG_ERROR( - "Failed to read {} into buffer, error={}", - file_to_compress.get_path(), - error_code - ); - } - return false; - } - char const* utf8_validation_buf{nullptr}; - size_t peek_size{0}; - m_file_reader.peek_buffered_data(utf8_validation_buf, peek_size); - auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); - if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { - parse_and_encode( + return false; + } + char const* utf8_validation_buf{nullptr}; + size_t peek_size{0}; + m_file_reader.peek_buffered_data(utf8_validation_buf, peek_size); + bool succeeded = true; + auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); + if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { + if (use_heuristic) { + parse_and_encode_with_heuristic( target_data_size_of_dicts, archive_user_config, target_encoded_file_size, file_to_compress.get_path_for_compression(), file_to_compress.get_group_id(), archive_writer, - m_file_reader, - use_heuristic + m_file_reader ); } else { - if (false - == try_compressing_as_archive( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - file_to_compress, - archive_writer, - use_heuristic - )) - { - succeeded = false; - } + parse_and_encode_with_library( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress.get_path_for_compression(), + file_to_compress.get_group_id(), + archive_writer, + m_file_reader + ); + } + } else { + if (false + == try_compressing_as_archive( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + file_to_compress, + archive_writer, + use_heuristic + )) + { + succeeded = false; } - - m_file_reader.close(); } + m_file_reader.close(); + Profiler::stop_continuous_measurement(); LOG_CONTINUOUS_MEASUREMENT(Profiler::ContinuousMeasurementIndex::ParseLogFile) PROFILER_SPDLOG_INFO("Done parsing {}", file_name) @@ -240,39 +193,6 @@ bool FileCompressor::compress_file( return succeeded; } -auto FileCompressor::parse_and_encode( - size_t target_data_size_of_dicts, - streaming_archive::writer::Archive::UserConfig& archive_user_config, - size_t target_encoded_file_size, - string const& path_for_compression, - group_id_t group_id, - streaming_archive::writer::Archive& archive_writer, - ReaderInterface& reader, - bool use_heuristic -) -> void { - if (use_heuristic) { - parse_and_encode_with_heuristic( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - path_for_compression, - group_id, - archive_writer, - reader - ); - } else { - parse_and_encode_with_library( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - path_for_compression, - group_id, - archive_writer, - reader - ); - } -} - void FileCompressor::parse_and_encode_with_library( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, @@ -443,16 +363,27 @@ bool FileCompressor::try_compressing_as_archive( auto const utf8_validation_buf_len = std::min(peek_size, cUtfMaxValidationLen); if (is_utf8_encoded({utf8_validation_buf, utf8_validation_buf_len})) { auto boost_path_for_compression = parent_boost_path / file_path; - parse_and_encode( - target_data_size_of_dicts, - archive_user_config, - target_encoded_file_size, - boost_path_for_compression.string(), - file_to_compress.get_group_id(), - archive_writer, - m_libarchive_file_reader, - use_heuristic - ); + if (use_heuristic) { + parse_and_encode_with_heuristic( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + boost_path_for_compression.string(), + file_to_compress.get_group_id(), + archive_writer, + m_libarchive_file_reader + ); + } else { + parse_and_encode_with_library( + target_data_size_of_dicts, + archive_user_config, + target_encoded_file_size, + boost_path_for_compression.string(), + file_to_compress.get_group_id(), + archive_writer, + m_libarchive_file_reader + ); + } } else if (has_ir_stream_magic_number({utf8_validation_buf, peek_size})) { // Remove .clp suffix if found static constexpr char cIrStreamExtension[] = ".clp"; diff --git a/components/core/src/clp/clp/FileCompressor.hpp b/components/core/src/clp/clp/FileCompressor.hpp index 0a4a583c4..b8b6c55fd 100644 --- a/components/core/src/clp/clp/FileCompressor.hpp +++ b/components/core/src/clp/clp/FileCompressor.hpp @@ -12,11 +12,8 @@ #include "../LibarchiveFileReader.hpp" #include "../LibarchiveReader.hpp" #include "../MessageParser.hpp" -#include "../NetworkReader.hpp" #include "../ParsedMessage.hpp" #include "../streaming_archive/writer/Archive.hpp" -#include "aws/AwsAuthenticationSigner.hpp" -#include "CommandLineArguments.hpp" #include "FileToCompress.hpp" namespace clp::clp { @@ -27,12 +24,11 @@ class FileCompressor { public: // Constructors FileCompressor( - CommandLineArguments::InputSource input_source, boost::uuids::random_generator& uuid_generator, std::unique_ptr reader_parser - ); - - ~FileCompressor(); + ) + : m_uuid_generator(uuid_generator), + m_reader_parser(std::move(reader_parser)) {} // Methods /** @@ -58,16 +54,6 @@ class FileCompressor { static constexpr size_t cUtfMaxValidationLen = 4096; // Methods - auto parse_and_encode( - size_t target_data_size_of_dicts, - streaming_archive::writer::Archive::UserConfig& archive_user_config, - size_t target_encoded_file_size, - std::string const& path_for_compression, - group_id_t group_id, - streaming_archive::writer::Archive& archive_writer, - ReaderInterface& reader, - bool use_heuristic - ) -> void; /** * Parses and encodes content from the given reader into the given archive_writer * @param target_data_size_of_dicts @@ -163,12 +149,10 @@ class FileCompressor { ); // Variables - CommandLineArguments::InputSource m_input_source; boost::uuids::random_generator& m_uuid_generator; BufferedFileReader m_file_reader; LibarchiveReader m_libarchive_reader; LibarchiveFileReader m_libarchive_file_reader; - std::optional m_aws_auth_signer; MessageParser m_message_parser; ParsedMessage m_parsed_message; std::unique_ptr m_reader_parser; diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 3852afbd3..1741557bc 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -117,11 +117,7 @@ bool compress( archive_writer.add_empty_directories(empty_directory_paths); bool all_files_compressed_successfully = true; - FileCompressor file_compressor( - command_line_args.get_input_source(), - uuid_generator, - std::move(reader_parser) - ); + FileCompressor file_compressor(uuid_generator, std::move(reader_parser)); auto target_data_size_of_dictionaries = command_line_args.get_target_data_size_of_dictionaries(); @@ -131,9 +127,7 @@ bool compress( if (command_line_args.show_progress()) { num_files_to_compress = files_to_compress.size() + grouped_files_to_compress.size(); } - if (CommandLineArguments::InputSource::Filesystem == command_line_args.get_input_source() - && command_line_args.sort_input_files()) - { + if (command_line_args.sort_input_files()) { sort(files_to_compress.begin(), files_to_compress.end(), file_gt_last_write_time_comparator ); } diff --git a/components/core/src/clp/clp/run.cpp b/components/core/src/clp/clp/run.cpp index 4d35b34b1..5a3b0eb27 100644 --- a/components/core/src/clp/clp/run.cpp +++ b/components/core/src/clp/clp/run.cpp @@ -63,45 +63,37 @@ int run(int argc, char const* argv[]) { reader_parser = std::make_unique(schema_file_path); } + boost::filesystem::path path_prefix_to_remove(command_line_args.get_path_prefix_to_remove() + ); + + // Validate input paths exist + if (false == validate_paths_exist(input_paths)) { + return -1; + } + // Get paths of all files we need to compress vector files_to_compress; - vector grouped_files_to_compress; vector empty_directory_paths; - if (CommandLineArguments::InputSource::S3 == command_line_args.get_input_source()) { - for (auto const& input_path : input_paths) { - files_to_compress.emplace_back(input_path, input_path, 0); - } - } else if ((CommandLineArguments::InputSource::Filesystem - == command_line_args.get_input_source())) - { - boost::filesystem::path path_prefix_to_remove( - command_line_args.get_path_prefix_to_remove() - ); - - // Validate input paths exist - if (false == validate_paths_exist(input_paths)) { + for (auto const& input_path : input_paths) { + if (false + == find_all_files_and_empty_directories( + path_prefix_to_remove, + input_path, + files_to_compress, + empty_directory_paths + )) + { return -1; } + } - for (auto const& input_path : input_paths) { - if (false - == find_all_files_and_empty_directories( - path_prefix_to_remove, - input_path, - files_to_compress, - empty_directory_paths - )) - { - return -1; - } - } + vector grouped_files_to_compress; - if (files_to_compress.empty() && empty_directory_paths.empty() - && grouped_files_to_compress.empty()) - { - SPDLOG_ERROR("No files/directories to compress."); - return -1; - } + if (files_to_compress.empty() && empty_directory_paths.empty() + && grouped_files_to_compress.empty()) + { + SPDLOG_ERROR("No files/directories to compress."); + return -1; } bool compression_successful; From 3acb83837ef74bccd00cc7ebe68f3be5505c6b64 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:08:14 -0400 Subject: [PATCH 21/55] polishing --- components/core/src/clp/aws/Constants.hpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp index 320432822..dd5032df7 100644 --- a/components/core/src/clp/aws/Constants.hpp +++ b/components/core/src/clp/aws/Constants.hpp @@ -6,19 +6,18 @@ namespace clp::aws { // Query String Parameter Names constexpr std::string_view cXAmzAlgorithm{"X-Amz-Algorithm"}; -constexpr std::string_view cXAmzCredential = "X-Amz-Credential"; -constexpr std::string_view cXAmzContentSha256 = "X-Amz-Content-Sha256"; -constexpr std::string_view cXAmzDate = "X-Amz-Date"; -constexpr std::string_view cXAmzExpires = "X-Amz-Expires"; -constexpr std::string_view cXAmzSignature = "X-Amz-Signature"; -constexpr std::string_view cXAmzSignedHeaders = "X-Amz-SignedHeaders"; +constexpr std::string_view cXAmzCredential{"X-Amz-Credential"}; +constexpr std::string_view cXAmzDate{"X-Amz-Date"}; +constexpr std::string_view cXAmzExpires{"X-Amz-Expires"}; +constexpr std::string_view cXAmzSignature{"X-Amz-Signature"}; +constexpr std::string_view cXAmzSignedHeaders{"X-Amz-SignedHeaders"}; // Other Constants constexpr std::string_view cAws4{"AWS4"}; constexpr std::string_view cAws4Request{"aws4_request"}; -constexpr std::string_view cAws4HmacSha256 = "AWS4-HMAC-SHA256"; -constexpr std::string_view cDefaultSignedHeaders = "host"; -constexpr std::string_view cDefaultRegion = "us-east-1"; +constexpr std::string_view cAws4HmacSha256{"AWS4-HMAC-SHA256"}; +constexpr std::string_view cDefaultSignedHeaders{"host"}; +constexpr std::string_view cDefaultRegion{"us-east-1"}; constexpr std::string_view cS3Service{"s3"}; constexpr std::string_view cUnsignedPayload{"UNSIGNED-PAYLOAD"}; } // namespace clp::aws From 8ba922803eb730d1ee6f92327030685209945a89 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:09:23 -0400 Subject: [PATCH 22/55] polishing --- components/core/src/clp/aws/AwsAuthenticationSigner.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 7b4a16a22..b650d7403 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -2,12 +2,14 @@ #include "../hash_utils.hpp" #include "../type_utils.hpp" +#include "../ErrorCode.hpp" using clp::size_checked_pointer_cast; #include #include #include +#include using std::span; using std::string; From ba9729544d96d93e274590299a1d29eb1c24568b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:12:43 -0400 Subject: [PATCH 23/55] Linter --- components/core/src/clp/aws/AwsAuthenticationSigner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index b650d7403..67b7cb00e 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -1,15 +1,15 @@ #include "AwsAuthenticationSigner.hpp" +#include "../ErrorCode.hpp" #include "../hash_utils.hpp" #include "../type_utils.hpp" -#include "../ErrorCode.hpp" using clp::size_checked_pointer_cast; +#include #include #include #include -#include using std::span; using std::string; From 3c5fb34f84ec5d1e93b948c3791599ee47a2ddcf Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 23 Jul 2024 18:03:46 -0400 Subject: [PATCH 24/55] Remove unused constructors --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 21 ++----------------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 3 +-- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 67b7cb00e..7c5dab99d 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -157,11 +157,11 @@ namespace { namespace clp::aws { S3Url::S3Url(string const& url) { - // Regular expression to match virtual host-style HTTP URL format + // Regular expression to match Virtual-hosted-style HTTP URL format std::regex const host_style_url_regex( R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)" ); - // Regular expression to match path-style HTTP URL format + // Regular expression to match Path-style HTTP URL format std::regex const path_style_url_regex( R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)" ); @@ -186,23 +186,6 @@ S3Url::S3Url(string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", bucket, m_region); } -S3Url::S3Url(string const& s3_uri, string_view region) : m_region{region} { - // Regular expression to match S3 URI format. - // Note it does not include region. - std::regex const s3_uri_regex(R"(s3://([a-z0-9.-]+)(/[^?]+).*)"); - - std::smatch match; - string bucket{}; - if (std::regex_match(s3_uri, match, s3_uri_regex)) { - bucket = match[1].str(); - m_path = match[2].str(); - } else { - throw std::invalid_argument("S3 URI format"); - } - - m_host = fmt::format("{}.s3.{}.amazonaws.com", bucket, m_region); -} - auto AwsAuthenticationSigner::generate_presigned_url( S3Url& s3_url, string& presigned_url, diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 5c1a0f94e..d9eb65817 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -14,13 +14,12 @@ namespace clp::aws { /** - * Class for parsing S3 URL + * Class for parsing S3 URL. The format of S3 URL is specie */ class S3Url { public: // Constructor S3Url(std::string const& url); - S3Url(std::string const& s3_uri, std::string_view region); // Methods [[nodiscard]] auto get_host() -> std::string_view { return m_host; } From 98188f11e97d011cbf4b3f70cb85b6bc35fad014 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:45:02 -0400 Subject: [PATCH 25/55] Refactor --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 17 ++++---- .../src/clp/aws/AwsAuthenticationSigner.hpp | 41 +++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 7c5dab99d..a5009b00d 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -139,7 +139,7 @@ namespace { */ [[nodiscard]] auto get_canonical_request( clp::aws::AwsAuthenticationSigner::HttpMethod method, - clp::aws::S3Url& url, + clp::aws::S3Url const& url, string_view query_string ) -> string { return fmt::format( @@ -167,27 +167,30 @@ S3Url::S3Url(string const& url) { ); std::smatch match; - string bucket{}; if (std::regex_match(url, match, host_style_url_regex)) { - bucket = match[1].str(); + m_bucket = match[1].str(); m_region = match[3].str(); m_path = match[4].str(); } else if (std::regex_match(url, match, path_style_url_regex)) { m_region = match[2].str(); - bucket = match[3].str(); + m_bucket = match[3].str(); m_path = match[4].str(); } else { - throw std::invalid_argument("Invalid S3 HTTP URL format"); + throw OperationFailed(ErrorCode_BadParam, __FILENAME__, __LINE__, "Invalid S3 HTTP URL format"); } if (m_region.empty()) { m_region = cDefaultRegion; } - m_host = fmt::format("{}.s3.{}.amazonaws.com", bucket, m_region); + m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); +} + +auto S3Url::get_compression_path () const -> string { + return fmt::format("/{}/{}{}", m_region, m_bucket, m_path); } auto AwsAuthenticationSigner::generate_presigned_url( - S3Url& s3_url, + S3Url const& s3_url, string& presigned_url, HttpMethod method ) -> ErrorCode { diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index d9eb65817..f656d3319 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -11,6 +11,7 @@ #include "../ErrorCode.hpp" #include "Constants.hpp" +#include "../TraceableException.hpp" namespace clp::aws { /** @@ -18,15 +19,47 @@ namespace clp::aws { */ class S3Url { public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : OperationFailed( + error_code, + filename, + line_number, + "S3Url operation failed" + ) {} + + OperationFailed( + ErrorCode error_code, + char const* const filename, + int line_number, + std::string message + ) + : TraceableException(error_code, filename, line_number), + m_message(std::move(message)) {} + + // Methods + [[nodiscard]] auto what() const noexcept -> char const* override { + return m_message.c_str(); + } + + private: + std::string m_message; + }; + // Constructor S3Url(std::string const& url); // Methods - [[nodiscard]] auto get_host() -> std::string_view { return m_host; } + [[nodiscard]] auto get_host() const -> std::string_view { return m_host; } + + [[nodiscard]] auto get_region() const -> std::string_view { return m_region; } - [[nodiscard]] auto get_region() -> std::string_view { return m_region; } + [[nodiscard]] auto get_path() const-> std::string_view { return m_path; } - [[nodiscard]] auto get_path() -> std::string_view { return m_path; } + [[nodiscard]] auto get_compression_path() const -> std::string; private: std::string m_host; @@ -69,7 +102,7 @@ class AwsAuthenticationSigner { * */ [[nodiscard]] auto generate_presigned_url( - S3Url& s3_url, + S3Url const& s3_url, std::string& presigned_url, HttpMethod method = HttpMethod::GET ) -> ErrorCode; From fdcb235c4b47693f8aaab4da7064f009bf1d06a6 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:51:02 -0400 Subject: [PATCH 26/55] touches --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 22 +++++++++---------- .../src/clp/aws/AwsAuthenticationSigner.hpp | 20 +++++------------ 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index a5009b00d..b106dd6e9 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -176,7 +176,12 @@ S3Url::S3Url(string const& url) { m_bucket = match[3].str(); m_path = match[4].str(); } else { - throw OperationFailed(ErrorCode_BadParam, __FILENAME__, __LINE__, "Invalid S3 HTTP URL format"); + throw OperationFailed( + ErrorCode_BadParam, + __FILENAME__, + __LINE__, + "Invalid S3 HTTP URL format" + ); } if (m_region.empty()) { @@ -185,19 +190,12 @@ S3Url::S3Url(string const& url) { m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); } -auto S3Url::get_compression_path () const -> string { +auto S3Url::get_compression_path() const -> string { return fmt::format("/{}/{}{}", m_region, m_bucket, m_path); } -auto AwsAuthenticationSigner::generate_presigned_url( - S3Url const& s3_url, - string& presigned_url, - HttpMethod method -) -> ErrorCode { - if (HttpMethod::GET != method) { - throw std::runtime_error("Unsupported HTTP method!"); - } - +auto AwsAuthenticationSigner::generate_presigned_url(S3Url const& s3_url, string& presigned_url) + -> ErrorCode { auto const s3_region = s3_url.get_region(); // Gets current time @@ -208,7 +206,7 @@ auto AwsAuthenticationSigner::generate_presigned_url( auto scope = get_scope(date_string, s3_region); auto canonical_query_string = generate_canonical_query_string(scope, timestamp_string); - auto canonical_request = get_canonical_request(method, s3_url, canonical_query_string); + auto canonical_request = get_canonical_request(HttpMethod::GET, s3_url, canonical_query_string); string string_to_sign{}; if (auto error_code diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index f656d3319..244711785 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -10,8 +10,8 @@ #include #include "../ErrorCode.hpp" -#include "Constants.hpp" #include "../TraceableException.hpp" +#include "Constants.hpp" namespace clp::aws { /** @@ -24,12 +24,7 @@ class S3Url { public: // Constructors OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : OperationFailed( - error_code, - filename, - line_number, - "S3Url operation failed" - ) {} + : OperationFailed(error_code, filename, line_number, "S3Url operation failed") {} OperationFailed( ErrorCode error_code, @@ -38,7 +33,7 @@ class S3Url { std::string message ) : TraceableException(error_code, filename, line_number), - m_message(std::move(message)) {} + m_message(std::move(message)) {} // Methods [[nodiscard]] auto what() const noexcept -> char const* override { @@ -57,7 +52,7 @@ class S3Url { [[nodiscard]] auto get_region() const -> std::string_view { return m_region; } - [[nodiscard]] auto get_path() const-> std::string_view { return m_path; } + [[nodiscard]] auto get_path() const -> std::string_view { return m_path; } [[nodiscard]] auto get_compression_path() const -> std::string; @@ -101,11 +96,8 @@ class AwsAuthenticationSigner { * On failure, same as get_string_to_sign and AwsAuthenticationSigner::get_signature * */ - [[nodiscard]] auto generate_presigned_url( - S3Url const& s3_url, - std::string& presigned_url, - HttpMethod method = HttpMethod::GET - ) -> ErrorCode; + [[nodiscard]] auto + generate_presigned_url(S3Url const& s3_url, std::string& presigned_url) -> ErrorCode; private: /** From 31473161bf6de65b9edde471848cca992969f2df Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:10:13 -0400 Subject: [PATCH 27/55] Update function interface --- components/core/src/clp/aws/AwsAuthenticationSigner.cpp | 6 +++++- components/core/src/clp/hash_utils.cpp | 4 ++-- components/core/src/clp/hash_utils.hpp | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 67b7cb00e..d27b107ec 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -74,7 +74,11 @@ namespace { string& string_to_sign ) -> clp::ErrorCode { vector signed_canonical_request; - if (auto error_code = clp::get_sha256_hash(canonical_request, signed_canonical_request); + if (auto error_code = clp::get_sha256_hash( + {size_checked_pointer_cast(canonical_request.data()), + canonical_request.size()}, + signed_canonical_request + ); clp::ErrorCode_Success != error_code) { return error_code; diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 02105e4ad..6648572b6 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -20,7 +20,7 @@ using std::string_view; using std::vector; namespace clp { -auto convert_hash_to_hex_string(std::span input) -> string { +auto convert_hash_to_hex_string(span input) -> string { string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); @@ -68,7 +68,7 @@ auto get_hmac_sha256_hash( * @param input * @return The SHA256 hash */ -auto get_sha256_hash(string_view input, vector& hash) -> ErrorCode { +auto get_sha256_hash(span input, vector& hash) -> ErrorCode { EvpCtxManager evp_ctx_manager{}; if (1 != evp_ctx_manager.digest_init_ex(EVP_sha256(), nullptr)) { diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 4d277af9e..fdfcb6552 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -93,6 +93,7 @@ auto get_hmac_sha256_hash( * ErrorCode_Success on success * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initalized */ -auto get_sha256_hash(std::string_view input, std::vector& hash) -> ErrorCode; +auto get_sha256_hash(std::span input, std::vector& hash) + -> ErrorCode; } // namespace clp #endif // CLP_HASH_UTILS_HPP From 48c14b0af16882de6db729e9ffdffd5c5cc43d4b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:24:10 -0400 Subject: [PATCH 28/55] Add simply unit-test --- components/core/CMakeLists.txt | 3 ++ components/core/tests/test-hash_utils.cpp | 50 +++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 components/core/tests/test-hash_utils.cpp diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 57ae8cda1..6080e32f1 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -344,6 +344,8 @@ set(SOURCE_FILES_unitTest src/clp/GlobalSQLiteMetadataDB.hpp src/clp/Grep.cpp src/clp/Grep.hpp + src/clp/hash_utils.cpp + src/clp/hash_utils.hpp src/clp/ir/constants.hpp src/clp/ir/LogEvent.hpp src/clp/ir/LogEventDeserializer.cpp @@ -462,6 +464,7 @@ set(SOURCE_FILES_unitTest tests/test-encoding_methods.cpp tests/test-ffi_SchemaTree.cpp tests/test-Grep.cpp + tests/test-hash_utils.cpp tests/test-ir_encoding_methods.cpp tests/test-ir_parsing.cpp tests/test-kql.cpp diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp new file mode 100644 index 000000000..6c32d1a95 --- /dev/null +++ b/components/core/tests/test-hash_utils.cpp @@ -0,0 +1,50 @@ +#include +#include + +#include + +#include "../src/clp/ErrorCode.hpp" +#include "../src/clp/hash_utils.hpp" +#include "../src/clp/type_utils.hpp" +using clp::convert_hash_to_hex_string; +using clp::ErrorCode_Success; +using clp::get_hmac_sha256_hash; +using clp::get_sha256_hash; +using clp::size_checked_pointer_cast; +using std::string_view; +using std::vector; + +TEST_CASE("test_sha256", "[hash_utils]") { + constexpr string_view input_string{"ThisIsARandomTestInput"}; + constexpr string_view reference_sha256{ + "c3a1d9f04ada1198c4c63bf51d9933fc2cc216429275cadabdcb2178775add0c" + }; + vector hash{}; + + CHECK(ErrorCode_Success + == get_sha256_hash( + {size_checked_pointer_cast(input_string.data()), + input_string.size()}, + hash + )); + CHECK(convert_hash_to_hex_string(hash) == reference_sha256); +} + +TEST_CASE("test_hmac", "[hash_utils]") { + string_view input_string{"ThisIsARandomTestInput"}; + string_view input_key{"ThisIsATestKey"}; + string_view reference_hmac_sha256{ + "38373057694c1038a6895212bea46849eb7a59b73a2ec175883ae095fb91ffda" + }; + vector hmac_hash{}; + + CHECK(ErrorCode_Success + == get_hmac_sha256_hash( + {size_checked_pointer_cast(input_string.data()), + input_string.size()}, + {size_checked_pointer_cast(input_key.data()), + input_key.size()}, + hmac_hash + )); + CHECK(convert_hash_to_hex_string(hmac_hash) == reference_hmac_sha256); +} From 7b2b83bf503a7e689cbe4adb0259ffb419db7095 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:26:34 -0400 Subject: [PATCH 29/55] Remove S3 related changes --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 349 ------------------ .../src/clp/aws/AwsAuthenticationSigner.hpp | 150 -------- components/core/src/clp/aws/Constants.hpp | 25 -- 3 files changed, 524 deletions(-) delete mode 100644 components/core/src/clp/aws/AwsAuthenticationSigner.cpp delete mode 100644 components/core/src/clp/aws/AwsAuthenticationSigner.hpp delete mode 100644 components/core/src/clp/aws/Constants.hpp diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp deleted file mode 100644 index 4371ad9af..000000000 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ /dev/null @@ -1,349 +0,0 @@ -#include "AwsAuthenticationSigner.hpp" - -#include "../ErrorCode.hpp" -#include "../hash_utils.hpp" -#include "../type_utils.hpp" - -using clp::size_checked_pointer_cast; - -#include -#include -#include -#include - -using std::span; -using std::string; -using std::string_view; -using std::vector; - -namespace { -/** - * Gets the timestamp string specified by AWS Signature Version 4 format - * @param timestamp - * @return The timestamp string - */ -[[nodiscard]] auto get_timestamp_string(std::chrono::system_clock::time_point const& timestamp -) -> string { - return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp); -} - -/** - * Gets the date string specified by AWS Signature Version 4 format - * @param timestamp - * @return The date string - */ -[[nodiscard]] auto get_date_string(std::chrono::system_clock::time_point const& timestamp -) -> string { - return fmt::format("{:%Y%m%d}", timestamp); -} - -/** - * Converts an HttpMethod to a string - * @param method HTTP method - * @return The converted string - */ -[[nodiscard]] auto get_method_string(clp::aws::AwsAuthenticationSigner::HttpMethod method -) -> string { - switch (method) { - case clp::aws::AwsAuthenticationSigner::HttpMethod::GET: - return "GET"; - case clp::aws::AwsAuthenticationSigner::HttpMethod::PUT: - return "PUT"; - case clp::aws::AwsAuthenticationSigner::HttpMethod::POST: - return "POST"; - case clp::aws::AwsAuthenticationSigner::HttpMethod::DELETE: - return "DELETE"; - default: - throw std::runtime_error("Invalid HTTP method"); - } -} - -/** - * Gets the string to sign specified by AWS Signature Version 4 format - * @param scope - * @param timestamp_string - * @param canonical_request - * @param string_to_sign Returns the string to sign - * @return clp::ErrorCode_Success on success - * On failure, same as clp::get_sha256_hash - */ -[[nodiscard]] auto get_string_to_sign( - string_view scope, - string_view timestamp_string, - string_view canonical_request, - string& string_to_sign -) -> clp::ErrorCode { - vector signed_canonical_request; - if (auto error_code = clp::get_sha256_hash( - {size_checked_pointer_cast(canonical_request.data()), - canonical_request.size()}, - signed_canonical_request - ); - clp::ErrorCode_Success != error_code) - { - return error_code; - } - auto const signed_canonical_request_str = clp::convert_hash_to_hex_string( - {signed_canonical_request.data(), signed_canonical_request.size()} - ); - string_to_sign = fmt::format( - "{}\n{}\n{}\n{}", - clp::aws::cAws4HmacSha256, - timestamp_string, - scope, - signed_canonical_request_str - ); - return clp::ErrorCode_Success; -} - -/** - * Encode the uri as specified by AWS Signature Version 4's UriEncode() - * @param uri - * @param encode_slash - * @return The encoded URI - */ -[[nodiscard]] auto encode_uri(string_view uri, bool encode_slash = true) -> string { - string encoded_uri; - - for (auto const c : uri) { - if ((std::isalnum(c) != 0) || c == '-' || c == '_' || c == '.' || c == '~') { - encoded_uri += c; - } else if (c == '/' && false == encode_slash) { - encoded_uri += c; - } else { - encoded_uri += fmt::format("%{:02X}", c); - } - } - - return encoded_uri; -} - -/** - * Gets the scope specified by AWS Signature Version 4 format - * @param date_string - * @param region - * @return The scope as a string - */ -[[nodiscard]] auto get_scope(string_view date_string, string_view region) -> string { - return fmt::format( - "{}/{}/{}/{}", - date_string, - region, - clp::aws::cS3Service, - clp::aws::cAws4Request - ); -} - -/** - * Gets the canonical request string - * @param method HTTP method - * @param url S3 URL - * @param query_string Query string - * @return Canonical request - */ -[[nodiscard]] auto get_canonical_request( - clp::aws::AwsAuthenticationSigner::HttpMethod method, - clp::aws::S3Url const& url, - string_view query_string -) -> string { - return fmt::format( - "{}\n{}\n{}\n{}:{}\n\n{}\n{}", - get_method_string(method), - encode_uri(url.get_path(), false), - query_string, - clp::aws::cDefaultSignedHeaders, - url.get_host(), - clp::aws::cDefaultSignedHeaders, - clp::aws::cUnsignedPayload - ); -} -} // namespace - -namespace clp::aws { -S3Url::S3Url(string const& url) { - // Regular expression to match Virtual-hosted-style HTTP URL format - std::regex const host_style_url_regex( - R"(https://([a-z0-9.-]+)\.s3(\.([a-z0-9-]+))?\.amazonaws\.com(/[^?]+).*)" - ); - // Regular expression to match Path-style HTTP URL format - std::regex const path_style_url_regex( - R"(https://s3(\.([a-z0-9-]+))?\.amazonaws\.com/([a-z0-9.-]+)(/[^?]+).*)" - ); - - std::smatch match; - if (std::regex_match(url, match, host_style_url_regex)) { - m_bucket = match[1].str(); - m_region = match[3].str(); - m_path = match[4].str(); - } else if (std::regex_match(url, match, path_style_url_regex)) { - m_region = match[2].str(); - m_bucket = match[3].str(); - m_path = match[4].str(); - } else { - throw OperationFailed( - ErrorCode_BadParam, - __FILENAME__, - __LINE__, - "Invalid S3 HTTP URL format" - ); - } - - if (m_region.empty()) { - m_region = cDefaultRegion; - } - m_host = fmt::format("{}.s3.{}.amazonaws.com", m_bucket, m_region); -} - -auto S3Url::get_compression_path() const -> string { - return fmt::format("/{}/{}{}", m_region, m_bucket, m_path); -} - -auto AwsAuthenticationSigner::generate_presigned_url(S3Url const& s3_url, string& presigned_url) - -> ErrorCode { - auto const s3_region = s3_url.get_region(); - - // Gets current time - auto const now = std::chrono::system_clock::now(); - auto const date_string = get_date_string(now); - auto const timestamp_string = get_timestamp_string(now); - - auto scope = get_scope(date_string, s3_region); - auto canonical_query_string = generate_canonical_query_string(scope, timestamp_string); - - auto canonical_request = get_canonical_request(HttpMethod::GET, s3_url, canonical_query_string); - - string string_to_sign{}; - if (auto error_code - = get_string_to_sign(scope, timestamp_string, canonical_request, string_to_sign); - ErrorCode_Success != error_code) - { - return error_code; - } - - vector signature{}; - if (auto error_code = get_signature(s3_region, date_string, string_to_sign, signature); - ErrorCode_Success != error_code) - { - return error_code; - } - auto const signature_str = convert_hash_to_hex_string({signature.data(), signature.size()}); - - presigned_url = fmt::format( - "https://{}{}?{}&{}={}", - s3_url.get_host(), - s3_url.get_path(), - canonical_query_string, - cXAmzSignature, - signature_str - ); - return ErrorCode_Success; -} - -auto AwsAuthenticationSigner::get_signature( - string_view region, - string_view date_string, - string_view string_to_sign, - vector& signature -) -> ErrorCode { - vector signing_key{}; - if (auto error_code = get_signing_key(region, date_string, signing_key); - ErrorCode_Success != error_code) - { - return error_code; - } - - if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(string_to_sign.data()), - string_to_sign.size()}, - {size_checked_pointer_cast(signing_key.data()), - signing_key.size()}, - signature - ); - ErrorCode_Success != error_code) - { - return error_code; - } - return ErrorCode_Success; -} - -auto AwsAuthenticationSigner::get_signing_key( - string_view region, - string_view date_string, - vector& signing_key -) -> ErrorCode { - string key{cAws4}; - key += m_secret_access_key; - - vector date_key{}; - vector date_region_key{}; - vector date_region_service_key{}; - if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(date_string.data()), - date_string.size()}, - {size_checked_pointer_cast(key.data()), key.size()}, - date_key - ); - error_code != ErrorCode_Success) - { - return error_code; - } - - if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(region.data()), region.size()}, - {size_checked_pointer_cast(date_key.data()), date_key.size()}, - date_region_key - ); - error_code != ErrorCode_Success) - { - return error_code; - } - - if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(cS3Service.data()), - cS3Service.size()}, - {size_checked_pointer_cast(date_region_key.data()), - date_region_key.size()}, - date_region_service_key - ); - error_code != ErrorCode_Success) - { - return error_code; - } - - if (auto error_code = get_hmac_sha256_hash( - {size_checked_pointer_cast(cAws4Request.data()), - cAws4Request.size()}, - {size_checked_pointer_cast(date_region_service_key.data()), - date_region_service_key.size()}, - signing_key - ); - error_code != ErrorCode_Success) - { - return error_code; - } - - return ErrorCode_Success; -} - -auto AwsAuthenticationSigner::generate_canonical_query_string( - string_view scope, - string_view timestamp_string -) -> string { - string uri{m_access_key_id + "/"}; - uri += scope; - return fmt::format( - "{}={}&{}={}&{}={}&{}={}&{}={}", - cXAmzAlgorithm, - cAws4HmacSha256, - cXAmzCredential, - encode_uri(uri), - cXAmzDate, - timestamp_string, - cXAmzExpires, - cDefaultExpireTime, - cXAmzSignedHeaders, - cDefaultSignedHeaders - ); -} - -} // namespace clp::aws diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp deleted file mode 100644 index 244711785..000000000 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP -#define CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP - -#include -#include -#include -#include - -#include -#include - -#include "../ErrorCode.hpp" -#include "../TraceableException.hpp" -#include "Constants.hpp" - -namespace clp::aws { -/** - * Class for parsing S3 URL. The format of S3 URL is specie - */ -class S3Url { -public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : OperationFailed(error_code, filename, line_number, "S3Url operation failed") {} - - OperationFailed( - ErrorCode error_code, - char const* const filename, - int line_number, - std::string message - ) - : TraceableException(error_code, filename, line_number), - m_message(std::move(message)) {} - - // Methods - [[nodiscard]] auto what() const noexcept -> char const* override { - return m_message.c_str(); - } - - private: - std::string m_message; - }; - - // Constructor - S3Url(std::string const& url); - - // Methods - [[nodiscard]] auto get_host() const -> std::string_view { return m_host; } - - [[nodiscard]] auto get_region() const -> std::string_view { return m_region; } - - [[nodiscard]] auto get_path() const -> std::string_view { return m_path; } - - [[nodiscard]] auto get_compression_path() const -> std::string; - -private: - std::string m_host; - std::string m_bucket; - std::string m_region; - std::string m_path; -}; - -/** - * Class for signing AWS requests. The class is based on AWS Signature Version 4. - * A detailed explanation on how the signature is generated can be found at - * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html - */ -class AwsAuthenticationSigner { -public: - // Default expire time of presigned URL in seconds - static constexpr int cDefaultExpireTime = 86'400; // 24 hours - - // Types - enum class HttpMethod : uint8_t { - GET, - PUT, - POST, - DELETE - }; - - // Constructors - AwsAuthenticationSigner(std::string_view access_key_id, std::string_view secret_access_key) - : m_access_key_id{access_key_id}, - m_secret_access_key{secret_access_key} {} - - // Methods - /** - * Generates a presigned URL using AWS Signature Version 4 - * @param s3_url S3 URL - * @param presigned_url Returns the generated presigned url - * @param method HTTP method - * @return ErrorCode_Success on success. - * On failure, same as get_string_to_sign and AwsAuthenticationSigner::get_signature - * - */ - [[nodiscard]] auto - generate_presigned_url(S3Url const& s3_url, std::string& presigned_url) -> ErrorCode; - -private: - /** - * Generates the canonical query string - * @param scope - * @param timestamp_string - * @return the canonical query string - */ - [[nodiscard]] auto generate_canonical_query_string( - std::string_view scope, - std::string_view timestamp_string - ) -> std::string; - - /** - * Gets the signature signing key for the request - * @param region - * @param date_string - * @param signing_key Returns the signing key - * @return ErrorCode_Success on success. - * Same as get_hmac_sha256_hash on Failure. - */ - [[nodiscard]] auto get_signing_key( - std::string_view region, - std::string_view date_string, - std::vector& signing_key - ) -> ErrorCode; - - /** - * Signs the string_to_sign and returns the request signature by reference - * @param region - * @param date_string - * @param string_to_sign StringToSign specified by the AWS Signature Version 4 - * @param signature Returns the signature - * @return ErrorCode_Success on success. - * Same as get_hmac_sha256_hash on Failure. - */ - [[nodiscard]] ErrorCode get_signature( - std::string_view region, - std::string_view date_string, - std::string_view string_to_sign, - std::vector& signature - ); - - // Variables - std::string m_access_key_id; - std::string m_secret_access_key; -}; -} // namespace clp::aws - -#endif // CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP diff --git a/components/core/src/clp/aws/Constants.hpp b/components/core/src/clp/aws/Constants.hpp deleted file mode 100644 index dd5032df7..000000000 --- a/components/core/src/clp/aws/Constants.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef CLP_AWS_CONSTANTS_HPP -#define CLP_AWS_CONSTANTS_HPP - -#include - -namespace clp::aws { -// Query String Parameter Names -constexpr std::string_view cXAmzAlgorithm{"X-Amz-Algorithm"}; -constexpr std::string_view cXAmzCredential{"X-Amz-Credential"}; -constexpr std::string_view cXAmzDate{"X-Amz-Date"}; -constexpr std::string_view cXAmzExpires{"X-Amz-Expires"}; -constexpr std::string_view cXAmzSignature{"X-Amz-Signature"}; -constexpr std::string_view cXAmzSignedHeaders{"X-Amz-SignedHeaders"}; - -// Other Constants -constexpr std::string_view cAws4{"AWS4"}; -constexpr std::string_view cAws4Request{"aws4_request"}; -constexpr std::string_view cAws4HmacSha256{"AWS4-HMAC-SHA256"}; -constexpr std::string_view cDefaultSignedHeaders{"host"}; -constexpr std::string_view cDefaultRegion{"us-east-1"}; -constexpr std::string_view cS3Service{"s3"}; -constexpr std::string_view cUnsignedPayload{"UNSIGNED-PAYLOAD"}; -} // namespace clp::aws - -#endif // CLP_AWS_CONSTANTS_HPP From 2e582001da833ad9451efdd0df2f9da6c2c7a70a Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:55:53 -0400 Subject: [PATCH 30/55] Include header properly --- components/core/src/clp/hash_utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index fdfcb6552..d6ac399b4 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -7,7 +7,7 @@ #include #include -#include +#include #include "ErrorCode.hpp" #include "TraceableException.hpp" From 2cfeee04f75551714a53a78e81656b0fbba902a5 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 17:59:16 -0400 Subject: [PATCH 31/55] try a fix --- components/core/src/clp/hash_utils.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index d6ac399b4..0e1e4f422 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -33,7 +33,7 @@ class EvpCtxManager { }; // Constructors - EvpCtxManager() : m_md_ctx{EVP_MD_CTX_new()} { + EvpCtxManager() : m_md_ctx{EVP_MD_CTX_create()} { if (nullptr == m_md_ctx) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -44,7 +44,7 @@ class EvpCtxManager { auto operator=(EvpCtxManager const&) -> EvpCtxManager& = delete; // Destructor - ~EvpCtxManager() { EVP_MD_CTX_free(m_md_ctx); } + ~EvpCtxManager() { EVP_MD_CTX_destroy(m_md_ctx); } auto digest_init_ex(const EVP_MD* type, ENGINE* impl) -> int { return EVP_DigestInit_ex(m_md_ctx, type, impl); From 8c41959953debd5a490427f50923e6fad917283b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:21:05 -0400 Subject: [PATCH 32/55] try another fix for macos --- components/core/CMakeLists.txt | 9 +++++++++ .../tools/scripts/lib_install/macos-12/install-all.sh | 1 + 2 files changed, 10 insertions(+) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 6080e32f1..f8b391b1d 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -146,6 +146,14 @@ else() message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for CURL") endif() +# Find OpenSSL +find_package(OpenSSL REQUIRED) +if (OPENSSL_FOUND) + message(STATUS "Found OpenSSL (${OPENSSL_VERSION})") +else () + message(FATAL_ERROR "OpenSSL not found") +endif () + # Add log surgeon add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL) @@ -500,6 +508,7 @@ target_link_libraries(unitTest LibArchive::LibArchive MariaDBClient::MariaDBClient spdlog::spdlog + OpenSSL::SSL OpenSSL::Crypto ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} clp::regex_utils diff --git a/components/core/tools/scripts/lib_install/macos-12/install-all.sh b/components/core/tools/scripts/lib_install/macos-12/install-all.sh index c13223ba6..ebac5314b 100755 --- a/components/core/tools/scripts/lib_install/macos-12/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos-12/install-all.sh @@ -19,5 +19,6 @@ brew install \ mongo-cxx-driver \ msgpack-cxx \ spdlog \ + openssl \ pkg-config \ zstd From d67e7ab2f135f8ef778969bec84477b8cdd322b9 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 21:34:06 -0400 Subject: [PATCH 33/55] remove unnecessary install --- .../core/tools/scripts/lib_install/macos-12/install-all.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/macos-12/install-all.sh b/components/core/tools/scripts/lib_install/macos-12/install-all.sh index ebac5314b..c13223ba6 100755 --- a/components/core/tools/scripts/lib_install/macos-12/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos-12/install-all.sh @@ -19,6 +19,5 @@ brew install \ mongo-cxx-driver \ msgpack-cxx \ spdlog \ - openssl \ pkg-config \ zstd From 6e5bf52ca624ec94b022efbfeb03a23c353cea13 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 21:54:51 -0400 Subject: [PATCH 34/55] Maybe I Only Need crypto --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index f8b391b1d..cc56a8e09 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -508,7 +508,7 @@ target_link_libraries(unitTest LibArchive::LibArchive MariaDBClient::MariaDBClient spdlog::spdlog - OpenSSL::SSL OpenSSL::Crypto + OpenSSL::Crypto ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} clp::regex_utils From 4b3338ca3add8f037306c59b295273a3e044bf2b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 22:25:54 -0400 Subject: [PATCH 35/55] Remove openssl makefile --- .../core/cmake/Modules/FindOpenSSL.cmake | 534 ------------------ 1 file changed, 534 deletions(-) delete mode 100644 components/core/cmake/Modules/FindOpenSSL.cmake diff --git a/components/core/cmake/Modules/FindOpenSSL.cmake b/components/core/cmake/Modules/FindOpenSSL.cmake deleted file mode 100644 index 79c3b044b..000000000 --- a/components/core/cmake/Modules/FindOpenSSL.cmake +++ /dev/null @@ -1,534 +0,0 @@ -# NOTE: This is FindOpenSSL.cmake from cmake-3.16.0-rc3. It fixes issues with missing dependencies for the OpenSSL static library. -# This file should be deleted when cmake-3.16 is released. - -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -#[=======================================================================[.rst: -FindOpenSSL ------------ - -Find the OpenSSL encryption library. - -Optional COMPONENTS -^^^^^^^^^^^^^^^^^^^ - -This module supports two optional COMPONENTS: ``Crypto`` and ``SSL``. Both -components have associated imported targets, as described below. - -Imported Targets -^^^^^^^^^^^^^^^^ - -This module defines the following :prop_tgt:`IMPORTED` targets: - -``OpenSSL::SSL`` - The OpenSSL ``ssl`` library, if found. -``OpenSSL::Crypto`` - The OpenSSL ``crypto`` library, if found. - -Result Variables -^^^^^^^^^^^^^^^^ - -This module will set the following variables in your project: - -``OPENSSL_FOUND`` - System has the OpenSSL library. If no components are requested it only - requires the crypto library. -``OPENSSL_INCLUDE_DIR`` - The OpenSSL include directory. -``OPENSSL_CRYPTO_LIBRARY`` - The OpenSSL crypto library. -``OPENSSL_CRYPTO_LIBRARIES`` - The OpenSSL crypto library and its dependencies. -``OPENSSL_SSL_LIBRARY`` - The OpenSSL SSL library. -``OPENSSL_SSL_LIBRARIES`` - The OpenSSL SSL library and its dependencies. -``OPENSSL_LIBRARIES`` - All OpenSSL libraries and their dependencies. -``OPENSSL_VERSION`` - This is set to ``$major.$minor.$revision$patch`` (e.g. ``0.9.8s``). - -Hints -^^^^^ - -Set ``OPENSSL_ROOT_DIR`` to the root directory of an OpenSSL installation. -Set ``OPENSSL_USE_STATIC_LIBS`` to ``TRUE`` to look for static libraries. -Set ``OPENSSL_MSVC_STATIC_RT`` set ``TRUE`` to choose the MT version of the lib. -#]=======================================================================] - -macro(_OpenSSL_test_and_find_dependencies ssl_library crypto_library) - if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND - (("${ssl_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$") OR - ("${crypto_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$"))) - set(_OpenSSL_has_dependencies TRUE) - find_package(Threads) - else() - set(_OpenSSL_has_dependencies FALSE) - endif() -endmacro() - -function(_OpenSSL_add_dependencies libraries_var library) - if(CMAKE_THREAD_LIBS_INIT) - list(APPEND ${libraries_var} ${CMAKE_THREAD_LIBS_INIT}) - endif() - list(APPEND ${libraries_var} ${CMAKE_DL_LIBS}) - set(${libraries_var} ${${libraries_var}} PARENT_SCOPE) -endfunction() - -function(_OpenSSL_target_add_dependencies target) - if(_OpenSSL_has_dependencies) - set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads ) - set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS} ) - endif() -endfunction() - -if (UNIX) - find_package(PkgConfig QUIET) - pkg_check_modules(_OPENSSL QUIET openssl) -endif () - -# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES -if(OPENSSL_USE_STATIC_LIBS) - set(_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a ) - endif() -endif() - -if (WIN32) - # http://www.slproweb.com/products/Win32OpenSSL.html - set(_OPENSSL_ROOT_HINTS - ${OPENSSL_ROOT_DIR} - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (32-bit)_is1;Inno Setup: App Path]" - "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (64-bit)_is1;Inno Setup: App Path]" - ENV OPENSSL_ROOT_DIR - ) - file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles) - set(_OPENSSL_ROOT_PATHS - "${_programfiles}/OpenSSL" - "${_programfiles}/OpenSSL-Win32" - "${_programfiles}/OpenSSL-Win64" - "C:/OpenSSL/" - "C:/OpenSSL-Win32/" - "C:/OpenSSL-Win64/" - ) - unset(_programfiles) -else () - set(_OPENSSL_ROOT_HINTS - ${OPENSSL_ROOT_DIR} - ENV OPENSSL_ROOT_DIR - ) -endif () - -set(_OPENSSL_ROOT_HINTS_AND_PATHS - HINTS ${_OPENSSL_ROOT_HINTS} - PATHS ${_OPENSSL_ROOT_PATHS} - ) - -find_path(OPENSSL_INCLUDE_DIR - NAMES - openssl/ssl.h - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - HINTS - ${_OPENSSL_INCLUDEDIR} - PATH_SUFFIXES - include -) - -if(WIN32 AND NOT CYGWIN) - if(MSVC) - # /MD and /MDd are the standard values - if someone wants to use - # others, the libnames have to change here too - # use also ssl and ssleay32 in debug as fallback for openssl < 0.9.8b - # enable OPENSSL_MSVC_STATIC_RT to get the libs build /MT (Multithreaded no-DLL) - # In Visual C++ naming convention each of these four kinds of Windows libraries has it's standard suffix: - # * MD for dynamic-release - # * MDd for dynamic-debug - # * MT for static-release - # * MTd for static-debug - - # Implementation details: - # We are using the libraries located in the VC subdir instead of the parent directory even though : - # libeay32MD.lib is identical to ../libeay32.lib, and - # ssleay32MD.lib is identical to ../ssleay32.lib - # enable OPENSSL_USE_STATIC_LIBS to use the static libs located in lib/VC/static - - if (OPENSSL_MSVC_STATIC_RT) - set(_OPENSSL_MSVC_RT_MODE "MT") - else () - set(_OPENSSL_MSVC_RT_MODE "MD") - endif () - - # Since OpenSSL 1.1, lib names are like libcrypto32MTd.lib and libssl32MTd.lib - if( "${CMAKE_SIZEOF_VOID_P}" STREQUAL "8" ) - set(_OPENSSL_MSVC_ARCH_SUFFIX "64") - else() - set(_OPENSSL_MSVC_ARCH_SUFFIX "32") - endif() - - if(OPENSSL_USE_STATIC_LIBS) - set(_OPENSSL_PATH_SUFFIXES - "lib/VC/static" - "VC/static" - "lib" - ) - else() - set(_OPENSSL_PATH_SUFFIXES - "lib/VC" - "VC" - "lib" - ) - endif () - - find_library(LIB_EAY_DEBUG - NAMES - libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d - libcrypto${_OPENSSL_MSVC_RT_MODE}d - libcryptod - libeay32${_OPENSSL_MSVC_RT_MODE}d - libeay32d - cryptod - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - ${_OPENSSL_PATH_SUFFIXES} - ) - - find_library(LIB_EAY_RELEASE - NAMES - libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE} - libcrypto${_OPENSSL_MSVC_RT_MODE} - libcrypto - libeay32${_OPENSSL_MSVC_RT_MODE} - libeay32 - crypto - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - ${_OPENSSL_PATH_SUFFIXES} - ) - - find_library(SSL_EAY_DEBUG - NAMES - libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d - libssl${_OPENSSL_MSVC_RT_MODE}d - libssld - ssleay32${_OPENSSL_MSVC_RT_MODE}d - ssleay32d - ssld - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - ${_OPENSSL_PATH_SUFFIXES} - ) - - find_library(SSL_EAY_RELEASE - NAMES - libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE} - libssl${_OPENSSL_MSVC_RT_MODE} - libssl - ssleay32${_OPENSSL_MSVC_RT_MODE} - ssleay32 - ssl - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - ${_OPENSSL_PATH_SUFFIXES} - ) - - set(LIB_EAY_LIBRARY_DEBUG "${LIB_EAY_DEBUG}") - set(LIB_EAY_LIBRARY_RELEASE "${LIB_EAY_RELEASE}") - set(SSL_EAY_LIBRARY_DEBUG "${SSL_EAY_DEBUG}") - set(SSL_EAY_LIBRARY_RELEASE "${SSL_EAY_RELEASE}") - - include(SelectLibraryConfigurations) - select_library_configurations(LIB_EAY) - select_library_configurations(SSL_EAY) - - mark_as_advanced(LIB_EAY_LIBRARY_DEBUG LIB_EAY_LIBRARY_RELEASE - SSL_EAY_LIBRARY_DEBUG SSL_EAY_LIBRARY_RELEASE) - set(OPENSSL_SSL_LIBRARY ${SSL_EAY_LIBRARY} ) - set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY_LIBRARY} ) - elseif(MINGW) - # same player, for MinGW - set(LIB_EAY_NAMES crypto libeay32) - set(SSL_EAY_NAMES ssl ssleay32) - find_library(LIB_EAY - NAMES - ${LIB_EAY_NAMES} - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - "lib/MinGW" - "lib" - ) - - find_library(SSL_EAY - NAMES - ${SSL_EAY_NAMES} - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - PATH_SUFFIXES - "lib/MinGW" - "lib" - ) - - mark_as_advanced(SSL_EAY LIB_EAY) - set(OPENSSL_SSL_LIBRARY ${SSL_EAY} ) - set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} ) - unset(LIB_EAY_NAMES) - unset(SSL_EAY_NAMES) - else() - # Not sure what to pick for -say- intel, let's use the toplevel ones and hope someone report issues: - find_library(LIB_EAY - NAMES - libcrypto - libeay32 - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - HINTS - ${_OPENSSL_LIBDIR} - PATH_SUFFIXES - lib - ) - - find_library(SSL_EAY - NAMES - libssl - ssleay32 - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - HINTS - ${_OPENSSL_LIBDIR} - PATH_SUFFIXES - lib - ) - - mark_as_advanced(SSL_EAY LIB_EAY) - set(OPENSSL_SSL_LIBRARY ${SSL_EAY} ) - set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} ) - endif() -else() - - find_library(OPENSSL_SSL_LIBRARY - NAMES - ssl - ssleay32 - ssleay32MD - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - HINTS - ${_OPENSSL_LIBDIR} - PATH_SUFFIXES - lib - ) - - find_library(OPENSSL_CRYPTO_LIBRARY - NAMES - crypto - NAMES_PER_DIR - ${_OPENSSL_ROOT_HINTS_AND_PATHS} - HINTS - ${_OPENSSL_LIBDIR} - PATH_SUFFIXES - lib - ) - - mark_as_advanced(OPENSSL_CRYPTO_LIBRARY OPENSSL_SSL_LIBRARY) - -endif() - -# compat defines -set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY}) -set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY}) -_OpenSSL_test_and_find_dependencies("${OPENSSL_SSL_LIBRARY}" "${OPENSSL_CRYPTO_LIBRARY}") -if(_OpenSSL_has_dependencies) - _OpenSSL_add_dependencies( OPENSSL_SSL_LIBRARIES "${OPENSSL_SSL_LIBRARY}" ) - _OpenSSL_add_dependencies( OPENSSL_CRYPTO_LIBRARIES "${OPENSSL_CRYPTO_LIBRARY}" ) -endif() - -function(from_hex HEX DEC) - string(TOUPPER "${HEX}" HEX) - set(_res 0) - string(LENGTH "${HEX}" _strlen) - - while (_strlen GREATER 0) - math(EXPR _res "${_res} * 16") - string(SUBSTRING "${HEX}" 0 1 NIBBLE) - string(SUBSTRING "${HEX}" 1 -1 HEX) - if (NIBBLE STREQUAL "A") - math(EXPR _res "${_res} + 10") - elseif (NIBBLE STREQUAL "B") - math(EXPR _res "${_res} + 11") - elseif (NIBBLE STREQUAL "C") - math(EXPR _res "${_res} + 12") - elseif (NIBBLE STREQUAL "D") - math(EXPR _res "${_res} + 13") - elseif (NIBBLE STREQUAL "E") - math(EXPR _res "${_res} + 14") - elseif (NIBBLE STREQUAL "F") - math(EXPR _res "${_res} + 15") - else() - math(EXPR _res "${_res} + ${NIBBLE}") - endif() - - string(LENGTH "${HEX}" _strlen) - endwhile() - - set(${DEC} ${_res} PARENT_SCOPE) -endfunction() - -if(OPENSSL_INCLUDE_DIR AND EXISTS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h") - file(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h" openssl_version_str - REGEX "^#[\t ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])+.*") - - if(openssl_version_str) - # The version number is encoded as 0xMNNFFPPS: major minor fix patch status - # The status gives if this is a developer or prerelease and is ignored here. - # Major, minor, and fix directly translate into the version numbers shown in - # the string. The patch field translates to the single character suffix that - # indicates the bug fix state, which 00 -> nothing, 01 -> a, 02 -> b and so - # on. - - string(REGEX REPLACE "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F]).*$" - "\\1;\\2;\\3;\\4;\\5" OPENSSL_VERSION_LIST "${openssl_version_str}") - list(GET OPENSSL_VERSION_LIST 0 OPENSSL_VERSION_MAJOR) - list(GET OPENSSL_VERSION_LIST 1 OPENSSL_VERSION_MINOR) - from_hex("${OPENSSL_VERSION_MINOR}" OPENSSL_VERSION_MINOR) - list(GET OPENSSL_VERSION_LIST 2 OPENSSL_VERSION_FIX) - from_hex("${OPENSSL_VERSION_FIX}" OPENSSL_VERSION_FIX) - list(GET OPENSSL_VERSION_LIST 3 OPENSSL_VERSION_PATCH) - - if (NOT OPENSSL_VERSION_PATCH STREQUAL "00") - from_hex("${OPENSSL_VERSION_PATCH}" _tmp) - # 96 is the ASCII code of 'a' minus 1 - math(EXPR OPENSSL_VERSION_PATCH_ASCII "${_tmp} + 96") - unset(_tmp) - # Once anyone knows how OpenSSL would call the patch versions beyond 'z' - # this should be updated to handle that, too. This has not happened yet - # so it is simply ignored here for now. - string(ASCII "${OPENSSL_VERSION_PATCH_ASCII}" OPENSSL_VERSION_PATCH_STRING) - endif () - - set(OPENSSL_VERSION "${OPENSSL_VERSION_MAJOR}.${OPENSSL_VERSION_MINOR}.${OPENSSL_VERSION_FIX}${OPENSSL_VERSION_PATCH_STRING}") - endif () -endif () - -set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES} ) -list(REMOVE_DUPLICATES OPENSSL_LIBRARIES) - -foreach(_comp IN LISTS OpenSSL_FIND_COMPONENTS) - if(_comp STREQUAL "Crypto") - if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND - (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR - EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR - EXISTS "${LIB_EAY_LIBRARY_RELEASE}") - ) - set(OpenSSL_${_comp}_FOUND TRUE) - else() - set(OpenSSL_${_comp}_FOUND FALSE) - endif() - elseif(_comp STREQUAL "SSL") - if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND - (EXISTS "${OPENSSL_SSL_LIBRARY}" OR - EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR - EXISTS "${SSL_EAY_LIBRARY_RELEASE}") - ) - set(OpenSSL_${_comp}_FOUND TRUE) - else() - set(OpenSSL_${_comp}_FOUND FALSE) - endif() - else() - message(WARNING "${_comp} is not a valid OpenSSL component") - set(OpenSSL_${_comp}_FOUND FALSE) - endif() -endforeach() -unset(_comp) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(OpenSSL - REQUIRED_VARS - OPENSSL_CRYPTO_LIBRARY - OPENSSL_INCLUDE_DIR - VERSION_VAR - OPENSSL_VERSION - HANDLE_COMPONENTS - FAIL_MESSAGE - "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR" -) - -mark_as_advanced(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES) - -if(OPENSSL_FOUND) - if(NOT TARGET OpenSSL::Crypto AND - (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR - EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR - EXISTS "${LIB_EAY_LIBRARY_RELEASE}") - ) - add_library(OpenSSL::Crypto UNKNOWN IMPORTED) - set_target_properties(OpenSSL::Crypto PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - if(EXISTS "${OPENSSL_CRYPTO_LIBRARY}") - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}") - endif() - if(EXISTS "${LIB_EAY_LIBRARY_RELEASE}") - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C" - IMPORTED_LOCATION_RELEASE "${LIB_EAY_LIBRARY_RELEASE}") - endif() - if(EXISTS "${LIB_EAY_LIBRARY_DEBUG}") - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C" - IMPORTED_LOCATION_DEBUG "${LIB_EAY_LIBRARY_DEBUG}") - endif() - _OpenSSL_target_add_dependencies(OpenSSL::Crypto) - endif() - - if(NOT TARGET OpenSSL::SSL AND - (EXISTS "${OPENSSL_SSL_LIBRARY}" OR - EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR - EXISTS "${SSL_EAY_LIBRARY_RELEASE}") - ) - add_library(OpenSSL::SSL UNKNOWN IMPORTED) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - if(EXISTS "${OPENSSL_SSL_LIBRARY}") - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}") - endif() - if(EXISTS "${SSL_EAY_LIBRARY_RELEASE}") - set_property(TARGET OpenSSL::SSL APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C" - IMPORTED_LOCATION_RELEASE "${SSL_EAY_LIBRARY_RELEASE}") - endif() - if(EXISTS "${SSL_EAY_LIBRARY_DEBUG}") - set_property(TARGET OpenSSL::SSL APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C" - IMPORTED_LOCATION_DEBUG "${SSL_EAY_LIBRARY_DEBUG}") - endif() - if(TARGET OpenSSL::Crypto) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) - endif() - _OpenSSL_target_add_dependencies(OpenSSL::SSL) - endif() -endif() - -# Restore the original find library ordering -if(OPENSSL_USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -endif() From 6f6687f1cdc33926ea57a9ddac075aa646ba52b1 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 24 Jul 2024 22:49:49 -0400 Subject: [PATCH 36/55] Update minimal cmake version --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index cc56a8e09..528d980fe 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5.1) +cmake_minimum_required(VERSION 3.16.3) project(CLP LANGUAGES CXX C) if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) From adf85e8f320bdd83ffb3ad0c47275fb17e578483 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:07:55 -0400 Subject: [PATCH 37/55] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- components/core/src/clp/hash_utils.cpp | 21 ++++++++--------- components/core/src/clp/hash_utils.hpp | 28 +++++++++++------------ components/core/tests/test-hash_utils.cpp | 1 + 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 6648572b6..16d33e902 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -40,17 +40,16 @@ auto get_hmac_sha256_hash( SPDLOG_ERROR("Input key exceeds maximum length"); return ErrorCode_BadParam; } - int const key_length{static_cast(key.size())}; - auto* res - = HMAC(EVP_sha256(), - key.data(), - key_length, - input.data(), - input.size(), - hash.data(), - &hash_length); - - if (nullptr == res) { + auto const key_length{static_cast(key.size())}; + if (nullptr + == HMAC(EVP_sha256(), + key.data(), + key_length, + input.data(), + input.size(), + hash.data(), + &hash_length)) + { SPDLOG_ERROR("Failed to get HMAC hashes"); return ErrorCode_Failure; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 0e1e4f422..8e3a6433c 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -1,10 +1,9 @@ #ifndef CLP_HASH_UTILS_HPP #define CLP_HASH_UTILS_HPP -#include +#include #include #include -#include #include #include @@ -14,8 +13,7 @@ namespace clp { /** - * A class the wraps around openssl EVP_MD_CTX to manage the life cycle of - * dynamically allocated EVP_MD_CTX object. +* A C++ wrapper for openssl's EVP digest message digest context (EVP_MD_CTX). */ class EvpCtxManager { public: @@ -45,12 +43,12 @@ class EvpCtxManager { // Destructor ~EvpCtxManager() { EVP_MD_CTX_destroy(m_md_ctx); } - +// Methods auto digest_init_ex(const EVP_MD* type, ENGINE* impl) -> int { return EVP_DigestInit_ex(m_md_ctx, type, impl); } - auto digest_update(void const* d, size_t cnt) -> int { + [[nodiscard]] auto digest_update(void const* d, size_t cnt) -> int { return EVP_DigestUpdate(m_md_ctx, d, cnt); } @@ -63,20 +61,20 @@ class EvpCtxManager { }; /** - * Converts input hash into a hex string + * Converts input hash into a hex string. * @param input * @return input hash as a hex string */ -auto convert_hash_to_hex_string(std::span input) -> std::string; +[[nodiscard]] auto convert_hash_to_hex_string(std::span input) -> std::string; /** - * Gets the HMAC-SHA256 hash of input with key + * Gets the HMAC-SHA256 hash of input with key. * @param input * @param key - * @param hash Returns the result hash by reference. + * @param hash Returns the hashing result. + * @return ErrorCode_Success on success. * @return ErrorCode_BadParam if input key exceeds maximum length. - * ErrorCode_Failure if hash generation fails - * ErrorCode_Success on success + * @return ErrorCode_Failure if hash generation fails. */ auto get_hmac_sha256_hash( std::span input, @@ -87,10 +85,10 @@ auto get_hmac_sha256_hash( /** * Gets the SHA256 hash of input * @param input - * @param hash Returns the result hash by reference. + * @param hash Returns the hashing result. + * @return ErrorCode_Success on success. * @return ErrorCode_BadParam if input key exceeds maximum length. - * ErrorCode_Failure if hash generation fails - * ErrorCode_Success on success + * @return ErrorCode_Failure if hash generation fails. * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initalized */ auto get_sha256_hash(std::span input, std::vector& hash) diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp index 6c32d1a95..0e9dbc43a 100644 --- a/components/core/tests/test-hash_utils.cpp +++ b/components/core/tests/test-hash_utils.cpp @@ -6,6 +6,7 @@ #include "../src/clp/ErrorCode.hpp" #include "../src/clp/hash_utils.hpp" #include "../src/clp/type_utils.hpp" + using clp::convert_hash_to_hex_string; using clp::ErrorCode_Success; using clp::get_hmac_sha256_hash; From 21087c9edbb3f025a386353ba31d43ec874c4774 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:02:07 -0400 Subject: [PATCH 38/55] address code review concern --- components/core/src/clp/hash_utils.cpp | 27 +++++++++----- components/core/src/clp/hash_utils.hpp | 44 ++++++++++++----------- components/core/tests/test-hash_utils.cpp | 42 +++++++++++----------- 3 files changed, 64 insertions(+), 49 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 16d33e902..8bd364359 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -20,6 +20,22 @@ using std::string_view; using std::vector; namespace clp { +auto EvpDigestContext::digest_update(std::span input) -> bool { + if (m_is_digest_finalized) { + throw OperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); + } + return static_cast(EVP_DigestUpdate(m_md_ctx, input.data(), input.size())); +} + +auto EvpDigestContext::digest_final_ex(std::vector& hash, unsigned int& length) + -> bool { + if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { + return false; + } + m_is_digest_finalized = true; + return true; +} + auto convert_hash_to_hex_string(span input) -> string { string hex_string; for (auto const c : input) { @@ -68,14 +84,9 @@ auto get_hmac_sha256_hash( * @return The SHA256 hash */ auto get_sha256_hash(span input, vector& hash) -> ErrorCode { - EvpCtxManager evp_ctx_manager{}; - - if (1 != evp_ctx_manager.digest_init_ex(EVP_sha256(), nullptr)) { - SPDLOG_ERROR("Failed to initialize ctx manager"); - return ErrorCode_Failure; - } + EvpDigestContext evp_ctx_manager{EVP_sha256(), nullptr}; - if (1 != evp_ctx_manager.digest_update(input.data(), input.size())) { + if (false == evp_ctx_manager.digest_update(input)) { SPDLOG_ERROR("Failed to digest input"); return ErrorCode_Failure; } @@ -83,7 +94,7 @@ auto get_sha256_hash(span input, vector& has unsigned int digest_len{0}; hash.resize(SHA256_DIGEST_LENGTH); - if (1 != evp_ctx_manager.digest_final_ex(hash.data(), &digest_len)) { + if (false == evp_ctx_manager.digest_final_ex(hash, digest_len)) { SPDLOG_ERROR("Failed to Finalize digest"); return ErrorCode_Failure; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 8e3a6433c..a36db4710 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -13,9 +13,9 @@ namespace clp { /** -* A C++ wrapper for openssl's EVP digest message digest context (EVP_MD_CTX). + * A C++ wrapper for openssl's EVP digest message digest context (EVP_MD_CTX). */ -class EvpCtxManager { +class EvpDigestContext { public: // Types class OperationFailed : public TraceableException { @@ -31,33 +31,35 @@ class EvpCtxManager { }; // Constructors - EvpCtxManager() : m_md_ctx{EVP_MD_CTX_create()} { + EvpDigestContext(const EVP_MD* type, ENGINE* impl) : m_md_ctx{EVP_MD_CTX_create()} { if (nullptr == m_md_ctx) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } + if (1 != EVP_DigestInit_ex(m_md_ctx, type, impl)) { + throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); + } } - // Disable copy and move constructor/assignment - EvpCtxManager(EvpCtxManager const&) = delete; - auto operator=(EvpCtxManager const&) -> EvpCtxManager& = delete; + // Disable copy constructor/assignment operator + EvpDigestContext(EvpDigestContext const&) = delete; + auto operator=(EvpDigestContext const&) -> EvpDigestContext& = delete; + + // disable move constructor/assignment operator + EvpDigestContext(EvpDigestContext&&) = delete; + auto operator=(EvpDigestContext&&) -> EvpDigestContext& = delete; // Destructor - ~EvpCtxManager() { EVP_MD_CTX_destroy(m_md_ctx); } -// Methods - auto digest_init_ex(const EVP_MD* type, ENGINE* impl) -> int { - return EVP_DigestInit_ex(m_md_ctx, type, impl); - } + ~EvpDigestContext() { EVP_MD_CTX_destroy(m_md_ctx); } - [[nodiscard]] auto digest_update(void const* d, size_t cnt) -> int { - return EVP_DigestUpdate(m_md_ctx, d, cnt); - } + // Methods + [[nodiscard]] auto digest_update(std::span input) -> bool; - auto digest_final_ex(unsigned char* md, unsigned int* s) -> int { - return EVP_DigestFinal_ex(m_md_ctx, md, s); - } + [[nodiscard]] auto + digest_final_ex(std::vector& hash, unsigned int& length) -> bool; private: EVP_MD_CTX* m_md_ctx{nullptr}; + bool m_is_digest_finalized{false}; }; /** @@ -76,7 +78,7 @@ class EvpCtxManager { * @return ErrorCode_BadParam if input key exceeds maximum length. * @return ErrorCode_Failure if hash generation fails. */ -auto get_hmac_sha256_hash( +[[nodiscard]] auto get_hmac_sha256_hash( std::span input, std::span key, std::vector& hash @@ -91,7 +93,9 @@ auto get_hmac_sha256_hash( * @return ErrorCode_Failure if hash generation fails. * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initalized */ -auto get_sha256_hash(std::span input, std::vector& hash) - -> ErrorCode; +[[nodiscard]] auto get_sha256_hash( + std::span input, + std::vector& hash +) -> ErrorCode; } // namespace clp #endif // CLP_HASH_UTILS_HPP diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp index 0e9dbc43a..8d0b11e0b 100644 --- a/components/core/tests/test-hash_utils.cpp +++ b/components/core/tests/test-hash_utils.cpp @@ -16,36 +16,36 @@ using std::string_view; using std::vector; TEST_CASE("test_sha256", "[hash_utils]") { - constexpr string_view input_string{"ThisIsARandomTestInput"}; - constexpr string_view reference_sha256{ + constexpr string_view cInputString{"ThisIsARandomTestInput"}; + constexpr string_view cReferenceSha256{ "c3a1d9f04ada1198c4c63bf51d9933fc2cc216429275cadabdcb2178775add0c" }; vector hash{}; - CHECK(ErrorCode_Success - == get_sha256_hash( - {size_checked_pointer_cast(input_string.data()), - input_string.size()}, - hash - )); - CHECK(convert_hash_to_hex_string(hash) == reference_sha256); + REQUIRE(ErrorCode_Success + == get_sha256_hash( + {size_checked_pointer_cast(cInputString.data()), + cInputString.size()}, + hash + )); + REQUIRE(convert_hash_to_hex_string(hash) == cReferenceSha256); } TEST_CASE("test_hmac", "[hash_utils]") { - string_view input_string{"ThisIsARandomTestInput"}; - string_view input_key{"ThisIsATestKey"}; - string_view reference_hmac_sha256{ + constexpr string_view cInputString{"ThisIsARandomTestInput"}; + constexpr string_view cInputKey{"ThisIsATestKey"}; + constexpr string_view cReferenceHmacSha256{ "38373057694c1038a6895212bea46849eb7a59b73a2ec175883ae095fb91ffda" }; vector hmac_hash{}; - CHECK(ErrorCode_Success - == get_hmac_sha256_hash( - {size_checked_pointer_cast(input_string.data()), - input_string.size()}, - {size_checked_pointer_cast(input_key.data()), - input_key.size()}, - hmac_hash - )); - CHECK(convert_hash_to_hex_string(hmac_hash) == reference_hmac_sha256); + REQUIRE(ErrorCode_Success + == get_hmac_sha256_hash( + {size_checked_pointer_cast(cInputString.data()), + cInputString.size()}, + {size_checked_pointer_cast(cInputKey.data()), + cInputKey.size()}, + hmac_hash + )); + REQUIRE(convert_hash_to_hex_string(hmac_hash) == cReferenceHmacSha256); } From d18d204df08793d7e0abaef7e4344e5cf42157a5 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:07:13 -0400 Subject: [PATCH 39/55] fix based on offline discussion --- components/core/src/clp/hash_utils.cpp | 45 +++++++++++++------------- components/core/src/clp/hash_utils.hpp | 27 ++++++++++++---- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 8bd364359..94381f90c 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -11,29 +10,37 @@ #include "ErrorCode.hpp" #include "spdlog_with_specializations.hpp" -#include "type_utils.hpp" -using clp::size_checked_pointer_cast; using std::span; using std::string; -using std::string_view; using std::vector; namespace clp { -auto EvpDigestContext::digest_update(std::span input) -> bool { +auto EvpDigestContext::digest_update(std::span input) -> ErrorCode { if (m_is_digest_finalized) { - throw OperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); + return ErrorCode_Unsupported; } - return static_cast(EVP_DigestUpdate(m_md_ctx, input.data(), input.size())); + if (1 != EVP_DigestUpdate(m_md_ctx, input.data(), input.size())) { + return ErrorCode_Failure; + } + return ErrorCode_Success; } -auto EvpDigestContext::digest_final_ex(std::vector& hash, unsigned int& length) - -> bool { +auto EvpDigestContext::digest_final_ex(std::vector& hash) -> ErrorCode { + if (m_is_digest_finalized) { + return ErrorCode_Unsupported; + } + + hash.resize(SHA256_DIGEST_LENGTH); + unsigned int length; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { - return false; + return ErrorCode_Failure; + } + if (SHA256_DIGEST_LENGTH != length) { + return ErrorCode_Corrupt; } m_is_digest_finalized = true; - return true; + return ErrorCode_Success; } auto convert_hash_to_hex_string(span input) -> string { @@ -78,25 +85,17 @@ auto get_hmac_sha256_hash( return ErrorCode_Success; } -/** - * Gets the SHA256 hash - * @param input - * @return The SHA256 hash - */ auto get_sha256_hash(span input, vector& hash) -> ErrorCode { EvpDigestContext evp_ctx_manager{EVP_sha256(), nullptr}; - if (false == evp_ctx_manager.digest_update(input)) { + if (auto error_code = evp_ctx_manager.digest_update(input); ErrorCode_Success != error_code) { SPDLOG_ERROR("Failed to digest input"); - return ErrorCode_Failure; + return error_code; } - unsigned int digest_len{0}; - hash.resize(SHA256_DIGEST_LENGTH); - - if (false == evp_ctx_manager.digest_final_ex(hash, digest_len)) { + if (auto error_code = evp_ctx_manager.digest_final_ex(hash); ErrorCode_Success != error_code) { SPDLOG_ERROR("Failed to Finalize digest"); - return ErrorCode_Failure; + return error_code; } return ErrorCode_Success; diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index a36db4710..b9e168d32 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -52,10 +52,24 @@ class EvpDigestContext { ~EvpDigestContext() { EVP_MD_CTX_destroy(m_md_ctx); } // Methods - [[nodiscard]] auto digest_update(std::span input) -> bool; + /** + * Calls EVP_DigestUpdate to digest the input and update the hash context + * @param input + * @return ErrorCode_Success on success. + * @return ErrorCode_Unsupported if context is already finalized. + * @return ErrorCode_Failure if EVP_DigestUpdate fails. + */ + [[nodiscard]] auto digest_update(std::span input) -> ErrorCode; - [[nodiscard]] auto - digest_final_ex(std::vector& hash, unsigned int& length) -> bool; + /** + * Calls EVP_DigestFinal_ex to generate the final hash result + * @param hash Returns the hashing result. + * @return ErrorCode_Success on success. + * @return ErrorCode_Unsupported if context is already finalized. + * @return ErrorCode_Corrupt if the hashing result has an unexpected length. + * @return ErrorCode_Failure if EVP_DigestFinal_ex fails. + */ + [[nodiscard]] auto digest_final_ex(std::vector& hash) -> ErrorCode; private: EVP_MD_CTX* m_md_ctx{nullptr}; @@ -63,7 +77,7 @@ class EvpDigestContext { }; /** - * Converts input hash into a hex string. + * Converts input hash into a hex string * @param input * @return input hash as a hex string */ @@ -89,9 +103,8 @@ class EvpDigestContext { * @param input * @param hash Returns the hashing result. * @return ErrorCode_Success on success. - * @return ErrorCode_BadParam if input key exceeds maximum length. - * @return ErrorCode_Failure if hash generation fails. - * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initalized + * @return Same as digest_final_ex and digest_update on failure. + * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initialized. */ [[nodiscard]] auto get_sha256_hash( std::span input, From 1894ee4245bad80c37386315fc767ee9a5d8e02d Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:21:51 -0400 Subject: [PATCH 40/55] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- components/core/src/clp/hash_utils.cpp | 10 +++++++--- components/core/src/clp/hash_utils.hpp | 6 +++--- components/core/tests/test-hash_utils.cpp | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 94381f90c..8e185f1a8 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -32,7 +32,7 @@ auto EvpDigestContext::digest_final_ex(std::vector& hash) -> Erro } hash.resize(SHA256_DIGEST_LENGTH); - unsigned int length; + unsigned int length{}; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { return ErrorCode_Failure; } @@ -88,12 +88,16 @@ auto get_hmac_sha256_hash( auto get_sha256_hash(span input, vector& hash) -> ErrorCode { EvpDigestContext evp_ctx_manager{EVP_sha256(), nullptr}; - if (auto error_code = evp_ctx_manager.digest_update(input); ErrorCode_Success != error_code) { + if (auto const error_code = evp_ctx_manager.digest_update(input); + ErrorCode_Success != error_code) + { SPDLOG_ERROR("Failed to digest input"); return error_code; } - if (auto error_code = evp_ctx_manager.digest_final_ex(hash); ErrorCode_Success != error_code) { + if (auto const error_code = evp_ctx_manager.digest_final_ex(hash); + ErrorCode_Success != error_code) + { SPDLOG_ERROR("Failed to Finalize digest"); return error_code; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index b9e168d32..9bae295af 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -31,7 +31,7 @@ class EvpDigestContext { }; // Constructors - EvpDigestContext(const EVP_MD* type, ENGINE* impl) : m_md_ctx{EVP_MD_CTX_create()} { + EvpDigestContext(EVP_MD const* type, ENGINE* impl) : m_md_ctx{EVP_MD_CTX_create()} { if (nullptr == m_md_ctx) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -77,7 +77,7 @@ class EvpDigestContext { }; /** - * Converts input hash into a hex string + * Converts input hash into a hex string. * @param input * @return input hash as a hex string */ @@ -99,7 +99,7 @@ class EvpDigestContext { ) -> ErrorCode; /** - * Gets the SHA256 hash of input + * Gets the SHA256 hash of the input. * @param input * @param hash Returns the hashing result. * @return ErrorCode_Success on success. diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp index 8d0b11e0b..63958fcc5 100644 --- a/components/core/tests/test-hash_utils.cpp +++ b/components/core/tests/test-hash_utils.cpp @@ -20,7 +20,7 @@ TEST_CASE("test_sha256", "[hash_utils]") { constexpr string_view cReferenceSha256{ "c3a1d9f04ada1198c4c63bf51d9933fc2cc216429275cadabdcb2178775add0c" }; - vector hash{}; + vector hash; REQUIRE(ErrorCode_Success == get_sha256_hash( @@ -37,7 +37,7 @@ TEST_CASE("test_hmac", "[hash_utils]") { constexpr string_view cReferenceHmacSha256{ "38373057694c1038a6895212bea46849eb7a59b73a2ec175883ae095fb91ffda" }; - vector hmac_hash{}; + vector hmac_hash; REQUIRE(ErrorCode_Success == get_hmac_sha256_hash( From 9286d05eb360cef1441592ce5aa4d26703639772 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:29:30 -0400 Subject: [PATCH 41/55] Polishing --- components/core/src/clp/hash_utils.cpp | 2 +- components/core/src/clp/hash_utils.hpp | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 8e185f1a8..4ff2508d3 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -86,7 +86,7 @@ auto get_hmac_sha256_hash( } auto get_sha256_hash(span input, vector& hash) -> ErrorCode { - EvpDigestContext evp_ctx_manager{EVP_sha256(), nullptr}; + EvpDigestContext evp_ctx_manager{EVP_sha256()}; if (auto const error_code = evp_ctx_manager.digest_update(input); ErrorCode_Success != error_code) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 9bae295af..bbdf670c6 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -31,13 +31,21 @@ class EvpDigestContext { }; // Constructors - EvpDigestContext(EVP_MD const* type, ENGINE* impl) : m_md_ctx{EVP_MD_CTX_create()} { + /** + * Create and initialize a hash context by calling EVP_MD_CTX_create + * and EVP_DigestInit_ex from OpenSSL library + * @param type the type of digest (hash algorithm) from OpenSSL library. + * @throw EvpCtxManager::ErrorCode_Failure if EVP_MD_CTX_create fails. + * @throw EvpCtxManager::ErrorCode_NotInit if EVP_DigestInit_ex fails. + */ + EvpDigestContext(EVP_MD const* type) : m_md_ctx{EVP_MD_CTX_create()} { if (nullptr == m_md_ctx) { - throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); - } - if (1 != EVP_DigestInit_ex(m_md_ctx, type, impl)) { throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); } + // Set impl to nullptr to use the default implementation of digest type + if (1 != EVP_DigestInit_ex(m_md_ctx, type, nullptr)) { + throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); + } } // Disable copy constructor/assignment operator From d5ef0feda059b9b0f8d7af8c9b830a46d76c5d8a Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:30:20 -0400 Subject: [PATCH 42/55] Grammar and linter --- components/core/src/clp/hash_utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index bbdf670c6..646822a13 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -32,7 +32,7 @@ class EvpDigestContext { // Constructors /** - * Create and initialize a hash context by calling EVP_MD_CTX_create + * Creates and initializes a hash context by calling EVP_MD_CTX_create * and EVP_DigestInit_ex from OpenSSL library * @param type the type of digest (hash algorithm) from OpenSSL library. * @throw EvpCtxManager::ErrorCode_Failure if EVP_MD_CTX_create fails. From 1822fb28ead767b792bfb99350086ef30bae2adc Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Sat, 27 Jul 2024 17:32:19 -0400 Subject: [PATCH 43/55] Apply suggestions from code review commit code review suggestions Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/hash_utils.cpp | 6 ++--- components/core/src/clp/hash_utils.hpp | 37 ++++++++++++-------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 4ff2508d3..921ae3e91 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -16,7 +16,7 @@ using std::string; using std::vector; namespace clp { -auto EvpDigestContext::digest_update(std::span input) -> ErrorCode { +auto EvpDigestContext::digest_update(span input) -> ErrorCode { if (m_is_digest_finalized) { return ErrorCode_Unsupported; } @@ -31,12 +31,12 @@ auto EvpDigestContext::digest_final_ex(std::vector& hash) -> Erro return ErrorCode_Unsupported; } - hash.resize(SHA256_DIGEST_LENGTH); + hash.resize(EVP_MD_CTX_size(m_md_ctx)); unsigned int length{}; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { return ErrorCode_Failure; } - if (SHA256_DIGEST_LENGTH != length) { + if (hash.size() != length) { return ErrorCode_Corrupt; } m_is_digest_finalized = true; diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 646822a13..d99b40369 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -13,7 +13,7 @@ namespace clp { /** - * A C++ wrapper for openssl's EVP digest message digest context (EVP_MD_CTX). + * A C++ wrapper for OpenSSL's EVP message digest context (EVP_MD_CTX). */ class EvpDigestContext { public: @@ -26,17 +26,15 @@ class EvpDigestContext { // Methods [[nodiscard]] auto what() const noexcept -> char const* override { - return "EvpCtxManager operation failed"; + return "EvpDigestContext operation failed"; } }; // Constructors /** - * Creates and initializes a hash context by calling EVP_MD_CTX_create - * and EVP_DigestInit_ex from OpenSSL library - * @param type the type of digest (hash algorithm) from OpenSSL library. - * @throw EvpCtxManager::ErrorCode_Failure if EVP_MD_CTX_create fails. - * @throw EvpCtxManager::ErrorCode_NotInit if EVP_DigestInit_ex fails. + * @param type The type of digest (hash algorithm). + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_MD_CTX_create` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_NotInit if `EVP_DigestInit_ex` fails. */ EvpDigestContext(EVP_MD const* type) : m_md_ctx{EVP_MD_CTX_create()} { if (nullptr == m_md_ctx) { @@ -48,11 +46,11 @@ class EvpDigestContext { } } - // Disable copy constructor/assignment operator + // Disable copy constructor and assignment operator EvpDigestContext(EvpDigestContext const&) = delete; auto operator=(EvpDigestContext const&) -> EvpDigestContext& = delete; - // disable move constructor/assignment operator + // Disable move constructor and assignment operator EvpDigestContext(EvpDigestContext&&) = delete; auto operator=(EvpDigestContext&&) -> EvpDigestContext& = delete; @@ -61,21 +59,21 @@ class EvpDigestContext { // Methods /** - * Calls EVP_DigestUpdate to digest the input and update the hash context + * Hashes `input` into the digest. * @param input * @return ErrorCode_Success on success. * @return ErrorCode_Unsupported if context is already finalized. - * @return ErrorCode_Failure if EVP_DigestUpdate fails. + * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. */ [[nodiscard]] auto digest_update(std::span input) -> ErrorCode; /** - * Calls EVP_DigestFinal_ex to generate the final hash result + * Writes the digest into `hash`. * @param hash Returns the hashing result. * @return ErrorCode_Success on success. * @return ErrorCode_Unsupported if context is already finalized. * @return ErrorCode_Corrupt if the hashing result has an unexpected length. - * @return ErrorCode_Failure if EVP_DigestFinal_ex fails. + * @return ErrorCode_Failure if `EVP_DigestFinal_ex` fails. */ [[nodiscard]] auto digest_final_ex(std::vector& hash) -> ErrorCode; @@ -85,19 +83,18 @@ class EvpDigestContext { }; /** - * Converts input hash into a hex string. * @param input - * @return input hash as a hex string + * @return `input` as a hex string (without the "0x" prefix). */ [[nodiscard]] auto convert_hash_to_hex_string(std::span input) -> std::string; /** - * Gets the HMAC-SHA256 hash of input with key. + * Gets the HMAC-SHA256 hash of `input` with `key`. * @param input * @param key - * @param hash Returns the hashing result. + * @param hash Returns the HMAC. * @return ErrorCode_Success on success. - * @return ErrorCode_BadParam if input key exceeds maximum length. + * @return ErrorCode_BadParam if `key` is longer than `INT32_MAX`. * @return ErrorCode_Failure if hash generation fails. */ [[nodiscard]] auto get_hmac_sha256_hash( @@ -107,9 +104,9 @@ class EvpDigestContext { ) -> ErrorCode; /** - * Gets the SHA256 hash of the input. + * Gets the SHA256 hash of `input`. * @param input - * @param hash Returns the hashing result. + * @param hash Returns the hash. * @return ErrorCode_Success on success. * @return Same as digest_final_ex and digest_update on failure. * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initialized. From 9cfcb8599a6c6f825e22b196b0604f5eb2aaa511 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sat, 27 Jul 2024 17:56:43 -0400 Subject: [PATCH 44/55] refactor to address code review concerns --- components/core/src/clp/hash_utils.cpp | 126 +++++++++++++++++----- components/core/src/clp/hash_utils.hpp | 77 +------------ components/core/tests/test-hash_utils.cpp | 6 +- 3 files changed, 103 insertions(+), 106 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 921ae3e91..8ed6fae4c 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -9,41 +9,115 @@ #include #include "ErrorCode.hpp" -#include "spdlog_with_specializations.hpp" +#include "TraceableException.hpp" using std::span; using std::string; using std::vector; -namespace clp { -auto EvpDigestContext::digest_update(span input) -> ErrorCode { - if (m_is_digest_finalized) { - return ErrorCode_Unsupported; +namespace { +/** + * A C++ wrapper for OpenSSL's EVP message digest context (EVP_MD_CTX). + */ +class EvpDigestContext { +public: + // Types + class OperationFailed : public clp::TraceableException { + public: + // Constructors + OperationFailed(clp::ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + + // Methods + [[nodiscard]] auto what() const noexcept -> char const* override { + return "EvpDigestContext operation failed"; + } + }; + + // Constructors + /** + * @param type The type of digest (hash algorithm). + * @throw EvpDigestContext::OperationFailed with ErrorCode_NoMem if + * `EVP_MD_CTX_create` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if + * `EVP_DigestInit_ex` fails. + */ + EvpDigestContext(EVP_MD const* type) + : m_md_ctx{EVP_MD_CTX_create()}, + m_digest_nid{EVP_MD_type(type)} { + if (nullptr == m_md_ctx) { + throw OperationFailed(clp::ErrorCode_NoMem, __FILENAME__, __LINE__); + } + // Set impl to nullptr to use the default implementation of digest type + if (1 != EVP_DigestInit_ex(m_md_ctx, type, nullptr)) { + throw OperationFailed(clp::ErrorCode_Failure, __FILENAME__, __LINE__); + } } + + // Disable copy constructor and assignment operator + EvpDigestContext(EvpDigestContext const&) = delete; + auto operator=(EvpDigestContext const&) -> EvpDigestContext& = delete; + + // Disable move constructor and assignment operator + EvpDigestContext(EvpDigestContext&&) = delete; + auto operator=(EvpDigestContext&&) -> EvpDigestContext& = delete; + + // Destructor + ~EvpDigestContext() { EVP_MD_CTX_destroy(m_md_ctx); } + + // Methods + /** + * Hashes `input` into the digest. + * @param input + * @return ErrorCode_Success on success. + * @return ErrorCode_Unsupported if context is already finalized. + * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. + */ + [[nodiscard]] auto digest_update(std::span input) -> clp::ErrorCode; + + /** + * Writes the digest into `hash` and clears the digest. + * @param hash Returns the hashing result. + * @return ErrorCode_Success on success. + * @return ErrorCode_Unsupported if context is already finalized. + * @return ErrorCode_Corrupt if the hashing result has an unexpected length. + * @return ErrorCode_Failure if `EVP_DigestFinal_ex` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if + * `EVP_DigestInit_ex` fails. + */ + [[nodiscard]] auto digest_final(std::vector& hash) -> clp::ErrorCode; + +private: + EVP_MD_CTX* m_md_ctx{nullptr}; + int m_digest_nid{}; +}; + +auto EvpDigestContext::digest_update(span input) -> clp::ErrorCode { if (1 != EVP_DigestUpdate(m_md_ctx, input.data(), input.size())) { - return ErrorCode_Failure; + return clp::ErrorCode_Failure; } - return ErrorCode_Success; + return clp::ErrorCode_Success; } -auto EvpDigestContext::digest_final_ex(std::vector& hash) -> ErrorCode { - if (m_is_digest_finalized) { - return ErrorCode_Unsupported; - } - +auto EvpDigestContext::digest_final(std::vector& hash) -> clp::ErrorCode { hash.resize(EVP_MD_CTX_size(m_md_ctx)); unsigned int length{}; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { - return ErrorCode_Failure; + return clp::ErrorCode_Failure; } if (hash.size() != length) { - return ErrorCode_Corrupt; + return clp::ErrorCode_Corrupt; } - m_is_digest_finalized = true; - return ErrorCode_Success; + + if (1 != EVP_DigestInit_ex(m_md_ctx, EVP_get_digestbynid(m_digest_nid), nullptr)) { + throw OperationFailed(clp::ErrorCode_Failure, __FILENAME__, __LINE__); + } + return clp::ErrorCode_Success; } +} // namespace -auto convert_hash_to_hex_string(span input) -> string { +namespace clp { +auto convert_to_hex_string(std::span input) -> string { string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); @@ -56,13 +130,12 @@ auto get_hmac_sha256_hash( span key, vector& hash ) -> ErrorCode { - hash.resize(SHA256_DIGEST_LENGTH); - unsigned int hash_length{0}; - if (key.size() > INT32_MAX) { - SPDLOG_ERROR("Input key exceeds maximum length"); return ErrorCode_BadParam; } + + hash.resize(SHA256_DIGEST_LENGTH); + unsigned int hash_length{0}; auto const key_length{static_cast(key.size())}; if (nullptr == HMAC(EVP_sha256(), @@ -73,13 +146,11 @@ auto get_hmac_sha256_hash( hash.data(), &hash_length)) { - SPDLOG_ERROR("Failed to get HMAC hashes"); return ErrorCode_Failure; } - if (hash_length != SHA256_DIGEST_LENGTH) { - SPDLOG_ERROR("Unexpected hash length"); - return ErrorCode_Failure; + if (hash.size() != hash_length) { + return ErrorCode_Corrupt; } return ErrorCode_Success; @@ -91,14 +162,11 @@ auto get_sha256_hash(span input, vector& has if (auto const error_code = evp_ctx_manager.digest_update(input); ErrorCode_Success != error_code) { - SPDLOG_ERROR("Failed to digest input"); return error_code; } - if (auto const error_code = evp_ctx_manager.digest_final_ex(hash); - ErrorCode_Success != error_code) + if (auto const error_code = evp_ctx_manager.digest_final(hash); ErrorCode_Success != error_code) { - SPDLOG_ERROR("Failed to Finalize digest"); return error_code; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index d99b40369..2fb78d1bc 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -9,84 +9,13 @@ #include #include "ErrorCode.hpp" -#include "TraceableException.hpp" namespace clp { -/** - * A C++ wrapper for OpenSSL's EVP message digest context (EVP_MD_CTX). - */ -class EvpDigestContext { -public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - - // Methods - [[nodiscard]] auto what() const noexcept -> char const* override { - return "EvpDigestContext operation failed"; - } - }; - - // Constructors - /** - * @param type The type of digest (hash algorithm). - * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_MD_CTX_create` fails. - * @throw EvpDigestContext::OperationFailed with ErrorCode_NotInit if `EVP_DigestInit_ex` fails. - */ - EvpDigestContext(EVP_MD const* type) : m_md_ctx{EVP_MD_CTX_create()} { - if (nullptr == m_md_ctx) { - throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); - } - // Set impl to nullptr to use the default implementation of digest type - if (1 != EVP_DigestInit_ex(m_md_ctx, type, nullptr)) { - throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); - } - } - - // Disable copy constructor and assignment operator - EvpDigestContext(EvpDigestContext const&) = delete; - auto operator=(EvpDigestContext const&) -> EvpDigestContext& = delete; - - // Disable move constructor and assignment operator - EvpDigestContext(EvpDigestContext&&) = delete; - auto operator=(EvpDigestContext&&) -> EvpDigestContext& = delete; - - // Destructor - ~EvpDigestContext() { EVP_MD_CTX_destroy(m_md_ctx); } - - // Methods - /** - * Hashes `input` into the digest. - * @param input - * @return ErrorCode_Success on success. - * @return ErrorCode_Unsupported if context is already finalized. - * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. - */ - [[nodiscard]] auto digest_update(std::span input) -> ErrorCode; - - /** - * Writes the digest into `hash`. - * @param hash Returns the hashing result. - * @return ErrorCode_Success on success. - * @return ErrorCode_Unsupported if context is already finalized. - * @return ErrorCode_Corrupt if the hashing result has an unexpected length. - * @return ErrorCode_Failure if `EVP_DigestFinal_ex` fails. - */ - [[nodiscard]] auto digest_final_ex(std::vector& hash) -> ErrorCode; - -private: - EVP_MD_CTX* m_md_ctx{nullptr}; - bool m_is_digest_finalized{false}; -}; - /** * @param input * @return `input` as a hex string (without the "0x" prefix). */ -[[nodiscard]] auto convert_hash_to_hex_string(std::span input) -> std::string; +[[nodiscard]] auto convert_to_hex_string(std::span input) -> std::string; /** * Gets the HMAC-SHA256 hash of `input` with `key`. @@ -108,8 +37,8 @@ class EvpDigestContext { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. - * @return Same as digest_final_ex and digest_update on failure. - * @throw EvpCtxManager::OperationFailed if EvpCtxManager can not be initialized. + * @return Same as `digest_final` and `digest_update` on failure. + * @throw EvpDigestContext::OperationFailed if `EvpDigestContext` cannot be initialized. */ [[nodiscard]] auto get_sha256_hash( std::span input, diff --git a/components/core/tests/test-hash_utils.cpp b/components/core/tests/test-hash_utils.cpp index 63958fcc5..9e5aacd47 100644 --- a/components/core/tests/test-hash_utils.cpp +++ b/components/core/tests/test-hash_utils.cpp @@ -7,7 +7,7 @@ #include "../src/clp/hash_utils.hpp" #include "../src/clp/type_utils.hpp" -using clp::convert_hash_to_hex_string; +using clp::convert_to_hex_string; using clp::ErrorCode_Success; using clp::get_hmac_sha256_hash; using clp::get_sha256_hash; @@ -28,7 +28,7 @@ TEST_CASE("test_sha256", "[hash_utils]") { cInputString.size()}, hash )); - REQUIRE(convert_hash_to_hex_string(hash) == cReferenceSha256); + REQUIRE(convert_to_hex_string(hash) == cReferenceSha256); } TEST_CASE("test_hmac", "[hash_utils]") { @@ -47,5 +47,5 @@ TEST_CASE("test_hmac", "[hash_utils]") { cInputKey.size()}, hmac_hash )); - REQUIRE(convert_hash_to_hex_string(hmac_hash) == cReferenceHmacSha256); + REQUIRE(convert_to_hex_string(hmac_hash) == cReferenceHmacSha256); } From 2fc868d66d7cee84128d826d9d0aa1375a6b7f82 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sat, 27 Jul 2024 18:09:56 -0400 Subject: [PATCH 45/55] Add openssl error string --- components/core/src/clp/hash_utils.cpp | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 8ed6fae4c..2cdaafb7f 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -16,6 +17,15 @@ using std::string; using std::vector; namespace { +/** + * Gets the OpenSSL error string as a c++ string + * @return The string representing the OpenSSL error + */ +auto get_openssl_error_string() -> string { + auto const openssl_err = ERR_get_error(); + return {ERR_error_string(openssl_err, nullptr)}; +} + /** * A C++ wrapper for OpenSSL's EVP message digest context (EVP_MD_CTX). */ @@ -26,12 +36,29 @@ class EvpDigestContext { public: // Constructors OperationFailed(clp::ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} + : OperationFailed( + error_code, + filename, + line_number, + "EvpDigestContext operation failed" + ) {} + + OperationFailed( + clp::ErrorCode error_code, + char const* const filename, + int line_number, + std::string message + ) + : TraceableException(error_code, filename, line_number), + m_message(std::move(message)) {} // Methods [[nodiscard]] auto what() const noexcept -> char const* override { - return "EvpDigestContext operation failed"; + return m_message.c_str(); } + + private: + string m_message; }; // Constructors @@ -50,7 +77,12 @@ class EvpDigestContext { } // Set impl to nullptr to use the default implementation of digest type if (1 != EVP_DigestInit_ex(m_md_ctx, type, nullptr)) { - throw OperationFailed(clp::ErrorCode_Failure, __FILENAME__, __LINE__); + throw OperationFailed( + clp::ErrorCode_Failure, + __FILENAME__, + __LINE__, + get_openssl_error_string() + ); } } @@ -70,7 +102,6 @@ class EvpDigestContext { * Hashes `input` into the digest. * @param input * @return ErrorCode_Success on success. - * @return ErrorCode_Unsupported if context is already finalized. * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. */ [[nodiscard]] auto digest_update(std::span input) -> clp::ErrorCode; @@ -79,7 +110,6 @@ class EvpDigestContext { * Writes the digest into `hash` and clears the digest. * @param hash Returns the hashing result. * @return ErrorCode_Success on success. - * @return ErrorCode_Unsupported if context is already finalized. * @return ErrorCode_Corrupt if the hashing result has an unexpected length. * @return ErrorCode_Failure if `EVP_DigestFinal_ex` fails. * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if @@ -110,7 +140,12 @@ auto EvpDigestContext::digest_final(std::vector& hash) -> clp::Er } if (1 != EVP_DigestInit_ex(m_md_ctx, EVP_get_digestbynid(m_digest_nid), nullptr)) { - throw OperationFailed(clp::ErrorCode_Failure, __FILENAME__, __LINE__); + throw OperationFailed( + clp::ErrorCode_Failure, + __FILENAME__, + __LINE__, + get_openssl_error_string() + ); } return clp::ErrorCode_Success; } From 8ba21773870d72027069f425e71b1e3deceeed0a Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sat, 27 Jul 2024 21:10:19 -0400 Subject: [PATCH 46/55] move namespace --- components/core/src/clp/hash_utils.cpp | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 2cdaafb7f..5d0e48af1 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -16,6 +16,7 @@ using std::span; using std::string; using std::vector; +namespace clp { namespace { /** * Gets the OpenSSL error string as a c++ string @@ -35,7 +36,7 @@ class EvpDigestContext { class OperationFailed : public clp::TraceableException { public: // Constructors - OperationFailed(clp::ErrorCode error_code, char const* const filename, int line_number) + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) : OperationFailed( error_code, filename, @@ -44,7 +45,7 @@ class EvpDigestContext { ) {} OperationFailed( - clp::ErrorCode error_code, + ErrorCode error_code, char const* const filename, int line_number, std::string message @@ -73,12 +74,12 @@ class EvpDigestContext { : m_md_ctx{EVP_MD_CTX_create()}, m_digest_nid{EVP_MD_type(type)} { if (nullptr == m_md_ctx) { - throw OperationFailed(clp::ErrorCode_NoMem, __FILENAME__, __LINE__); + throw OperationFailed(ErrorCode_NoMem, __FILENAME__, __LINE__); } // Set impl to nullptr to use the default implementation of digest type if (1 != EVP_DigestInit_ex(m_md_ctx, type, nullptr)) { throw OperationFailed( - clp::ErrorCode_Failure, + ErrorCode_Failure, __FILENAME__, __LINE__, get_openssl_error_string() @@ -104,7 +105,7 @@ class EvpDigestContext { * @return ErrorCode_Success on success. * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. */ - [[nodiscard]] auto digest_update(std::span input) -> clp::ErrorCode; + [[nodiscard]] auto digest_update(std::span input) -> ErrorCode; /** * Writes the digest into `hash` and clears the digest. @@ -115,43 +116,42 @@ class EvpDigestContext { * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if * `EVP_DigestInit_ex` fails. */ - [[nodiscard]] auto digest_final(std::vector& hash) -> clp::ErrorCode; + [[nodiscard]] auto digest_final(std::vector& hash) -> ErrorCode; private: EVP_MD_CTX* m_md_ctx{nullptr}; int m_digest_nid{}; }; -auto EvpDigestContext::digest_update(span input) -> clp::ErrorCode { +auto EvpDigestContext::digest_update(span input) -> ErrorCode { if (1 != EVP_DigestUpdate(m_md_ctx, input.data(), input.size())) { - return clp::ErrorCode_Failure; + return ErrorCode_Failure; } - return clp::ErrorCode_Success; + return ErrorCode_Success; } -auto EvpDigestContext::digest_final(std::vector& hash) -> clp::ErrorCode { +auto EvpDigestContext::digest_final(std::vector& hash) -> ErrorCode { hash.resize(EVP_MD_CTX_size(m_md_ctx)); unsigned int length{}; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { - return clp::ErrorCode_Failure; + return ErrorCode_Failure; } if (hash.size() != length) { - return clp::ErrorCode_Corrupt; + return ErrorCode_Corrupt; } if (1 != EVP_DigestInit_ex(m_md_ctx, EVP_get_digestbynid(m_digest_nid), nullptr)) { throw OperationFailed( - clp::ErrorCode_Failure, + ErrorCode_Failure, __FILENAME__, __LINE__, get_openssl_error_string() ); } - return clp::ErrorCode_Success; + return ErrorCode_Success; } } // namespace -namespace clp { auto convert_to_hex_string(std::span input) -> string { string hex_string; for (auto const c : input) { From fc38bedff8d11c13e6e6078c4024ec4f1b7cb752 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sat, 27 Jul 2024 21:47:18 -0400 Subject: [PATCH 47/55] update --- components/core/src/clp/hash_utils.cpp | 16 ++++++++++++---- components/core/src/clp/hash_utils.hpp | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 5d0e48af1..a58ce5aa8 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -1,5 +1,6 @@ #include "hash_utils.hpp" +#include #include #include #include @@ -12,8 +13,10 @@ #include "ErrorCode.hpp" #include "TraceableException.hpp" +using std::make_unique; using std::span; using std::string; +using std::unique_ptr; using std::vector; namespace clp { @@ -192,15 +195,20 @@ auto get_hmac_sha256_hash( } auto get_sha256_hash(span input, vector& hash) -> ErrorCode { - EvpDigestContext evp_ctx_manager{EVP_sha256()}; - - if (auto const error_code = evp_ctx_manager.digest_update(input); + unique_ptr evp_ctx_manager; + try { + evp_ctx_manager = make_unique(EVP_sha256()); + } catch (EvpDigestContext::OperationFailed const& err) { + return err.get_error_code(); + } + if (auto const error_code = evp_ctx_manager->digest_update(input); ErrorCode_Success != error_code) { return error_code; } - if (auto const error_code = evp_ctx_manager.digest_final(hash); ErrorCode_Success != error_code) + if (auto const error_code = evp_ctx_manager->digest_final(hash); + ErrorCode_Success != error_code) { return error_code; } diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 2fb78d1bc..408cbbd27 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -37,8 +37,9 @@ namespace clp { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. + * @return The error code specified by EvpDigestContext constructor + * if the EvpDigestContext instance fails to initialize. * @return Same as `digest_final` and `digest_update` on failure. - * @throw EvpDigestContext::OperationFailed if `EvpDigestContext` cannot be initialized. */ [[nodiscard]] auto get_sha256_hash( std::span input, From d8277a4dc2f1971cc02fd46d5d5f252b05df6753 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sun, 28 Jul 2024 23:00:38 -0400 Subject: [PATCH 48/55] introduce a public exception class --- components/core/src/clp/hash_utils.cpp | 9 ++++---- components/core/src/clp/hash_utils.hpp | 32 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index a58ce5aa8..ccd707905 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -68,10 +68,9 @@ class EvpDigestContext { // Constructors /** * @param type The type of digest (hash algorithm). - * @throw EvpDigestContext::OperationFailed with ErrorCode_NoMem if - * `EVP_MD_CTX_create` fails. - * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if - * `EVP_DigestInit_ex` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_NoMem if `EVP_MD_CTX_create` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_DigestInit_ex` + * fails. */ EvpDigestContext(EVP_MD const* type) : m_md_ctx{EVP_MD_CTX_create()}, @@ -199,7 +198,7 @@ auto get_sha256_hash(span input, vector& has try { evp_ctx_manager = make_unique(EVP_sha256()); } catch (EvpDigestContext::OperationFailed const& err) { - return err.get_error_code(); + throw HashUtilsOperationFailed(err.get_error_code(), __FILENAME__, __LINE__, err.what()); } if (auto const error_code = evp_ctx_manager->digest_update(input); ErrorCode_Success != error_code) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 408cbbd27..8aea2db78 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -9,8 +9,37 @@ #include #include "ErrorCode.hpp" +#include "TraceableException.hpp" namespace clp { +// Types +class HashUtilsOperationFailed : public TraceableException { +public: + // Constructors + HashUtilsOperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : HashUtilsOperationFailed( + error_code, + filename, + line_number, + "HashUtils operation failed" + ) {} + + HashUtilsOperationFailed( + ErrorCode error_code, + char const* const filename, + int line_number, + std::string message + ) + : TraceableException(error_code, filename, line_number), + m_message(std::move(message)) {} + + // Methods + [[nodiscard]] auto what() const noexcept -> char const* override { return m_message.c_str(); } + +private: + std::string m_message; +}; + /** * @param input * @return `input` as a hex string (without the "0x" prefix). @@ -37,8 +66,7 @@ namespace clp { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. - * @return The error code specified by EvpDigestContext constructor - * if the EvpDigestContext instance fails to initialize. + * @throw HashUtilsOperationFailed if `EvpDigestContext` fails to initialize. * @return Same as `digest_final` and `digest_update` on failure. */ [[nodiscard]] auto get_sha256_hash( From 02634a82eba5156fac678b569d1e17be8d584ecf Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:00:33 -0400 Subject: [PATCH 49/55] Small touch --- components/core/src/clp/hash_utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 8aea2db78..9a387fe3a 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -66,8 +66,8 @@ class HashUtilsOperationFailed : public TraceableException { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. - * @throw HashUtilsOperationFailed if `EvpDigestContext` fails to initialize. * @return Same as `digest_final` and `digest_update` on failure. + * @throw HashUtilsOperationFailed if `EvpDigestContext` fails to initialize. */ [[nodiscard]] auto get_sha256_hash( std::span input, From e6109d2211be771e2f9a14251dd67f581d66e043 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:11:31 -0400 Subject: [PATCH 50/55] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/hash_utils.cpp | 28 ++++++++++++++------------ components/core/src/clp/hash_utils.hpp | 6 ++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index ccd707905..07273f187 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -1,12 +1,15 @@ #include "hash_utils.hpp" +#include #include #include #include +#include #include #include #include +#include #include #include @@ -22,8 +25,8 @@ using std::vector; namespace clp { namespace { /** - * Gets the OpenSSL error string as a c++ string - * @return The string representing the OpenSSL error + * Pops the first OpenSSL error from its error queue and gets its string representation. + * @return The string representing the first OpenSSL error from its error queue. */ auto get_openssl_error_string() -> string { auto const openssl_err = ERR_get_error(); @@ -51,7 +54,7 @@ class EvpDigestContext { ErrorCode error_code, char const* const filename, int line_number, - std::string message + string message ) : TraceableException(error_code, filename, line_number), m_message(std::move(message)) {} @@ -69,10 +72,9 @@ class EvpDigestContext { /** * @param type The type of digest (hash algorithm). * @throw EvpDigestContext::OperationFailed with ErrorCode_NoMem if `EVP_MD_CTX_create` fails. - * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_DigestInit_ex` - * fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_DigestInit_ex` fails. */ - EvpDigestContext(EVP_MD const* type) + explicit EvpDigestContext(EVP_MD const* type) : m_md_ctx{EVP_MD_CTX_create()}, m_digest_nid{EVP_MD_type(type)} { if (nullptr == m_md_ctx) { @@ -107,18 +109,17 @@ class EvpDigestContext { * @return ErrorCode_Success on success. * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. */ - [[nodiscard]] auto digest_update(std::span input) -> ErrorCode; + [[nodiscard]] auto digest_update(span input) -> ErrorCode; /** * Writes the digest into `hash` and clears the digest. * @param hash Returns the hashing result. * @return ErrorCode_Success on success. - * @return ErrorCode_Corrupt if the hashing result has an unexpected length. + * @return ErrorCode_Corrupt if `hash` has an unexpected length. * @return ErrorCode_Failure if `EVP_DigestFinal_ex` fails. - * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if - * `EVP_DigestInit_ex` fails. + * @throw EvpDigestContext::OperationFailed with ErrorCode_Failure if `EVP_DigestInit_ex` fails. */ - [[nodiscard]] auto digest_final(std::vector& hash) -> ErrorCode; + [[nodiscard]] auto digest_final(vector& hash) -> ErrorCode; private: EVP_MD_CTX* m_md_ctx{nullptr}; @@ -132,7 +133,7 @@ auto EvpDigestContext::digest_update(span input) -> ErrorCo return ErrorCode_Success; } -auto EvpDigestContext::digest_final(std::vector& hash) -> ErrorCode { +auto EvpDigestContext::digest_final(vector& hash) -> ErrorCode { hash.resize(EVP_MD_CTX_size(m_md_ctx)); unsigned int length{}; if (1 != EVP_DigestFinal_ex(m_md_ctx, hash.data(), &length)) { @@ -154,7 +155,7 @@ auto EvpDigestContext::digest_final(std::vector& hash) -> ErrorCo } } // namespace -auto convert_to_hex_string(std::span input) -> string { +auto convert_to_hex_string(span input) -> string { string hex_string; for (auto const c : input) { hex_string += fmt::format("{:02x}", c); @@ -200,6 +201,7 @@ auto get_sha256_hash(span input, vector& has } catch (EvpDigestContext::OperationFailed const& err) { throw HashUtilsOperationFailed(err.get_error_code(), __FILENAME__, __LINE__, err.what()); } + if (auto const error_code = evp_ctx_manager->digest_update(input); ErrorCode_Success != error_code) { diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 9a387fe3a..a4ec1d8dc 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -6,8 +6,6 @@ #include #include -#include - #include "ErrorCode.hpp" #include "TraceableException.hpp" @@ -21,7 +19,7 @@ class HashUtilsOperationFailed : public TraceableException { error_code, filename, line_number, - "HashUtils operation failed" + "clp::hash_utils operation failed" ) {} HashUtilsOperationFailed( @@ -67,7 +65,7 @@ class HashUtilsOperationFailed : public TraceableException { * @param hash Returns the hash. * @return ErrorCode_Success on success. * @return Same as `digest_final` and `digest_update` on failure. - * @throw HashUtilsOperationFailed if `EvpDigestContext` fails to initialize. + * @throw HashUtilsOperationFailed if an OpenSSL EVP digest couldn't be created. */ [[nodiscard]] auto get_sha256_hash( std::span input, From 63f4673e10501a64bfd59aa74ddec77fa1f2df12 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:12:15 -0400 Subject: [PATCH 51/55] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/hash_utils.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index a4ec1d8dc..6e8600c2a 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -1,9 +1,9 @@ #ifndef CLP_HASH_UTILS_HPP #define CLP_HASH_UTILS_HPP -#include #include #include +#include #include #include "ErrorCode.hpp" @@ -52,6 +52,7 @@ class HashUtilsOperationFailed : public TraceableException { * @return ErrorCode_Success on success. * @return ErrorCode_BadParam if `key` is longer than `INT32_MAX`. * @return ErrorCode_Failure if hash generation fails. + * @return ErrorCode_Corrupt if `hash` has an unexpected length. */ [[nodiscard]] auto get_hmac_sha256_hash( std::span input, From 29e605884090709c155e302a2b9a4ce9e0d6a16b Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:54:41 -0400 Subject: [PATCH 52/55] Address codeview comments --- components/core/src/clp/hash_utils.cpp | 16 +++++++--------- components/core/src/clp/hash_utils.hpp | 5 +++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 07273f187..64e4d50d7 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -106,10 +106,9 @@ class EvpDigestContext { /** * Hashes `input` into the digest. * @param input - * @return ErrorCode_Success on success. - * @return ErrorCode_Failure if `EVP_DigestUpdate` fails. + * @return Whether `EVP_DigestUpdate` succeeded. */ - [[nodiscard]] auto digest_update(span input) -> ErrorCode; + [[nodiscard]] auto digest_update(span input) -> bool; /** * Writes the digest into `hash` and clears the digest. @@ -126,11 +125,11 @@ class EvpDigestContext { int m_digest_nid{}; }; -auto EvpDigestContext::digest_update(span input) -> ErrorCode { +auto EvpDigestContext::digest_update(span input) -> bool { if (1 != EVP_DigestUpdate(m_md_ctx, input.data(), input.size())) { - return ErrorCode_Failure; + return false; } - return ErrorCode_Success; + return true; } auto EvpDigestContext::digest_final(vector& hash) -> ErrorCode { @@ -202,10 +201,9 @@ auto get_sha256_hash(span input, vector& has throw HashUtilsOperationFailed(err.get_error_code(), __FILENAME__, __LINE__, err.what()); } - if (auto const error_code = evp_ctx_manager->digest_update(input); - ErrorCode_Success != error_code) + if (false == evp_ctx_manager->digest_update(input)) { - return error_code; + return ErrorCode_Failure; } if (auto const error_code = evp_ctx_manager->digest_final(hash); diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index 6e8600c2a..c53af65cf 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -52,7 +52,7 @@ class HashUtilsOperationFailed : public TraceableException { * @return ErrorCode_Success on success. * @return ErrorCode_BadParam if `key` is longer than `INT32_MAX`. * @return ErrorCode_Failure if hash generation fails. - * @return ErrorCode_Corrupt if `hash` has an unexpected length. + * @return ErrorCode_Corrupt if `hash` has an unexpected length. */ [[nodiscard]] auto get_hmac_sha256_hash( std::span input, @@ -65,7 +65,8 @@ class HashUtilsOperationFailed : public TraceableException { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. - * @return Same as `digest_final` and `digest_update` on failure. + * @return ErrorCode_Failure if digest_update` fails. + * @return Same as `digest_final` if `digest_final` fails. * @throw HashUtilsOperationFailed if an OpenSSL EVP digest couldn't be created. */ [[nodiscard]] auto get_sha256_hash( From c9d2a17f3806ea0aca5e97352c344567ee10fee7 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:01:23 -0400 Subject: [PATCH 53/55] Linter --- components/core/src/clp/hash_utils.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index 64e4d50d7..bcc5d611c 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -201,8 +201,7 @@ auto get_sha256_hash(span input, vector& has throw HashUtilsOperationFailed(err.get_error_code(), __FILENAME__, __LINE__, err.what()); } - if (false == evp_ctx_manager->digest_update(input)) - { + if (false == evp_ctx_manager->digest_update(input)) { return ErrorCode_Failure; } From 6f2296c690d297bd9dc851884caa3930cdbd51fb Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:09:46 -0400 Subject: [PATCH 54/55] update error message --- components/core/src/clp/hash_utils.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/hash_utils.cpp b/components/core/src/clp/hash_utils.cpp index bcc5d611c..a70cd8ff6 100644 --- a/components/core/src/clp/hash_utils.cpp +++ b/components/core/src/clp/hash_utils.cpp @@ -30,7 +30,14 @@ namespace { */ auto get_openssl_error_string() -> string { auto const openssl_err = ERR_get_error(); - return {ERR_error_string(openssl_err, nullptr)}; + if (0 == openssl_err) { + return {}; + } + auto* openssl_err_str = ERR_error_string(openssl_err, nullptr); + if (nullptr == openssl_err_str) { + return {"Error has no string representation, error_code: " + std::to_string(openssl_err)}; + } + return {openssl_err_str}; } /** From b1c3f0f8060824a977197b0c54940e5a3264d408 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:34:36 -0400 Subject: [PATCH 55/55] Minor comment fix. --- components/core/src/clp/hash_utils.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/hash_utils.hpp b/components/core/src/clp/hash_utils.hpp index c53af65cf..4606aed23 100644 --- a/components/core/src/clp/hash_utils.hpp +++ b/components/core/src/clp/hash_utils.hpp @@ -65,8 +65,8 @@ class HashUtilsOperationFailed : public TraceableException { * @param input * @param hash Returns the hash. * @return ErrorCode_Success on success. - * @return ErrorCode_Failure if digest_update` fails. - * @return Same as `digest_final` if `digest_final` fails. + * @return ErrorCode_Failure if `EvpDigestContext::digest_update` fails. + * @return Same as `EvpDigestContext::digest_final` if `EvpDigestContext::digest_final` fails. * @throw HashUtilsOperationFailed if an OpenSSL EVP digest couldn't be created. */ [[nodiscard]] auto get_sha256_hash(