From 1149158b7a5232cdc8165026fb84f32687af3bdb Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Tue, 9 May 2023 10:37:14 +0100 Subject: [PATCH 001/135] [release/4.x] Cherry pick: JS runtime traces in responses (#5237) (#5245) --- CHANGELOG.md | 10 ++++ doc/schemas/gov_openapi.json | 8 ++- include/ccf/odata_error.h | 25 +++++++-- include/ccf/rpc_context.h | 2 +- include/ccf/service/tables/jsengine.h | 13 ++++- samples/constitutions/default/actions.js | 10 ++++ src/apps/js_generic/js_generic_base.cpp | 32 ++++++++--- src/js/wrap.cpp | 3 + src/js/wrap.h | 3 + src/node/rpc/frontend.h | 17 ++++-- src/node/rpc/member_frontend.h | 2 +- src/node/rpc/rpc_context_impl.h | 2 +- tests/infra/consortium.py | 10 +++- tests/js-modules/modules.py | 71 ++++++++++++++++++++++++ tests/npm-app/app.json | 29 ++++++++++ tests/npm-app/src/endpoints/rpc.ts | 7 +++ 16 files changed, 220 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffa3ca39a8e4..a1924c050f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.1] + +[4.0.1]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.1 + +- The `set_js_runtime_options` action now accepts `return_exception_details` and `log_exception_details` boolean options, which set the corresponding keys in the `public:ccf.gov.js_runtime_options` KV map. When enabled, a stack trace is respectively returned to the caller, and emitted to the log, on uncaught JS exceptions in application code. + +## Changed + +- For security reasons, OpenSSL `>=1.1.1f` must be first installed on the system (Ubuntu) before installing the CCF Debian package (#5227). + ## [4.0.0] In order to upgrade an existing 3.x service to 4.x, CCF must be on the latest 3.x version (at least 3.0.10). For more information, see [our documentation](https://microsoft.github.io/CCF/main/operations/code_upgrade.html) diff --git a/doc/schemas/gov_openapi.json b/doc/schemas/gov_openapi.json index a3e695b4303f..c5d164791141 100644 --- a/doc/schemas/gov_openapi.json +++ b/doc/schemas/gov_openapi.json @@ -309,6 +309,9 @@ }, "JSRuntimeOptions": { "properties": { + "log_exception_details": { + "$ref": "#/components/schemas/boolean" + }, "max_execution_time_ms": { "$ref": "#/components/schemas/uint64" }, @@ -317,6 +320,9 @@ }, "max_stack_bytes": { "$ref": "#/components/schemas/uint64" + }, + "return_exception_details": { + "$ref": "#/components/schemas/boolean" } }, "required": [ @@ -1264,7 +1270,7 @@ "info": { "description": "This API is used to submit and query proposals which affect CCF's public governance tables.", "title": "CCF Governance API", - "version": "4.0.0" + "version": "4.1.0" }, "openapi": "3.0.0", "paths": { diff --git a/include/ccf/odata_error.h b/include/ccf/odata_error.h index 799ed6ab1280..bc6f9e88ec2b 100644 --- a/include/ccf/odata_error.h +++ b/include/ccf/odata_error.h @@ -7,23 +7,37 @@ namespace ccf { - struct ODataErrorDetails + struct ODataAuthErrorDetails { std::string auth_policy; std::string code; std::string message; - bool operator==(const ODataErrorDetails&) const = default; + bool operator==(const ODataAuthErrorDetails&) const = default; }; - DECLARE_JSON_TYPE(ODataErrorDetails); - DECLARE_JSON_REQUIRED_FIELDS(ODataErrorDetails, auth_policy, code, message); + DECLARE_JSON_TYPE(ODataAuthErrorDetails); + DECLARE_JSON_REQUIRED_FIELDS( + ODataAuthErrorDetails, auth_policy, code, message); + + struct ODataJSExceptionDetails + { + std::string code; + std::string message; + std::optional trace; + + bool operator==(const ODataJSExceptionDetails&) const = default; + }; + + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ODataJSExceptionDetails); + DECLARE_JSON_REQUIRED_FIELDS(ODataJSExceptionDetails, code, message); + DECLARE_JSON_OPTIONAL_FIELDS(ODataJSExceptionDetails, trace); struct ODataError { std::string code; std::string message; - std::vector details = {}; + std::vector details = {}; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ODataError); @@ -97,6 +111,7 @@ namespace ccf ERROR(ProposalReplay) ERROR(ProposalCreatedTooLongAgo) ERROR(InvalidCreatedAt) + ERROR(JSException) // node-to-node (/join and /create): ERROR(ConsensusTypeMismatch) diff --git a/include/ccf/rpc_context.h b/include/ccf/rpc_context.h index 5e0eb05727d3..79c803e17808 100644 --- a/include/ccf/rpc_context.h +++ b/include/ccf/rpc_context.h @@ -159,7 +159,7 @@ namespace ccf http_status status, const std::string& code, std::string&& msg, - const std::vector& details = {}) = 0; + const std::vector& details = {}) = 0; /// Construct error response, formatted according to the request content /// type (either JSON OData-formatted or gRPC error) diff --git a/include/ccf/service/tables/jsengine.h b/include/ccf/service/tables/jsengine.h index cc9b3b13af13..4dea68460611 100644 --- a/include/ccf/service/tables/jsengine.h +++ b/include/ccf/service/tables/jsengine.h @@ -14,11 +14,22 @@ namespace ccf size_t max_stack_bytes; /// @brief max execution time for QuickJS uint64_t max_execution_time_ms; + /// @brief emit exception details to the log + /// NOTE: this is a security risk as it may leak sensitive information + /// to anyone with access to the application log, which is + /// unprotected. + bool log_exception_details = false; + /// @brief return exception details in the response + /// NOTE: this is a security risk as it may leak sensitive information, + /// albeit to the caller only. + bool return_exception_details = false; }; - DECLARE_JSON_TYPE(JSRuntimeOptions) + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(JSRuntimeOptions) DECLARE_JSON_REQUIRED_FIELDS( JSRuntimeOptions, max_heap_bytes, max_stack_bytes, max_execution_time_ms) + DECLARE_JSON_OPTIONAL_FIELDS( + JSRuntimeOptions, log_exception_details, return_exception_details); using JSEngine = ServiceValue; diff --git a/samples/constitutions/default/actions.js b/samples/constitutions/default/actions.js index 2cac85630a84..5fdbb52bf5b5 100644 --- a/samples/constitutions/default/actions.js +++ b/samples/constitutions/default/actions.js @@ -783,6 +783,16 @@ const actions = new Map([ "integer", "max_execution_time_ms" ); + checkType( + args.log_exception_details, + "boolean?", + "log_exception_details" + ); + checkType( + args.return_exception_details, + "boolean?", + "return_exception_details" + ); }, function (args) { const js_engine_map = ccf.kv["public:ccf.gov.js_runtime_options"]; diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 729210dda760..bd0f1611e02e 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -307,18 +307,36 @@ namespace ccfapp { bool time_out = ctx.interrupt_data.request_timed_out; std::string error_msg = "Exception thrown while executing."; - - js::js_dump_error(ctx); - if (time_out) { error_msg = "Operation took too long to complete."; } - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - std::move(error_msg)); + auto [reason, trace] = js::js_error_message(ctx); + + if (rt.log_exception_details) + { + CCF_APP_FAIL("{}: {}", reason, trace.value_or("")); + } + + if (rt.return_exception_details) + { + std::vector details = { + ODataJSExceptionDetails{ccf::errors::JSException, reason, trace}}; + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + std::move(error_msg), + std::move(details)); + } + else + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + std::move(error_msg)); + } + return; } diff --git a/src/js/wrap.cpp b/src/js/wrap.cpp index ae196c204b53..33eae83809b0 100644 --- a/src/js/wrap.cpp +++ b/src/js/wrap.cpp @@ -176,6 +176,9 @@ namespace ccf::js stack_size = js_runtime_options.value().max_stack_bytes; max_exec_time = std::chrono::milliseconds{ js_runtime_options.value().max_execution_time_ms}; + log_exception_details = js_runtime_options.value().log_exception_details; + return_exception_details = + js_runtime_options.value().return_exception_details; } JS_SetMaxStackSize(rt, stack_size); diff --git a/src/js/wrap.h b/src/js/wrap.h index 6b57f6685af3..5f29ed51926a 100644 --- a/src/js/wrap.h +++ b/src/js/wrap.h @@ -243,6 +243,9 @@ namespace ccf::js std::chrono::milliseconds max_exec_time = default_max_execution_time; public: + bool log_exception_details = false; + bool return_exception_details = false; + Runtime(kv::Tx* tx); ~Runtime(); diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index 2cd888c18694..26503e7e4c70 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -244,7 +244,7 @@ namespace ccf std::unique_ptr identity = nullptr; std::string auth_error_reason; - std::vector error_details; + std::vector error_details; for (const auto& policy : endpoint->authn_policies) { identity = policy->authenticate(tx, ctx, auth_error_reason); @@ -255,10 +255,10 @@ namespace ccf else { // Collate error details - error_details.push_back( - {policy->get_security_scheme_name(), - ccf::errors::InvalidAuthenticationInfo, - auth_error_reason}); + error_details.emplace_back(ODataAuthErrorDetails{ + policy->get_security_scheme_name(), + ccf::errors::InvalidAuthenticationInfo, + auth_error_reason}); } } @@ -269,11 +269,16 @@ namespace ccf ctx, std::move(auth_error_reason)); // Return collated error details for the auth policies // declared in the request + std::vector json_details; + for (auto& details : error_details) + { + json_details.push_back(details); + } ctx->set_error( HTTP_STATUS_UNAUTHORIZED, ccf::errors::InvalidAuthenticationInfo, "Invalid authentication credentials.", - error_details); + std::move(json_details)); update_metrics(ctx); } diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 617d33ebf90e..ec5418a8fd3f 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -606,7 +606,7 @@ namespace ccf openapi_info.description = "This API is used to submit and query proposals which affect CCF's " "public governance tables."; - openapi_info.document_version = "4.0.0"; + openapi_info.document_version = "4.1.0"; } static std::optional get_caller_member_id( diff --git a/src/node/rpc/rpc_context_impl.h b/src/node/rpc/rpc_context_impl.h index 28c933815c2c..94b2638f63f5 100644 --- a/src/node/rpc/rpc_context_impl.h +++ b/src/node/rpc/rpc_context_impl.h @@ -76,7 +76,7 @@ namespace ccf http_status status, const std::string& code, std::string&& msg, - const std::vector& details = {}) override + const std::vector& details = {}) override { auto content_type = get_request_header(http::headers::CONTENT_TYPE); if ( diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index c3d9f161782b..387bf7b851af 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -528,13 +528,21 @@ def set_js_app_from_json( return self.vote_using_majority(remote_node, proposal, careful_vote, timeout=30) def set_js_runtime_options( - self, remote_node, max_heap_bytes, max_stack_bytes, max_execution_time_ms + self, + remote_node, + max_heap_bytes, + max_stack_bytes, + max_execution_time_ms, + log_exception_details=False, + return_exception_details=False, ): proposal_body, careful_vote = self.make_proposal( "set_js_runtime_options", max_heap_bytes=max_heap_bytes, max_stack_bytes=max_stack_bytes, max_execution_time_ms=max_execution_time_ms, + log_exception_details=log_exception_details, + return_exception_details=return_exception_details, ) proposal = self.get_any_active_member().propose(remote_node, proposal_body) return self.vote_using_majority(remote_node, proposal, careful_vote) diff --git a/tests/js-modules/modules.py b/tests/js-modules/modules.py index e698b7368806..0fb92e66af6a 100644 --- a/tests/js-modules/modules.py +++ b/tests/js-modules/modules.py @@ -1007,6 +1007,76 @@ def test_js_execution_time(network, args): return network +@reqs.description("Test JS exception output") +def test_js_exception_output(network, args): + primary, _ = network.find_nodes() + + with primary.client("user0") as c: + r = c.get("/node/js_metrics") + body = r.body.json() + default_max_heap_size = body["max_heap_size"] + default_max_stack_size = body["max_stack_size"] + default_max_execution_time = body["max_execution_time"] + + r = c.get("/app/throw") + assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r.status_code + body = r.body.json() + assert body["error"]["code"] == "InternalError" + assert body["error"]["message"] == "Exception thrown while executing." + assert "details" not in body["error"] + + network.consortium.set_js_runtime_options( + primary, + max_heap_bytes=default_max_heap_size, + max_stack_bytes=default_max_stack_size, + max_execution_time_ms=default_max_execution_time, + return_exception_details=True, + ) + r = c.get("/app/throw") + assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r.status_code + body = r.body.json() + assert body["error"]["code"] == "InternalError" + assert body["error"]["message"] == "Exception thrown while executing." + assert body["error"]["details"][0]["code"] == "JSException" + assert body["error"]["details"][0]["message"] == "Error: test error: 42" + assert ( + body["error"]["details"][0]["trace"] + == " at nested (endpoints/rpc.js:27)\n at throwError (endpoints/rpc.js:29)\n" + ) + + network.consortium.set_js_runtime_options( + primary, + max_heap_bytes=default_max_heap_size, + max_stack_bytes=default_max_stack_size, + max_execution_time_ms=default_max_execution_time, + return_exception_details=False, + ) + + r = c.get("/app/throw") + assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r.status_code + body = r.body.json() + assert body["error"]["code"] == "InternalError" + assert body["error"]["message"] == "Exception thrown while executing." + assert "details" not in body["error"] + + network.consortium.set_js_runtime_options( + primary, + max_heap_bytes=default_max_heap_size, + max_stack_bytes=default_max_stack_size, + max_execution_time_ms=default_max_execution_time, + log_exception_details=True, + ) + + r = c.get("/app/throw") + assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r.status_code + body = r.body.json() + assert body["error"]["code"] == "InternalError" + assert body["error"]["message"] == "Exception thrown while executing." + assert "details" not in body["error"] + + return network + + def run(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb @@ -1019,6 +1089,7 @@ def run(args): network = test_set_js_runtime(network, args) network = test_npm_app(network, args) network = test_js_execution_time(network, args) + network = test_js_exception_output(network, args) if __name__ == "__main__": diff --git a/tests/npm-app/app.json b/tests/npm-app/app.json index c065f3a5326f..8b13849225d5 100644 --- a/tests/npm-app/app.json +++ b/tests/npm-app/app.json @@ -1255,6 +1255,35 @@ } } } + }, + "/throw": { + "get": { + "js_module": "endpoints/rpc.js", + "js_function": "throwError", + "forwarding_required": "sometimes", + "authn_policies": ["user_cert"], + "mode": "readonly", + "openapi": { + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": {} + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } } } } diff --git a/tests/npm-app/src/endpoints/rpc.ts b/tests/npm-app/src/endpoints/rpc.ts index 248b00a99e62..1a99e557a21d 100644 --- a/tests/npm-app/src/endpoints/rpc.ts +++ b/tests/npm-app/src/endpoints/rpc.ts @@ -27,3 +27,10 @@ export function getApplyWrites(request: ccfapp.Request): ccfapp.Response { body: v, }; } + +export function throwError(request: ccfapp.Request): ccfapp.Response { + function nested(arg: number) { + throw new Error(`test error: ${arg}`); + } + nested(42); +} From 816454a9de2c48d03082afa561a3bd28e55f3496 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Tue, 9 May 2023 11:21:35 +0100 Subject: [PATCH 002/135] [release/4.x] Cherry pick: Add `populate_service_endorsements` to public headers (#5242) (#5243) --- CHANGELOG.md | 4 + cmake/common.cmake | 1 + include/ccf/historical_queries_utils.h | 27 ++++ .../ccf}/network_identity_interface.h | 0 .../external_executor/external_executor.cpp | 6 +- src/node/historical_queries_adapter.cpp | 6 +- src/node/historical_queries_utils.cpp | 147 ++++++++++++++++++ src/node/historical_queries_utils.h | 147 ------------------ src/node/rpc/network_identity_subsystem.h | 2 +- 9 files changed, 186 insertions(+), 154 deletions(-) create mode 100644 include/ccf/historical_queries_utils.h rename {src/node/rpc => include/ccf}/network_identity_interface.h (100%) create mode 100644 src/node/historical_queries_utils.cpp delete mode 100644 src/node/historical_queries_utils.h diff --git a/CHANGELOG.md b/CHANGELOG.md index a1924c050f2c..44390e45efa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - For security reasons, OpenSSL `>=1.1.1f` must be first installed on the system (Ubuntu) before installing the CCF Debian package (#5227). +## Added + +- Added `ccf::historical::populate_service_endorsements` to public C++ API, allowing custom historical endpoints to do the same work as adapters. + ## [4.0.0] In order to upgrade an existing 3.x service to 4.x, CCF must be on the latest 3.x version (at least 3.0.10). For more information, see [our documentation](https://microsoft.github.io/CCF/main/operations/code_upgrade.html) diff --git a/cmake/common.cmake b/cmake/common.cmake index 1b24143736ba..5d94274b3b5f 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -184,6 +184,7 @@ set(CCF_ENDPOINTS_SOURCES ${CCF_DIR}/src/indexing/strategies/seqnos_by_key_in_memory.cpp ${CCF_DIR}/src/indexing/strategies/visit_each_entry_in_map.cpp ${CCF_DIR}/src/node/historical_queries_adapter.cpp + ${CCF_DIR}/src/node/historical_queries_utils.cpp ${CCF_DIR}/src/node/receipt.cpp ) diff --git a/include/ccf/historical_queries_utils.h b/include/ccf/historical_queries_utils.h new file mode 100644 index 000000000000..9d1f375ad4d5 --- /dev/null +++ b/include/ccf/historical_queries_utils.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ccf/historical_queries_interface.h" +#include "ccf/network_identity_interface.h" +#include "ccf/rpc_context.h" +#include "ccf/tx.h" + +namespace ccf::historical +{ + // Modifies the receipt stored in state to include historical service + // endorsements, where required. If the state talks about a different service + // identity, which is known to be a predecessor of this service (via disaster + // recoveries), then an endorsement of the receipt's node certificate will be + // created. This may need to use the state_cache to request additional + // historical entries to construct this endorsement, and may read from the + // current/latest state via tx. Returns true if the operation is complete, + // though it may still have failed to produce an endorsement. Returns false if + // additional entries have been requested, in which case the caller should + // retry later. + bool populate_service_endorsements( + kv::ReadOnlyTx& tx, + ccf::historical::StatePtr& state, + AbstractStateCache& state_cache, + std::shared_ptr + network_identity_subsystem); +} \ No newline at end of file diff --git a/src/node/rpc/network_identity_interface.h b/include/ccf/network_identity_interface.h similarity index 100% rename from src/node/rpc/network_identity_interface.h rename to include/ccf/network_identity_interface.h diff --git a/src/apps/external_executor/external_executor.cpp b/src/apps/external_executor/external_executor.cpp index 3fd55b9e8b12..ad989776d0bd 100644 --- a/src/apps/external_executor/external_executor.cpp +++ b/src/apps/external_executor/external_executor.cpp @@ -6,6 +6,7 @@ #include "ccf/crypto/verifier.h" #include "ccf/entity_id.h" #include "ccf/historical_queries_adapter.h" +#include "ccf/historical_queries_utils.h" #include "ccf/http_consts.h" #include "ccf/http_responder.h" #include "ccf/json_handler.h" @@ -20,7 +21,6 @@ #include "kv.pb.h" #include "misc.pb.h" #include "node/endpoint_context_impl.h" -#include "node/historical_queries_utils.h" #include "node/rpc/network_identity_subsystem.h" #include "node/rpc/rpc_context_impl.h" @@ -690,8 +690,8 @@ namespace externalexecutor if ( historical_state == nullptr || - (!get_service_endorsements( - ctx, historical_state, state_cache, network_identity_subsystem))) + (!populate_service_endorsements( + ctx.tx, historical_state, state_cache, network_identity_subsystem))) { externalexecutor::protobuf::QueryResponse response; response.set_retry(true); diff --git a/src/node/historical_queries_adapter.cpp b/src/node/historical_queries_adapter.cpp index cda095a65d8a..83a21bbe8469 100644 --- a/src/node/historical_queries_adapter.cpp +++ b/src/node/historical_queries_adapter.cpp @@ -3,10 +3,10 @@ #include "ccf/historical_queries_adapter.h" +#include "ccf/historical_queries_utils.h" #include "ccf/rpc_context.h" #include "ccf/service/tables/service.h" #include "kv/kv_types.h" -#include "node/historical_queries_utils.h" #include "node/rpc/network_identity_subsystem.h" #include "node/tx_receipt_impl.h" @@ -318,8 +318,8 @@ namespace ccf::historical state_cache.get_state_at(historic_request_handle, target_tx_id.seqno); if ( historical_state == nullptr || - (!get_service_endorsements( - args, historical_state, state_cache, network_identity_subsystem))) + (!populate_service_endorsements( + args.tx, historical_state, state_cache, network_identity_subsystem))) { args.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); constexpr size_t retry_after_seconds = 3; diff --git a/src/node/historical_queries_utils.cpp b/src/node/historical_queries_utils.cpp new file mode 100644 index 000000000000..454db32f3d9b --- /dev/null +++ b/src/node/historical_queries_utils.cpp @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ccf/historical_queries_utils.h" + +#include "ccf/rpc_context.h" +#include "ccf/service/tables/service.h" +#include "kv/kv_types.h" +#include "node/tx_receipt_impl.h" + +namespace ccf +{ + static std::map> + service_endorsement_cache; + + namespace historical + { + std::optional find_previous_service_identity( + kv::ReadOnlyTx& tx, + ccf::historical::StatePtr& state, + AbstractStateCache& state_cache) + { + SeqNo target_seqno = state->transaction_id.seqno; + + // We start at the previous write to the latest (current) service info. + auto service = tx.template ro(Tables::SERVICE); + + // Iterate until we find the most recent write to the service info that + // precedes the target seqno. + std::optional hservice_info = service->get(); + SeqNo i = -1; + do + { + if (!hservice_info->previous_service_identity_version) + { + // Pre 2.0 we did not record the versions of previous identities in + // the service table. + throw std::runtime_error( + "The service identity that signed the receipt cannot be found " + "because it is in a pre-2.0 part of the ledger."); + } + i = hservice_info->previous_service_identity_version.value_or(i - 1); + LOG_TRACE_FMT("historical service identity search at: {}", i); + auto hstate = state_cache.get_state_at(i, i); + if (!hstate) + { + return std::nullopt; // Not available yet - retry later. + } + auto htx = hstate->store->create_read_only_tx(); + auto hservice = htx.ro(Tables::SERVICE); + hservice_info = hservice->get(); + } while (i > target_seqno || (i > 1 && !hservice_info)); + + if (!hservice_info) + { + throw std::runtime_error("Failed to locate previous service identity"); + } + + return hservice_info; + } + + bool populate_service_endorsements( + kv::ReadOnlyTx& tx, + ccf::historical::StatePtr& state, + AbstractStateCache& state_cache, + std::shared_ptr + network_identity_subsystem) + { + try + { + if (!network_identity_subsystem) + { + throw std::runtime_error( + "The service identity endorsement for this receipt cannot be " + "created " + "because the current network identity is not available."); + } + + const auto& network_identity = network_identity_subsystem->get(); + + if (state && state->receipt && state->receipt->node_cert) + { + auto& receipt = *state->receipt; + + if (receipt.node_cert->empty()) + { + // Pre 2.0 receipts did not contain node certs. + throw std::runtime_error( + "Node certificate in receipt is empty, likely because the " + "transaction is in a pre-2.0 part of the ledger."); + } + + auto v = crypto::make_unique_verifier(*receipt.node_cert); + if (!v->verify_certificate( + {&network_identity->cert}, {}, /* ignore_time */ true)) + { + // The current service identity does not endorse the node + // certificate in the receipt, so we search for the the most recent + // write to the service info table before the historical transaction + // ID to get the historical service identity. + + auto opt_psi = + find_previous_service_identity(tx, state, state_cache); + if (!opt_psi) + { + return false; + } + + auto hpubkey = crypto::public_key_pem_from_cert( + crypto::cert_pem_to_der(opt_psi->cert)); + + auto eit = service_endorsement_cache.find(hpubkey); + if (eit != service_endorsement_cache.end()) + { + // Note: validity period of service certificate may have changed + // since we created the cached endorsements. + receipt.service_endorsements = eit->second; + } + else + { + auto ncv = crypto::make_unique_verifier(network_identity->cert); + auto endorsement = create_endorsed_cert( + hpubkey, + ReplicatedNetworkIdentity::subject_name, + {}, + ncv->validity_period(), + network_identity->priv_key, + network_identity->cert, + true); + service_endorsement_cache[hpubkey] = {endorsement}; + receipt.service_endorsements = {endorsement}; + } + } + } + } + catch (std::exception& ex) + { + LOG_DEBUG_FMT( + "Exception while extracting previous service identities: {}", + ex.what()); + // (We keep the incomplete receipt, no further error reporting) + } + + return true; + } + } +} \ No newline at end of file diff --git a/src/node/historical_queries_utils.h b/src/node/historical_queries_utils.h deleted file mode 100644 index ced4c739115d..000000000000 --- a/src/node/historical_queries_utils.h +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. - -//#include "ccf/historical_queries_adapter.h" - -#include "ccf/rpc_context.h" -#include "ccf/service/tables/service.h" -#include "kv/kv_types.h" -#include "node/rpc/network_identity_subsystem.h" -#include "node/tx_receipt_impl.h" - -namespace ccf -{ - static std::map> - service_endorsement_cache; -} - -namespace ccf::historical -{ - std::optional find_previous_service_identity( - auto& ctx, - ccf::historical::StatePtr& state, - AbstractStateCache& state_cache) - { - SeqNo target_seqno = state->transaction_id.seqno; - - // We start at the previous write to the latest (current) service info. - auto service = ctx.tx.template ro(Tables::SERVICE); - - // Iterate until we find the most recent write to the service info that - // precedes the target seqno. - std::optional hservice_info = service->get(); - SeqNo i = -1; - do - { - if (!hservice_info->previous_service_identity_version) - { - // Pre 2.0 we did not record the versions of previous identities in the - // service table. - throw std::runtime_error( - "The service identity that signed the receipt cannot be found " - "because it is in a pre-2.0 part of the ledger."); - } - i = hservice_info->previous_service_identity_version.value_or(i - 1); - LOG_TRACE_FMT("historical service identity search at: {}", i); - auto hstate = state_cache.get_state_at(i, i); - if (!hstate) - { - return std::nullopt; // Not available yet - retry later. - } - auto htx = hstate->store->create_read_only_tx(); - auto hservice = htx.ro(Tables::SERVICE); - hservice_info = hservice->get(); - } while (i > target_seqno || (i > 1 && !hservice_info)); - - if (!hservice_info) - { - throw std::runtime_error("Failed to locate previous service identity"); - } - - return hservice_info; - } - - bool get_service_endorsements( - auto& ctx, - ccf::historical::StatePtr& state, - AbstractStateCache& state_cache, - std::shared_ptr - network_identity_subsystem) - { - try - { - if (!network_identity_subsystem) - { - throw std::runtime_error( - "The service identity endorsement for this receipt cannot be created " - "because the current network identity is not available."); - } - - const auto& network_identity = network_identity_subsystem->get(); - - if (state && state->receipt && state->receipt->node_cert) - { - auto& receipt = *state->receipt; - - if (receipt.node_cert->empty()) - { - // Pre 2.0 receipts did not contain node certs. - throw std::runtime_error( - "Node certificate in receipt is empty, likely because the " - "transaction is in a pre-2.0 part of the ledger."); - } - - auto v = crypto::make_unique_verifier(*receipt.node_cert); - if (!v->verify_certificate( - {&network_identity->cert}, {}, /* ignore_time */ true)) - { - // The current service identity does not endorse the node certificate - // in the receipt, so we search for the the most recent write to the - // service info table before the historical transaction ID to get the - // historical service identity. - - auto opt_psi = - find_previous_service_identity(ctx, state, state_cache); - if (!opt_psi) - { - return false; - } - - auto hpubkey = crypto::public_key_pem_from_cert( - crypto::cert_pem_to_der(opt_psi->cert)); - - auto eit = service_endorsement_cache.find(hpubkey); - if (eit != service_endorsement_cache.end()) - { - // Note: validity period of service certificate may have changed - // since we created the cached endorsements. - receipt.service_endorsements = eit->second; - } - else - { - auto ncv = crypto::make_unique_verifier(network_identity->cert); - auto endorsement = create_endorsed_cert( - hpubkey, - ReplicatedNetworkIdentity::subject_name, - {}, - ncv->validity_period(), - network_identity->priv_key, - network_identity->cert, - true); - service_endorsement_cache[hpubkey] = {endorsement}; - receipt.service_endorsements = {endorsement}; - } - } - } - } - catch (std::exception& ex) - { - LOG_DEBUG_FMT( - "Exception while extracting previous service identities: {}", - ex.what()); - // (We keep the incomplete receipt, no further error reporting) - } - - return true; - } -} \ No newline at end of file diff --git a/src/node/rpc/network_identity_subsystem.h b/src/node/rpc/network_identity_subsystem.h index 5656a4aed6e1..618e97c4ed34 100644 --- a/src/node/rpc/network_identity_subsystem.h +++ b/src/node/rpc/network_identity_subsystem.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once +#include "ccf/network_identity_interface.h" #include "node/identity.h" -#include "node/rpc/network_identity_interface.h" #include "node/rpc/node_interface.h" namespace ccf From ce80f18140b4b0a818f5d7939f1fc8c8c1200fe0 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 15 May 2023 17:23:04 +0100 Subject: [PATCH 003/135] [release/4.x] Cherry pick: User cose sign1 (#5248) (#5260) --- .daily_canary | 6 +- CHANGELOG.md | 6 + doc/build_apps/api.rst | 3 + doc/build_apps/js_app_bundle.rst | 1 + doc/schemas/app_openapi.json | 66 +++++- include/ccf/common_auth_policies.h | 5 + .../ccf/endpoints/authentication/cose_auth.h | 77 +++++-- samples/apps/logging/logging.cpp | 21 +- src/apps/js_generic/js_generic_base.cpp | 8 + src/apps/js_generic/named_auth_policies.h | 8 + src/endpoints/authentication/cose_auth.cpp | 188 +++++++++++++++++- tests/e2e_logging.py | 23 ++- tests/js-authentication/app.json | 10 +- 13 files changed, 389 insertions(+), 33 deletions(-) diff --git a/.daily_canary b/.daily_canary index 26d5af5546cc..6f567a4f2aeb 100644 --- a/.daily_canary +++ b/.daily_canary @@ -1,4 +1,4 @@ - --- ___ - (- -) (o o) | Y & +-- + --- ___ ___ + (- -) (o o) | Y & +-- ( V ) z x z O +---=---' -/--x-m- /--m-m---xXx--/--yY----- \ No newline at end of file +/--x-m- /--n-m---xXx--/--yY------ diff --git a/CHANGELOG.md b/CHANGELOG.md index 44390e45efa7..35b1bdce08a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.2] + +[4.0.2]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.2 + +- Added `ccf::UserCOSESign1AuthnPolicy` (C++) and `user_cose_sign1` (JavaScript) authentication policies. + ## [4.0.1] [4.0.1]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.1 diff --git a/doc/build_apps/api.rst b/doc/build_apps/api.rst index 02efb9d9b1cc..4c3a1e60ec5b 100644 --- a/doc/build_apps/api.rst +++ b/doc/build_apps/api.rst @@ -60,6 +60,9 @@ Policies .. doxygenvariable:: ccf::user_cert_auth_policy :project: CCF +.. doxygenvariable:: ccf::user_cose_sign1_auth_policy + :project: CCF + .. doxygenvariable:: ccf::jwt_auth_policy :project: CCF diff --git a/doc/build_apps/js_app_bundle.rst b/doc/build_apps/js_app_bundle.rst index cb0b6cc55012..02aeefee5f7e 100644 --- a/doc/build_apps/js_app_bundle.rst +++ b/doc/build_apps/js_app_bundle.rst @@ -71,6 +71,7 @@ Each endpoint object contains the following information: - ``"user_cert"`` - ``"member_cert"`` - ``"jwt"`` + - ``"user_cose_sign1"`` - ``"no_auth"`` - ``"forwarding_required"``: A string indicating whether the endpoint is always forwarded, or whether it is safe to sometimes execute on followers. Possible values are: diff --git a/doc/schemas/app_openapi.json b/doc/schemas/app_openapi.json index 12dd11b301aa..cbb5d401e694 100644 --- a/doc/schemas/app_openapi.json +++ b/doc/schemas/app_openapi.json @@ -286,6 +286,11 @@ "description": "Request payload must be a COSE Sign1 document, with expected protected headers. Signer must be a member identity registered with this service.", "scheme": "cose_sign1", "type": "http" + }, + "user_cose_sign1": { + "description": "Request payload must be a COSE Sign1 document, with expected protected headers. Signer must be a user identity registered with this service.", + "scheme": "cose_sign1", + "type": "http" } }, "x-ccf-forwarding": { @@ -306,7 +311,7 @@ "info": { "description": "This CCF sample app implements a simple logging application, securely recording messages at client-specified IDs. It demonstrates most of the features available to CCF apps.", "title": "CCF Sample Logging App", - "version": "2.0.0" + "version": "2.2.0" }, "openapi": "3.0.0", "paths": { @@ -502,6 +507,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -537,6 +545,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -572,6 +583,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -609,6 +623,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -636,6 +653,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -759,6 +779,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -796,6 +819,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -841,6 +867,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -878,6 +907,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -930,6 +962,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -960,6 +995,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1012,6 +1050,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1047,6 +1088,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1082,6 +1126,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1109,6 +1156,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1136,6 +1186,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1189,6 +1242,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1226,6 +1282,9 @@ "security": [ { "jwt": [] + }, + { + "user_cose_sign1": [] } ], "x-ccf-forwarding": { @@ -1256,7 +1315,7 @@ } }, "/app/multi_auth": { - "get": { + "post": { "responses": { "200": { "content": { @@ -1276,6 +1335,9 @@ { "jwt": [] }, + { + "user_cose_sign1": [] + }, {} ], "x-ccf-forwarding": { diff --git a/include/ccf/common_auth_policies.h b/include/ccf/common_auth_policies.h index 2df6827825ed..7aff34fb1ead 100644 --- a/include/ccf/common_auth_policies.h +++ b/include/ccf/common_auth_policies.h @@ -43,6 +43,11 @@ namespace ccf member_cose_sign1_auth_policy = std::make_shared(); + /** Authenticate using COSE Sign1 payloads, and + * @c public:ccf.gov.users.certs table */ + static std::shared_ptr user_cose_sign1_auth_policy = + std::make_shared(); + /** A clear name for the empty auth policy, to reduce errors where it is * accidentally defaulted or unspecified. */ diff --git a/include/ccf/endpoints/authentication/cose_auth.h b/include/ccf/endpoints/authentication/cose_auth.h index 30070ead9945..dc2754aad6c9 100644 --- a/include/ccf/endpoints/authentication/cose_auth.h +++ b/include/ccf/endpoints/authentication/cose_auth.h @@ -9,26 +9,21 @@ namespace ccf { - struct GovernanceProtectedHeader + struct ProtectedHeader { int64_t alg; std::string kid; + }; + + struct GovernanceProtectedHeader : ProtectedHeader + { std::optional gov_msg_type; std::optional gov_msg_proposal_id; uint64_t gov_msg_created_at; }; - struct MemberCOSESign1AuthnIdentity : public AuthnIdentity + struct COSESign1AuthnIdentity : public AuthnIdentity { - /** CCF member ID */ - MemberId member_id; - - /** Member certificate, used to sign this request, described by keyId */ - crypto::Pem member_cert; - - /** COSE Protected Header */ - GovernanceProtectedHeader protected_header; - /** COSE Content */ std::span content; @@ -43,7 +38,31 @@ namespace ccf std::span signature; }; - /** COSE Sign1 Authentication Policy + struct MemberCOSESign1AuthnIdentity : public COSESign1AuthnIdentity + { + /** CCF member ID */ + MemberId member_id; + + /** Member certificate, used to sign this request, described by keyId */ + crypto::Pem member_cert; + + /** COSE Protected Header */ + GovernanceProtectedHeader protected_header; + }; + + struct UserCOSESign1AuthnIdentity : public COSESign1AuthnIdentity + { + /** CCF user ID */ + UserId user_id; + + /** User certificate, used to sign this request, described by keyId */ + crypto::Pem user_cert; + + /** COSE Protected Header */ + ProtectedHeader protected_header; + }; + + /** Member COSE Sign1 Authentication Policy * * Allows ccf.gov.msg.type and ccf.gov.msg.proposal_id protected header * entries, to specify the type of governance action, and which proposal @@ -82,4 +101,38 @@ namespace ccf return SECURITY_SCHEME_NAME; } }; + + /** User COSE Sign1 Authentication Policy + */ + class UserCOSESign1AuthnPolicy : public AuthnPolicy + { + protected: + static const OpenAPISecuritySchema security_schema; + + public: + static constexpr auto SECURITY_SCHEME_NAME = "user_cose_sign1"; + + UserCOSESign1AuthnPolicy(); + ~UserCOSESign1AuthnPolicy(); + + std::unique_ptr authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) override; + + void set_unauthenticated_error( + std::shared_ptr ctx, + std::string&& error_reason) override; + + std::optional get_openapi_security_schema() + const override + { + return security_schema; + } + + std::string get_security_scheme_name() override + { + return SECURITY_SCHEME_NAME; + } + }; } diff --git a/samples/apps/logging/logging.cpp b/samples/apps/logging/logging.cpp index 97f83734b14d..8224f60d21ec 100644 --- a/samples/apps/logging/logging.cpp +++ b/samples/apps/logging/logging.cpp @@ -312,14 +312,16 @@ namespace loggingapp "recording messages at client-specified IDs. It demonstrates most of " "the features available to CCF apps."; - openapi_info.document_version = "2.0.0"; + openapi_info.document_version = "2.2.0"; index_per_public_key = std::make_shared( PUBLIC_RECORDS, context, 10000, 20); context.get_indexing_strategies().install_strategy(index_per_public_key); const ccf::AuthnPolicies auth_policies = { - ccf::jwt_auth_policy, ccf::user_cert_auth_policy}; + ccf::jwt_auth_policy, + ccf::user_cert_auth_policy, + ccf::user_cose_sign1_auth_policy}; // SNIPPET_START: record auto record = [this](auto& ctx, nlohmann::json&& params) { @@ -912,6 +914,18 @@ namespace loggingapp ctx.rpc_ctx->set_response_body(std::move(response)); return; } + else if ( + auto cose_ident = + ctx.template try_get_caller()) + { + auto response = std::string("User COSE Sign1"); + response += fmt::format( + "\nThe caller is identified by a COSE Sign1 signed by kid: {}", + cose_ident->user_id); + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_body(std::move(response)); + return; + } else if ( auto no_ident = ctx.template try_get_caller()) @@ -931,11 +945,12 @@ namespace loggingapp }; make_endpoint( "/multi_auth", - HTTP_GET, + HTTP_POST, multi_auth, {ccf::user_cert_auth_policy, ccf::member_cert_auth_policy, ccf::jwt_auth_policy, + ccf::user_cose_sign1_auth_policy, ccf::empty_auth_policy}) .set_auto_schema() .install(); diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index bd0f1611e02e..823601a066a2 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -91,6 +91,14 @@ namespace ccfapp id = member_cert_ident->member_id; is_member = true; } + else if ( + auto user_cose_ident = + endpoint_ctx.try_get_caller()) + { + policy_name = get_policy_name_from_ident(user_cose_ident); + id = user_cose_ident->user_id; + is_member = false; + } if (policy_name == nullptr) { diff --git a/src/apps/js_generic/named_auth_policies.h b/src/apps/js_generic/named_auth_policies.h index d7759df6a818..a1b757fe7a9c 100644 --- a/src/apps/js_generic/named_auth_policies.h +++ b/src/apps/js_generic/named_auth_policies.h @@ -27,6 +27,10 @@ namespace ccfapp policies.emplace( ccf::JwtAuthnPolicy::SECURITY_SCHEME_NAME, ccf::jwt_auth_policy); + policies.emplace( + ccf::UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME, + ccf::user_cose_sign1_auth_policy); + policies.emplace( ccf::EmptyAuthnPolicy::SECURITY_SCHEME_NAME, ccf::empty_auth_policy); } @@ -62,6 +66,10 @@ namespace ccfapp { return ccf::JwtAuthnPolicy::SECURITY_SCHEME_NAME; } + else if constexpr (std::is_same_v) + { + return ccf::UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME; + } else if constexpr (std::is_same_v) { return ccf::EmptyAuthnPolicy::SECURITY_SCHEME_NAME; diff --git a/src/endpoints/authentication/cose_auth.cpp b/src/endpoints/authentication/cose_auth.cpp index d98553fcfec9..acd9a755d752 100644 --- a/src/endpoints/authentication/cose_auth.cpp +++ b/src/endpoints/authentication/cose_auth.cpp @@ -8,6 +8,7 @@ #include "ccf/http_consts.h" #include "ccf/rpc_context.h" #include "ccf/service/tables/members.h" +#include "ccf/service/tables/users.h" #include "node/cose_common.h" #include @@ -25,7 +26,7 @@ namespace ccf "ccf.gov.msg.created_at"; std::pair - extract_protected_header_and_signature( + extract_governance_protected_header_and_signature( const std::vector& cose_sign1) { ccf::GovernanceProtectedHeader parsed; @@ -164,6 +165,101 @@ namespace ccf Signature sig{static_cast(signature.ptr), signature.len}; return {parsed, sig}; } + + std::pair + extract_protected_header_and_signature( + const std::vector& cose_sign1) + { + ccf::ProtectedHeader parsed; + + // Adapted from parse_cose_header_parameters in t_cose_parameters.c. + // t_cose doesn't support custom header parameters yet. + UsefulBufC msg{cose_sign1.data(), cose_sign1.size()}; + + QCBORError qcbor_result; + + QCBORDecodeContext ctx; + QCBORDecode_Init(&ctx, msg, QCBOR_DECODE_MODE_NORMAL); + + QCBORDecode_EnterArray(&ctx, nullptr); + qcbor_result = QCBORDecode_GetError(&ctx); + if (qcbor_result != QCBOR_SUCCESS) + { + throw COSEDecodeError("Failed to parse COSE_Sign1 outer array"); + } + + uint64_t tag = QCBORDecode_GetNthTagOfLast(&ctx, 0); + if (tag != CBOR_TAG_COSE_SIGN1) + { + throw COSEDecodeError("COSE_Sign1 is not tagged"); + } + + struct q_useful_buf_c protected_parameters; + QCBORDecode_EnterBstrWrapped( + &ctx, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, &protected_parameters); + QCBORDecode_EnterMap(&ctx, NULL); + + enum + { + ALG_INDEX, + KID_INDEX, + END_INDEX, + }; + QCBORItem header_items[END_INDEX + 1]; + + header_items[ALG_INDEX].label.int64 = headers::PARAM_ALG; + header_items[ALG_INDEX].uLabelType = QCBOR_TYPE_INT64; + header_items[ALG_INDEX].uDataType = QCBOR_TYPE_INT64; + + header_items[KID_INDEX].label.int64 = headers::PARAM_KID; + header_items[KID_INDEX].uLabelType = QCBOR_TYPE_INT64; + header_items[KID_INDEX].uDataType = QCBOR_TYPE_BYTE_STRING; + + header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE; + + QCBORDecode_GetItemsInMap(&ctx, header_items); + + qcbor_result = QCBORDecode_GetError(&ctx); + if (qcbor_result != QCBOR_SUCCESS) + { + throw COSEDecodeError( + fmt::format("Failed to decode protected header: {}", qcbor_result)); + } + + if (header_items[ALG_INDEX].uDataType == QCBOR_TYPE_NONE) + { + throw COSEDecodeError("Missing algorithm in protected header"); + } + parsed.alg = header_items[ALG_INDEX].val.int64; + + if (header_items[KID_INDEX].uDataType == QCBOR_TYPE_NONE) + { + throw COSEDecodeError("Missing kid in protected header"); + } + parsed.kid = qcbor_buf_to_string(header_items[KID_INDEX].val.string); + + QCBORDecode_ExitMap(&ctx); + QCBORDecode_ExitBstrWrapped(&ctx); + + QCBORItem item; + // skip unprotected header + QCBORDecode_VGetNextConsume(&ctx, &item); + // payload + QCBORDecode_GetNext(&ctx, &item); + // signature + QCBORDecode_GetNext(&ctx, &item); + auto signature = item.val.string; + + QCBORDecode_ExitArray(&ctx); + auto error = QCBORDecode_Finish(&ctx); + if (error) + { + throw COSEDecodeError("Failed to decode COSE_Sign1"); + } + + Signature sig{static_cast(signature.ptr), signature.len}; + return {parsed, sig}; + } } MemberCOSESign1AuthnPolicy::MemberCOSESign1AuthnPolicy( @@ -192,7 +288,8 @@ namespace ccf } auto [phdr, cose_signature] = - cose::extract_protected_header_and_signature(ctx->get_request_body()); + cose::extract_governance_protected_header_and_signature( + ctx->get_request_body()); if (!cose::is_ecdsa_alg(phdr.alg)) { @@ -275,4 +372,91 @@ namespace ccf "Request payload must be a COSE Sign1 document, with expected " "protected headers. " "Signer must be a member identity registered with this service."}}); + + UserCOSESign1AuthnPolicy::UserCOSESign1AuthnPolicy() = default; + UserCOSESign1AuthnPolicy::~UserCOSESign1AuthnPolicy() = default; + + std::unique_ptr UserCOSESign1AuthnPolicy::authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) + { + const auto& headers = ctx->get_request_headers(); + const auto content_type_it = headers.find(http::headers::CONTENT_TYPE); + if (content_type_it == headers.end()) + { + error_reason = + fmt::format("Missing {} header", http::headers::CONTENT_TYPE); + return nullptr; + } + if (content_type_it->second != http::headervalues::contenttype::COSE) + { + error_reason = fmt::format( + "Content type is not set to {}", http::headervalues::contenttype::COSE); + return nullptr; + } + + auto [phdr, cose_signature] = + cose::extract_protected_header_and_signature(ctx->get_request_body()); + + if (!cose::is_ecdsa_alg(phdr.alg)) + { + error_reason = fmt::format("Unsupported algorithm: {}", phdr.alg); + return nullptr; + } + + UserCerts users_certs_table(Tables::USER_CERTS); + auto user_certs = tx.ro(users_certs_table); + auto user_cert = user_certs->get(phdr.kid); + if (user_cert.has_value()) + { + auto verifier = crypto::make_cose_verifier(user_cert->raw()); + + std::span body = { + ctx->get_request_body().data(), ctx->get_request_body().size()}; + std::span authned_content; + if (!verifier->verify(body, authned_content)) + { + error_reason = fmt::format("Failed to validate COSE Sign1"); + return nullptr; + } + + auto identity = std::make_unique(); + identity->user_id = phdr.kid; + identity->user_cert = user_cert.value(); + identity->protected_header = phdr; + identity->envelope = body; + identity->content = authned_content; + identity->signature = cose_signature; + return identity; + } + else + { + error_reason = fmt::format("Signer is not a known user"); + return nullptr; + } + } + + void UserCOSESign1AuthnPolicy::set_unauthenticated_error( + std::shared_ptr ctx, std::string&& error_reason) + { + ctx->set_error( + HTTP_STATUS_UNAUTHORIZED, + ccf::errors::InvalidAuthenticationInfo, + std::move(error_reason)); + ctx->set_response_header( + http::headers::WWW_AUTHENTICATE, + "COSE-SIGN1 realm=\"Signed request access\""); + } + + const OpenAPISecuritySchema UserCOSESign1AuthnPolicy::security_schema = + std::make_pair( + UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME, + nlohmann::json{ + {"type", "http"}, + {"scheme", "cose_sign1"}, + {"description", + "Request payload must be a COSE Sign1 document, with expected " + "protected headers. " + "Signer must be a user identity registered with this service."}}); } diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 1e3ead78a52d..22a4c2e664f8 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -583,12 +583,12 @@ def require_new_response(r): LOG.info("Anonymous, no auth") with primary.client() as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as a user, via TLS cert") with primary.client(user.local_id) as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as same user, now with user data") @@ -596,17 +596,17 @@ def require_new_response(r): primary, user.service_id, {"some": ["interesting", "data", 42]} ) with primary.client(user.local_id) as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as a different user, via TLS cert") with primary.client("user1") as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as a member, via TLS cert") with primary.client(member.local_id) as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as same member, now with user data") @@ -614,12 +614,12 @@ def require_new_response(r): primary, member.service_id, {"distinct": {"arbitrary": ["data"]}} ) with primary.client(member.local_id) as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate as a different member, via TLS cert") with primary.client("member1") as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") require_new_response(r) LOG.info("Authenticate via JWT token") @@ -628,14 +628,19 @@ def require_new_response(r): jwt = jwt_issuer.issue_jwt(claims={"user": "Alice"}) with primary.client() as c: - r = c.get("/app/multi_auth", headers={"authorization": "Bearer " + jwt}) + r = c.post("/app/multi_auth", headers={"authorization": "Bearer " + jwt}) require_new_response(r) LOG.info("Authenticate via second JWT token") jwt2 = jwt_issuer.issue_jwt(claims={"user": "Bob"}) with primary.client(common_headers={"authorization": "Bearer " + jwt2}) as c: - r = c.get("/app/multi_auth") + r = c.post("/app/multi_auth") + require_new_response(r) + + LOG.info("Authenticate via COSE Sign1 payload") + with primary.client(None, None, "user1") as c: + r = c.post("/app/multi_auth") require_new_response(r) return network diff --git a/tests/js-authentication/app.json b/tests/js-authentication/app.json index 318f44253d6b..479e70725a73 100644 --- a/tests/js-authentication/app.json +++ b/tests/js-authentication/app.json @@ -21,11 +21,17 @@ } }, "/multi_auth": { - "get": { + "post": { "js_module": "endpoints.js", "js_function": "multi_auth", "forwarding_required": "sometimes", - "authn_policies": ["user_cert", "member_cert", "jwt", "no_auth"], + "authn_policies": [ + "user_cert", + "member_cert", + "jwt", + "user_cose_sign1", + "no_auth" + ], "mode": "readonly", "openapi": {} } From c65e490ec8194b48cf9dafa363e351e633499721 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 16 May 2023 09:24:09 +0100 Subject: [PATCH 004/135] [release/4.x] Cherry pick: Support action style (`:`) templated paths (#5258) (#5262) Co-authored-by: Eddy Ashton --- CMakeLists.txt | 6 ++ cmake/common.cmake | 19 +++-- src/endpoints/endpoint_registry.cpp | 37 +++++++++ src/endpoints/test/endpoint_registry.cpp | 97 ++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 src/endpoints/test/endpoint_registry.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index da5180fb5776..23a6442e5d53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,6 +513,12 @@ if(BUILD_TESTS) quickjs.host ) + add_unit_test( + endpoint_registry_test + ${CMAKE_CURRENT_SOURCE_DIR}/src/endpoints/test/endpoint_registry.cpp + ) + target_link_libraries(endpoint_registry_test PRIVATE ccf_endpoints.host) + add_unit_test( tx_status_test ${CMAKE_CURRENT_SOURCE_DIR}/src/node/rpc/test/tx_status_test.cpp diff --git a/cmake/common.cmake b/cmake/common.cmake index 5d94274b3b5f..edd723abdf86 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -377,8 +377,11 @@ endif() # CCF endpoints libs if(COMPILE_TARGET STREQUAL "sgx") add_enclave_library(ccf_endpoints.enclave "${CCF_ENDPOINTS_SOURCES}") - target_link_libraries(ccf_endpoints.enclave PUBLIC qcbor.enclave) - target_link_libraries(ccf_endpoints.enclave PUBLIC t_cose.enclave) + target_link_libraries( + ccf_endpoints.enclave + PUBLIC qcbor.enclave t_cose.enclave http_parser.enclave ccfcrypto.enclave + ccf_kv.enclave + ) add_warning_checks(ccf_endpoints.enclave) install( TARGETS ccf_endpoints.enclave @@ -387,8 +390,10 @@ if(COMPILE_TARGET STREQUAL "sgx") ) elseif(COMPILE_TARGET STREQUAL "snp") add_host_library(ccf_endpoints.snp "${CCF_ENDPOINTS_SOURCES}") - target_link_libraries(ccf_endpoints.snp PUBLIC qcbor.snp) - target_link_libraries(ccf_endpoints.snp PUBLIC t_cose.snp) + target_link_libraries( + ccf_endpoints.snp PUBLIC qcbor.snp t_cose.snp http_parser.snp ccfcrypto.snp + ccf_kv.snp + ) add_san(ccf_endpoints.snp) add_warning_checks(ccf_endpoints.snp) install( @@ -399,8 +404,10 @@ elseif(COMPILE_TARGET STREQUAL "snp") endif() add_host_library(ccf_endpoints.host "${CCF_ENDPOINTS_SOURCES}") -target_link_libraries(ccf_endpoints.host PUBLIC qcbor.host) -target_link_libraries(ccf_endpoints.host PUBLIC t_cose.host) +target_link_libraries( + ccf_endpoints.host PUBLIC qcbor.host t_cose.host http_parser.host + ccfcrypto.host ccf_kv.host +) add_san(ccf_endpoints.host) add_warning_checks(ccf_endpoints.host) diff --git a/src/endpoints/endpoint_registry.cpp b/src/endpoints/endpoint_registry.cpp index a6a165a386ce..965043a350bb 100644 --- a/src/endpoints/endpoint_registry.cpp +++ b/src/endpoints/endpoint_registry.cpp @@ -109,10 +109,25 @@ namespace ccf::endpoints PathTemplateSpec spec; + const std::string allowed_delimiters = "/:"; + std::string regex_s(uri); template_start = regex_s.find_first_of('{'); while (template_start != std::string::npos) { + if (template_start != 0) + { + const auto prev_char = regex_s[template_start - 1]; + if (allowed_delimiters.find(prev_char) == std::string::npos) + { + throw std::logic_error(fmt::format( + "Invalid templated path - illegal character ({}) preceding " + "template: {}", + prev_char, + uri)); + } + } + const auto template_end = regex_s.find_first_of('}', template_start); if (template_end == std::string::npos) { @@ -120,6 +135,19 @@ namespace ccf::endpoints "Invalid templated path - missing closing curly bracket: {}", uri)); } + if (template_end + 1 != regex_s.size()) + { + const auto next_char = regex_s[template_end + 1]; + if (allowed_delimiters.find(next_char) == std::string::npos) + { + throw std::logic_error(fmt::format( + "Invalid templated path - illegal character ({}) following " + "template: {}", + next_char, + uri)); + } + } + spec.template_component_names.push_back( regex_s.substr(template_start + 1, template_end - template_start - 1)); regex_s.replace( @@ -127,6 +155,15 @@ namespace ccf::endpoints template_start = regex_s.find_first_of('{', template_start + 1); } + auto& names = spec.template_component_names; + if (std::unique(names.begin(), names.end()) != names.end()) + { + throw std::logic_error(fmt::format( + "Invalid templated path - duplicated component names ({}): {}", + fmt::join(names, ", "), + uri)); + } + LOG_TRACE_FMT("Parsed a templated endpoint: {} became {}", uri, regex_s); LOG_TRACE_FMT( "Component names are: {}", diff --git a/src/endpoints/test/endpoint_registry.cpp b/src/endpoints/test/endpoint_registry.cpp new file mode 100644 index 000000000000..59e544aeb7ce --- /dev/null +++ b/src/endpoints/test/endpoint_registry.cpp @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN + +#include "ccf/endpoint_registry.h" + +#include + +using namespace ccf::endpoints; + +std::optional require_parsed_components( + const std::string& s, const std::vector& expected_components) +{ + std::optional spec; + REQUIRE_NOTHROW(spec = PathTemplateSpec::parse(s)); + + if (expected_components.size() == 0) + { + REQUIRE(!spec.has_value()); + } + else + { + REQUIRE(spec.has_value()); + REQUIRE(spec->template_component_names == expected_components); + } + + return spec; +} + +TEST_CASE("URL template parsing") +{ + logger::config::default_init(); + + std::optional parsed; + std::string path; + std::smatch match; + + for (const std::string prefix : {"", "/hello", "/foo/bar/baz"}) + { + require_parsed_components(prefix + "/bob", {}); + require_parsed_components(prefix + "/{name}", {"name"}); + require_parsed_components(prefix + "/{name}/world", {"name"}); + + auto parsed = + require_parsed_components(prefix + "/{name}/{place}", {"name", "place"}); + + path = prefix + "/alice/spain"; + REQUIRE(std::regex_match(path, match, parsed->template_regex)); + REQUIRE(match[1].str() == "alice"); + REQUIRE(match[2].str() == "spain"); + + path = prefix + "/alice:jump/spain"; + REQUIRE(std::regex_match(path, match, parsed->template_regex)); + REQUIRE(match[1].str() == "alice:jump"); + REQUIRE(match[2].str() == "spain"); + + require_parsed_components(prefix + "/{name}:do", {"name"}); + require_parsed_components(prefix + "/{name}:do/world", {"name"}); + require_parsed_components(prefix + "/{name}:do/{place}", {"name", "place"}); + + require_parsed_components(prefix + "/bob:{action}", {"action"}); + require_parsed_components(prefix + "/bob:{action}/world", {"action"}); + require_parsed_components( + prefix + "/bob:{action}/{place}", {"action", "place"}); + + require_parsed_components(prefix + "/{name}:{action}", {"name", "action"}); + require_parsed_components( + prefix + "/{name}:{action}/world", {"name", "action"}); + + parsed = require_parsed_components( + prefix + "/{name}:{action}/{place}", {"name", "action", "place"}); + + path = prefix + "/alice/spain"; + REQUIRE_FALSE(std::regex_match(path, match, parsed->template_regex)); + + path = prefix + "/alice:jump/spain"; + REQUIRE(std::regex_match(path, match, parsed->template_regex)); + REQUIRE(match[1].str() == "alice"); + REQUIRE(match[2].str() == "jump"); + REQUIRE(match[3].str() == "spain"); + } + + REQUIRE_THROWS(PathTemplateSpec::parse("/foo{id}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo{id}bar")); + REQUIRE_THROWS(PathTemplateSpec::parse("/{id}bar")); + REQUIRE_THROWS(PathTemplateSpec::parse("/{id}-{name}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/id{id}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo{id}:")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo{id}/bar")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo/{id}bar")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo/id{id}:bar")); + + REQUIRE_THROWS(PathTemplateSpec::parse("/{id}/{id}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/foo/{id}/{id}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/{id}/foo/{id}")); + REQUIRE_THROWS(PathTemplateSpec::parse("/{id}/{id}/foo")); +} From d17b47f5f27c6196683f34cdadb3595995dbb1f8 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 17 May 2023 09:07:13 +0100 Subject: [PATCH 005/135] [release/4.x] Cherry pick: Fix occasional issue for historical queries - process earlier ledger secrets whenever they arrive (#5257) (#5263) --- CMakeLists.txt | 10 ++ src/node/historical_queries.h | 192 ++++++++++++++++++++-------------- tests/e2e_suite.py | 6 +- tests/suite/test_suite.py | 13 ++- 4 files changed, 138 insertions(+), 83 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23a6442e5d53..b4ebb6693db3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -673,6 +673,16 @@ if(BUILD_TESTS) ${CMAKE_SOURCE_DIR}/samples/templates ) + if(LONG_TESTS) + add_e2e_test( + NAME regression_test_suite + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py + CONSENSUS cft + LABEL suite + ADDITIONAL_ARGS --test-duration 200 --test-suite regression_5236 + ) + endif() + add_e2e_test( NAME full_test_suite PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h index 9fb6d7b3af2c..684c5537a4d9 100644 --- a/src/node/historical_queries.h +++ b/src/node/historical_queries.h @@ -104,20 +104,29 @@ namespace ccf::historical using LedgerEntry = std::vector; - ccf::VersionedLedgerSecret get_earliest_known_ledger_secret() + void update_earliest_known_ledger_secret() { - if (historical_ledger_secrets->is_empty()) + if (earliest_secret_.secret == nullptr) { - return source_ledger_secrets->get_first(); - } - - auto tx = source_store.create_read_only_tx(); - CCF_ASSERT_FMT( - historical_ledger_secrets->get_latest(tx).first < - source_ledger_secrets->get_first().first, - "Historical ledger secrets are not older than main ledger secrets"); + // Haven't worked out earliest known secret yet - work it out now + if (historical_ledger_secrets->is_empty()) + { + CCF_ASSERT_FMT( + !source_ledger_secrets->is_empty(), + "Source ledger secrets are empty"); + earliest_secret_ = source_ledger_secrets->get_first(); + } + else + { + auto tx = source_store.create_read_only_tx(); + CCF_ASSERT_FMT( + historical_ledger_secrets->get_latest(tx).first < + source_ledger_secrets->get_first().first, + "Historical ledger secrets are not older than main ledger secrets"); - return historical_ledger_secrets->get_first(); + earliest_secret_ = historical_ledger_secrets->get_first(); + } + } } struct StoreDetails @@ -168,22 +177,23 @@ namespace ccf::historical using WeakStoreDetailsPtr = std::weak_ptr; using AllRequestedStores = std::map; - struct LedgerSecretRecoveryInfo + struct VersionedSecret { - ccf::SeqNo target_seqno = 0; - LedgerSecretPtr last_ledger_secret; - StoreDetailsPtr target_details; - - LedgerSecretRecoveryInfo( - ccf::SeqNo target_seqno_, - LedgerSecretPtr last_ledger_secret_, - StoreDetailsPtr target_details_) : - target_seqno(target_seqno_), - last_ledger_secret(last_ledger_secret_), - target_details(target_details_) + ccf::SeqNo valid_from = {}; + ccf::LedgerSecretPtr secret = nullptr; + + VersionedSecret() = default; + // NB: Can't use VersionedLedgerSecret directly because the first element + // is const, so the whole thing is non-copyable + VersionedSecret(const ccf::VersionedLedgerSecret& vls) : + valid_from(vls.first), + secret(vls.second) {} }; + VersionedSecret earliest_secret_ = {}; + StoreDetailsPtr next_secret_fetch_handle = nullptr; + struct Request { AllRequestedStores& all_stores; @@ -199,8 +209,7 @@ namespace ccf::historical RequestedStores supporting_signatures; // Only set when recovering ledger secrets - std::unique_ptr ledger_secret_recovery_info = - nullptr; + std::optional awaiting_ledger_secrets = std::nullopt; Request(AllRequestedStores& all_stores_) : all_stores(all_stores_) {} @@ -305,10 +314,6 @@ namespace ccf::historical return; } - // If the range has changed, forget what ledger secrets we may have been - // fetching - the caller can begin asking for them again - ledger_secret_recovery_info = nullptr; - const auto newly_requested_receipts = should_include_receipts && !include_receipts; @@ -508,16 +513,25 @@ namespace ccf::historical consensus::LedgerRequestPurpose::HistoricalQuery); } - std::unique_ptr fetch_supporting_secret_if_needed( + std::optional fetch_supporting_secret_if_needed( ccf::SeqNo seqno) { auto [earliest_ledger_secret_seqno, earliest_ledger_secret] = - get_earliest_known_ledger_secret(); - if (seqno < earliest_ledger_secret_seqno) + earliest_secret_; + CCF_ASSERT_FMT( + earliest_ledger_secret != nullptr, + "Can't fetch without knowing earliest"); + + const auto too_early = seqno < earliest_ledger_secret_seqno; + + auto previous_secret_stored_version = + earliest_ledger_secret->previous_secret_stored_version; + const auto is_next_secret = + previous_secret_stored_version.value_or(0) == seqno; + + if (too_early || is_next_secret) { // Still need more secrets, fetch the next - auto previous_secret_stored_version = - earliest_ledger_secret->previous_secret_stored_version; if (!previous_secret_stored_version.has_value()) { throw std::logic_error(fmt::format( @@ -544,11 +558,15 @@ namespace ccf::historical fetch_entry_at(seqno_to_fetch); } - return std::make_unique( - seqno_to_fetch, earliest_ledger_secret, details); + next_secret_fetch_handle = details; + + if (too_early) + { + return seqno_to_fetch; + } } - return nullptr; + return std::nullopt; } void process_deserialised_store( @@ -598,28 +616,15 @@ namespace ccf::historical // If this request was still waiting for a ledger secret, and this is // that secret if ( - request.ledger_secret_recovery_info != nullptr && - request.ledger_secret_recovery_info->target_seqno == seqno) + request.awaiting_ledger_secrets.has_value() && + request.awaiting_ledger_secrets.value() == seqno) { - // Handle it, hopefully extending earliest_known_ledger_secret to - // cover earlier entries - const auto valid_secret = handle_encrypted_past_ledger_secret( - store, std::move(request.ledger_secret_recovery_info)); - if (!valid_secret) - { - // Invalid! Erase this request: host gave us junk, need to start - // over - request_it = requests.erase(request_it); - continue; - } + LOG_TRACE_FMT( + "{} is a ledger secret seqno this request was waiting for", seqno); - auto new_secret_fetch = + request.awaiting_ledger_secrets = fetch_supporting_secret_if_needed(request.first_requested_seqno()); - if (new_secret_fetch != nullptr) - { - request.ledger_secret_recovery_info = std::move(new_secret_fetch); - } - else + if (!request.awaiting_ledger_secrets.has_value()) { // Newly have all required secrets - begin fetching the actual // entries. Note this is adding them to `all_stores`, from where @@ -665,33 +670,55 @@ namespace ccf::historical } bool handle_encrypted_past_ledger_secret( - const kv::StorePtr& store, - std::unique_ptr ledger_secret_recovery_info) + const kv::StorePtr& store, LedgerSecretPtr encrypting_secret) { // Read encrypted secrets from store auto tx = store->create_read_only_tx(); - auto encrypted_past_ledger_secret = + auto encrypted_past_ledger_secret_handle = tx.ro( ccf::Tables::ENCRYPTED_PAST_LEDGER_SECRET); - if (!encrypted_past_ledger_secret) + if (!encrypted_past_ledger_secret_handle) + { + return false; + } + + auto encrypted_past_ledger_secret = + encrypted_past_ledger_secret_handle->get(); + if (!encrypted_past_ledger_secret.has_value()) { return false; } // Construct description and decrypted secret auto previous_ledger_secret = - encrypted_past_ledger_secret->get()->previous_ledger_secret; + encrypted_past_ledger_secret->previous_ledger_secret; + if (!previous_ledger_secret.has_value()) + { + // The only write to this table that should not contain a previous + // secret is the initial service open + CCF_ASSERT_FMT( + encrypted_past_ledger_secret->next_version.has_value() && + encrypted_past_ledger_secret->next_version.value() == 1, + "Write to ledger secrets table at {} should contain a next_version " + "of 1", + store->current_version()); + return true; + } auto recovered_ledger_secret = std::make_shared( ccf::decrypt_previous_ledger_secret_raw( - ledger_secret_recovery_info->last_ledger_secret, - std::move(previous_ledger_secret->encrypted_data)), + encrypting_secret, std::move(previous_ledger_secret->encrypted_data)), previous_ledger_secret->previous_secret_stored_version); // Add recovered secret to historical secrets historical_ledger_secrets->set_secret( previous_ledger_secret->version, std::move(recovered_ledger_secret)); + // Update earliest_secret + CCF_ASSERT( + previous_ledger_secret->version < earliest_secret_.valid_from, ""); + earliest_secret_ = historical_ledger_secrets->get_first(); + return true; } @@ -738,8 +765,7 @@ namespace ccf::historical Request& request = it->second; - auto [earliest_ledger_secret_seqno, _] = - get_earliest_known_ledger_secret(); + update_earliest_known_ledger_secret(); // Update this Request to represent the currently requested ranges HISTORICAL_LOG( @@ -750,23 +776,13 @@ namespace ccf::historical *seqnos.begin(), include_receipts); request.adjust_ranges( - seqnos, include_receipts, earliest_ledger_secret_seqno); + seqnos, include_receipts, earliest_secret_.valid_from); // If the earliest target entry cannot be deserialised with the earliest // known ledger secret, record the target seqno and begin fetching the // previous historical ledger secret. - auto secret_fetch = + request.awaiting_ledger_secrets = fetch_supporting_secret_if_needed(request.first_requested_seqno()); - if (secret_fetch != nullptr) - { - if ( - request.ledger_secret_recovery_info == nullptr || - request.ledger_secret_recovery_info->target_seqno != - secret_fetch->target_seqno) - { - request.ledger_secret_recovery_info = std::move(secret_fetch); - } - } // Reset the expiry timer as this has just been requested request.time_to_expiry = ms_until_expiry; @@ -1050,6 +1066,23 @@ namespace ccf::historical const auto is_signature = deserialise_result == kv::ApplyResult::PASS_SIGNATURE; + update_earliest_known_ledger_secret(); + + auto [valid_from, secret] = earliest_secret_; + + if ( + secret->previous_secret_stored_version.has_value() && + secret->previous_secret_stored_version.value() == seqno) + { + HISTORICAL_LOG( + "Handling past ledger secret. Current earliest is valid from {}, now " + "processing secret stored at {}", + valid_from, + seqno); + handle_encrypted_past_ledger_secret(store, secret); + next_secret_fetch_handle = nullptr; + } + HISTORICAL_LOG( "Processing historical store at {} ({})", seqno, @@ -1168,9 +1201,8 @@ namespace ccf::historical bool public_only = false; for (const auto& [_, request] : requests) { - if ( - request.ledger_secret_recovery_info != nullptr && - request.ledger_secret_recovery_info->target_seqno == seqno) + const auto& als = request.awaiting_ledger_secrets; + if (als.has_value() && als.value() == seqno) { public_only = true; break; diff --git a/tests/e2e_suite.py b/tests/e2e_suite.py index 84c2ca743091..88bc09c50491 100644 --- a/tests/e2e_suite.py +++ b/tests/e2e_suite.py @@ -197,7 +197,10 @@ def filter_fun(x): def add(parser): parser.add_argument( - "--test-duration", help="Duration of full suite (s)", type=int + "--test-duration", + help="Duration of full suite (s)", + type=int, + required=True, ) parser.add_argument( "--test-suite", @@ -220,7 +223,6 @@ def add(parser): parser.add_argument( "--jinja-templates-path", help="Path to directory containing sample Jinja templates", - required=True, ) args = infra.e2e_args.cli_args(add) diff --git a/tests/suite/test_suite.py b/tests/suite/test_suite.py index b5c7c54d78ee..1016f8498b9d 100644 --- a/tests/suite/test_suite.py +++ b/tests/suite/test_suite.py @@ -138,10 +138,21 @@ governance_history.test_ledger_is_readable, governance_history.test_tables_doc, ] - suites["all"] = all_tests_suite +# https://github.com/microsoft/CCF/issues/5236 +regression_5236_suite = [ + recovery.test_recover_service, + memberclient.test_corrupted_signature, + e2e_operations.test_forced_snapshot, + recovery.test_recover_service, + e2e_logging.test_forwarding_frontends, + recovery.test_recover_service_aborted, +] +suites["regression_5236"] = regression_5236_suite + + # # Test functions are expected to be in the following format: # From 2bf49f2a8a33e2693d47a7a2c2d4a3fdfe38d1e0 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 18 May 2023 16:47:51 +0100 Subject: [PATCH 006/135] [release/4.x] Cherry pick: TLA CI validation fix: Call `apt update` (#5277) (#5278) --- .azure-pipelines-templates/trace_validation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines-templates/trace_validation.yml b/.azure-pipelines-templates/trace_validation.yml index a968cedda948..a07687656fc7 100644 --- a/.azure-pipelines-templates/trace_validation.yml +++ b/.azure-pipelines-templates/trace_validation.yml @@ -1,5 +1,7 @@ steps: - script: | + set -ex + sudo apt update sudo apt install -y default-jre python3 ./tla/install_deps.py displayName: "Install TLA dependencies" From 1b0e89d3d300b1960d9644bd9b721afb0f6b1e86 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 18 May 2023 17:21:46 +0100 Subject: [PATCH 007/135] [release/4.x] Cherry pick: Switch from pylint to ruff (#5270) (#5272) --- .ruff.toml | 2 ++ python/ccf/ledger.py | 2 +- python/ccf/ledger_viz.py | 22 +++++++++---------- python/ccf/read_ledger.py | 4 ++-- python/ledger_tutorial.py | 3 +-- scripts/ci-checks.sh | 4 ++-- tests/ca_certs.py | 2 +- tests/e2e_logging.py | 18 +++++++-------- tests/e2e_operations.py | 8 +++---- .../executors/logging_app/logging_app.py | 2 +- tests/external_executor/executors/util.py | 4 ++-- .../executors/wiki_cacher/wiki_cacher.py | 2 +- tests/governance.py | 7 +++--- tests/governance_history.py | 2 +- tests/infra/clients.py | 8 +++---- tests/infra/consortium.py | 4 ++-- tests/infra/logging_app.py | 12 +++++----- tests/infra/network.py | 2 +- tests/infra/piccolo_driver.py | 4 ++-- tests/infra/remote_client.py | 4 ++-- tests/js-modules/modules.py | 14 ++++++------ tests/lts_compatibility.py | 5 +++++ tests/raft_scenarios_runner.py | 8 +++---- tests/reconfiguration.py | 4 ++-- tests/recovery.py | 6 +++-- 25 files changed, 81 insertions(+), 72 deletions(-) create mode 100644 .ruff.toml diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000000..ac1a59c604b5 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,2 @@ +extend-exclude = ["*_pb2*.py"] +line-length = 2000 \ No newline at end of file diff --git a/python/ccf/ledger.py b/python/ccf/ledger.py index 37a94253273e..ed8f17ace1c2 100644 --- a/python/ccf/ledger.py +++ b/python/ccf/ledger.py @@ -507,7 +507,7 @@ def add_transaction(self, transaction): elif self.service_status == "Open": assert updated_status in ["Recovering"], updated_status else: - assert self.service_status == None, self.service_status + assert self.service_status is None, self.service_status self.service_status = updated_status # Checks complete, add this transaction to tree diff --git a/python/ccf/ledger_viz.py b/python/ccf/ledger_viz.py index 1655b81fe787..bf312da0d5cb 100644 --- a/python/ccf/ledger_viz.py +++ b/python/ccf/ledger_viz.py @@ -151,8 +151,8 @@ def main(): else None, ) - l = DefaultLiner(args.write_views, args.split_views, args.split_services) - l.help() + liner = DefaultLiner(args.write_views, args.split_views, args.split_services) + liner.help() current_service_identity = None for chunk in ledger: for tx in chunk: @@ -163,33 +163,33 @@ def main(): seqno = tx.gcm_header.seqno if not has_private: if ccf.ledger.SIGNATURE_TX_TABLE_NAME in public: - l.entry("Signature", view, seqno) + liner.entry("Signature", view, seqno) else: if all( table.startswith("public:ccf.internal.") for table in public ): - l.entry("Internal", view, seqno) + liner.entry("Internal", view, seqno) elif any(table.startswith("public:ccf.gov.") for table in public): service_info = try_get_service_info(public) if service_info is None: - l.entry("Governance", view, seqno) + liner.entry("Governance", view, seqno) elif service_info["status"] == "Opening": - l.entry("New Service", view, seqno) + liner.entry("New Service", view, seqno) current_service_identity = service_info["cert"] elif service_info["status"] == "Recovering": - l.entry("Recovering Service", view, seqno) + liner.entry("Recovering Service", view, seqno) current_service_identity = service_info["cert"] elif ( service_info["cert"] == current_service_identity and service_info["status"] == "Open" ): - l.entry("Service Open", view, seqno) + liner.entry("Service Open", view, seqno) else: - l.entry("User Public", view, seqno) + liner.entry("User Public", view, seqno) else: - l.entry("User Private", view, seqno) + liner.entry("User Private", view, seqno) - l.flush() + liner.flush() if __name__ == "__main__": diff --git a/python/ccf/read_ledger.py b/python/ccf/read_ledger.py index 842d68b2080d..ed1d0521e465 100644 --- a/python/ccf/read_ledger.py +++ b/python/ccf/read_ledger.py @@ -76,8 +76,8 @@ def print_key(key, table_name, tables_format_rules, indent_s, is_removed=False): LOG.info(f"{indent_s}{k}:") -def counted_string(l, name): - return f"{len(l)} {name}{'s' * bool(len(l) != 1)}" +def counted_string(string, name): + return f"{len(string)} {name}{'s' * bool(len(string) != 1)}" def dump_entry(entry, table_filter, tables_format_rules): diff --git a/python/ledger_tutorial.py b/python/ledger_tutorial.py index dce08b6f0351..aa35c4dd9e50 100644 --- a/python/ledger_tutorial.py +++ b/python/ledger_tutorial.py @@ -5,6 +5,7 @@ from loguru import logger as LOG import json import random +import ccf.ledger # Change default log format LOG.remove() @@ -19,12 +20,10 @@ ledger_dirs = sys.argv[1:] - # Because all ledger files are closed and are no longer being # written to, it is safe to read all of them, even those that may # contain uncommitted transactions. # SNIPPET_START: create_ledger -import ccf.ledger ledger = ccf.ledger.Ledger(ledger_dirs, committed_only=False) # SNIPPET_END: create_ledger diff --git a/scripts/ci-checks.sh b/scripts/ci-checks.sh index e6903efac861..2cc84bea2561 100755 --- a/scripts/ci-checks.sh +++ b/scripts/ci-checks.sh @@ -101,7 +101,7 @@ fi source scripts/env/bin/activate pip install -U pip -pip install -U wheel black pylint mypy 1>/dev/null +pip install -U wheel black pylint mypy ruff 1>/dev/null endgroup group "Python format" @@ -119,7 +119,7 @@ pip install -U -r python/requirements.txt 1>/dev/null endgroup group "Python lint" -PYTHONPATH=./tests git ls-files tests/ python/ | grep -e '\.py$' | xargs python -m pylint --ignored-modules "*_pb2" +ruff check python/ tests/ endgroup group "Python types" diff --git a/tests/ca_certs.py b/tests/ca_certs.py index 8bba6e3d61ed..208f3ed9527e 100644 --- a/tests/ca_certs.py +++ b/tests/ca_certs.py @@ -75,7 +75,7 @@ def test_cert_store(network, args): network.get_ledger_public_state_at(remove_proposal.completed_seqno)[ "public:ccf.gov.tls.ca_cert_bundles" ][raw_cert_name] - == None + is None ), "CA bundle was not removed" return network diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 22a4c2e664f8..5caeb641db96 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -1126,7 +1126,7 @@ def test_forwarding_frontends(network, args): ack = network.consortium.get_any_active_member().ack(backup) check_commit(ack) except AckException as e: - assert args.http2 == True + assert args.http2 is True assert e.response.status_code == http.HTTPStatus.NOT_IMPLEMENTED r = e.response.body.json() assert ( @@ -1134,7 +1134,7 @@ def test_forwarding_frontends(network, args): == "Request cannot be forwarded to primary on HTTP/2 interface." ), r else: - assert args.http2 == False + assert args.http2 is False try: msg = "forwarded_msg" @@ -1148,7 +1148,7 @@ def test_forwarding_frontends(network, args): msg=msg, ) except infra.logging_app.LoggingTxsIssueException as e: - assert args.http2 == True + assert args.http2 is True assert e.response.status_code == http.HTTPStatus.NOT_IMPLEMENTED r = e.response.body.json() assert ( @@ -1156,7 +1156,7 @@ def test_forwarding_frontends(network, args): == "Request cannot be forwarded to primary on HTTP/2 interface." ), r else: - assert args.http2 == False + assert args.http2 is False if args.package == "samples/apps/logging/liblogging" and not args.http2: with backup.client("user0") as c: @@ -1617,7 +1617,7 @@ def test_post_local_commit_failure(network, args): "/app/log/private/anonymous/v2?fail=false", {"id": 100, "msg": "hello"} ) assert r.status_code == http.HTTPStatus.OK.value, r.status_code - assert r.body.json()["success"] == True + assert r.body.json()["success"] is True TxID.from_str(r.body.json()["tx_id"]) r = c.post( @@ -1720,8 +1720,8 @@ def test_basic_constraints(network, args): basic_constraints = ca_cert.extensions.get_extension_for_oid( ObjectIdentifier("2.5.29.19") ) - assert basic_constraints.critical == True - assert basic_constraints.value.ca == True + assert basic_constraints.critical is True + assert basic_constraints.value.ca is True assert basic_constraints.value.path_length == 0 node_pem = primary.get_tls_certificate_pem() @@ -1729,8 +1729,8 @@ def test_basic_constraints(network, args): basic_constraints = node_cert.extensions.get_extension_for_oid( ObjectIdentifier("2.5.29.19") ) - assert basic_constraints.critical == True - assert basic_constraints.value.ca == False + assert basic_constraints.critical is True + assert basic_constraints.value.ca is False def run_udp_tests(args): diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 92aaf7f61f3d..f4ff012206a8 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -31,11 +31,11 @@ def test_save_committed_ledger_files(network, args): LOG.info(f"Moving committed ledger files to {args.common_read_only_ledger_dir}") primary, _ = network.find_primary() for ledger_dir in primary.remote.ledger_paths(): - for l in os.listdir(ledger_dir): - if infra.node.is_file_committed(l): + for ledger_file_path in os.listdir(ledger_dir): + if infra.node.is_file_committed(ledger_file_path): shutil.move( - os.path.join(ledger_dir, l), - os.path.join(args.common_read_only_ledger_dir, l), + os.path.join(ledger_dir, ledger_file_path), + os.path.join(args.common_read_only_ledger_dir, ledger_file_path), ) network.txs.verify(network) diff --git a/tests/external_executor/executors/logging_app/logging_app.py b/tests/external_executor/executors/logging_app/logging_app.py index 89a3b738bfaf..809dd825d104 100644 --- a/tests/external_executor/executors/logging_app/logging_app.py +++ b/tests/external_executor/executors/logging_app/logging_app.py @@ -113,7 +113,7 @@ def do_historical(self, table, request, response): response.body = e.details().encode() return - if result.retry == True: + if result.retry is True: response.status_code = HTTP.HttpStatusCode.ACCEPTED response.body = "Historical transaction is not currently available. Please retry.".encode() return diff --git a/tests/external_executor/executors/util.py b/tests/external_executor/executors/util.py index 28811453ef56..e3ebd001a6d1 100644 --- a/tests/external_executor/executors/util.py +++ b/tests/external_executor/executors/util.py @@ -13,7 +13,7 @@ def __init__(self, executor): self.activated_event = None def start(self): - assert self.thread == None, "Already started" + assert self.thread is None, "Already started" LOG.info("Starting executor") self.activated_event = threading.Event() self.thread = threading.Thread( @@ -25,7 +25,7 @@ def start(self): ), "Executor failed to activate after 3 seconds" def terminate(self): - assert self.thread != None, "Already terminated" + assert self.thread is not None, "Already terminated" LOG.info("Terminating executor") self.executor.terminate() self.thread.join() diff --git a/tests/external_executor/executors/wiki_cacher/wiki_cacher.py b/tests/external_executor/executors/wiki_cacher/wiki_cacher.py index 01df2f902be6..d81ace384a0c 100644 --- a/tests/external_executor/executors/wiki_cacher/wiki_cacher.py +++ b/tests/external_executor/executors/wiki_cacher/wiki_cacher.py @@ -74,7 +74,7 @@ def _execute_update_cache(self, kv_stub, request, response): prefix = "/update_cache/" title = request.uri[len(prefix) :] description = self._get_description(title) - if description == None: + if description is None: response.status_code = HTTP.HttpStatusCode.BAD_GATEWAY response.body = f"Error when fetching article with title '{title}'".encode( "utf-8" diff --git a/tests/governance.py b/tests/governance.py index fe8f2740808d..f67f7e96e0c9 100644 --- a/tests/governance.py +++ b/tests/governance.py @@ -633,9 +633,10 @@ def test_desc(s): test_desc("Logging levels of governance operations") consortium = network.consortium - rand_hex = lambda: md5( - random.getrandbits(32).to_bytes(4, "big") - ).hexdigest() + + def rand_hex(): + return md5(random.getrandbits(32).to_bytes(4, "big")).hexdigest() + validate_info = f"Logged at info during validate: {rand_hex()}" validate_warn = f"Logged at warn during validate: {rand_hex()}" validate_js = ( diff --git a/tests/governance_history.py b/tests/governance_history.py index c0fc558bff1f..c418e7c793fa 100644 --- a/tests/governance_history.py +++ b/tests/governance_history.py @@ -121,7 +121,7 @@ def check_signatures(ledger): sig_txid = TxID(tr.gcm_header.view, tr.gcm_header.seqno) # Adjacent signatures only occur on a view change - if prev_sig_txid != None: + if prev_sig_txid is not None: if prev_sig_txid.seqno + 1 == sig_txid.seqno: # Reduced from assert while investigating cause # https://github.com/microsoft/CCF/issues/5078 diff --git a/tests/infra/clients.py b/tests/infra/clients.py index d0472d73fc49..95229f94831e 100644 --- a/tests/infra/clients.py +++ b/tests/infra/clients.py @@ -446,7 +446,7 @@ def request( nf.write(msg_bytes) nf.flush() content_path = f"@{nf.name}" - if not "content-type" in headers and request.body: + if "content-type" not in headers and request.body: headers["content-type"] = content_type cmd = ["curl"] @@ -636,7 +636,7 @@ def request( request_body = json.dumps(request.body).encode() content_type = CONTENT_TYPE_JSON - if not "content-type" in request.headers and len(request.body) > 0: + if "content-type" not in request.headers and len(request.body) > 0: extra_headers["content-type"] = content_type if self.cose_signing_auth is not None and request.http_verb != "GET": @@ -837,10 +837,10 @@ def request( content_type = CONTENT_TYPE_JSON content_length = len(request_body) - if not "content-type" in request.headers and len(request.body) > 0: + if "content-type" not in request.headers and len(request.body) > 0: extra_headers["content-type"] = content_type - if not "content-length" in extra_headers: + if "content-length" not in extra_headers: extra_headers["content-length"] = content_length if self.signing_details is not None: diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 387bf7b851af..250c2374ea88 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -250,9 +250,9 @@ def get_active_non_recovery_members(self): def get_any_active_member(self, recovery_member=None): if recovery_member is not None: - if recovery_member == True: + if recovery_member is True: return random.choice(self.get_active_recovery_members()) - elif recovery_member == False: + elif recovery_member is False: return random.choice(self.get_active_non_recovery_members()) else: return random.choice(self.get_active_members()) diff --git a/tests/infra/logging_app.py b/tests/infra/logging_app.py index 725595a274d0..a029dd5da334 100644 --- a/tests/infra/logging_app.py +++ b/tests/infra/logging_app.py @@ -35,21 +35,21 @@ class LoggingTxsVerifyException(Exception): """ -def sample_list(l, n): - if n > len(l): +def sample_list(list_, n): + if n > len(list_): # Return all elements - return l + return list_ elif n == 0: return [] elif n == 1: # Return last element only - return l[-1:] + return list_[-1:] elif n == 2: # Return first and last elements - return l[:1] + l[-1:] + return list_[:1] + list_[-1:] else: # Return first, last, and random sample of values in-between - return l[:1] + random.sample(l[1:-1], n - 2) + l[-1:] + return list_[:1] + random.sample(list_[1:-1], n - 2) + list_[-1:] class LoggingTxs: diff --git a/tests/infra/network.py b/tests/infra/network.py index eb6f5bb6d7ff..7af7cb6c9a3d 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -1122,7 +1122,7 @@ def wait_for_all_nodes_to_commit(self, primary=None, tx_id=None, timeout=10): end_time = time.time() + timeout # If no TxID is specified, retrieve latest readable one - if tx_id == None: + if tx_id is None: while time.time() < end_time: with primary.client() as c: resp = c.get("/node/state") # Well-known read-only endpoint diff --git a/tests/infra/piccolo_driver.py b/tests/infra/piccolo_driver.py index ac135525f090..303fdae9c634 100644 --- a/tests/infra/piccolo_driver.py +++ b/tests/infra/piccolo_driver.py @@ -17,10 +17,10 @@ import json sys.path.insert(0, "../tests/perf-system/generator") -import generator +import generator # noqa: E402 sys.path.insert(0, "../tests/perf-system/analyzer") -import analyzer +import analyzer # noqa: E402 def get_command_args(args, network, get_command): diff --git a/tests/infra/remote_client.py b/tests/infra/remote_client.py index b227e8436db2..8dd6b805b8c8 100644 --- a/tests/infra/remote_client.py +++ b/tests/infra/remote_client.py @@ -87,13 +87,13 @@ def stop(self): remote_file_dst = f"{self.name}_{csv}" self.remote.get(csv, self.common_dir, 1, remote_file_dst) if csv == "perf_summary.csv": - with open("perf_summary.csv", "a", encoding="utf-8") as l: + with open("perf_summary.csv", "a", encoding="utf-8") as csvfd: with open( os.path.join(self.common_dir, remote_file_dst), "r", encoding="utf-8", ) as r: - l.write(r.read()) + csvfd.write(r.read()) def check_done(self): return self.remote.check_done() diff --git a/tests/js-modules/modules.py b/tests/js-modules/modules.py index 0fb92e66af6a..d3921db302e1 100644 --- a/tests/js-modules/modules.py +++ b/tests/js-modules/modules.py @@ -617,7 +617,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body try: infra.crypto.verify_signature( @@ -656,7 +656,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body try: infra.crypto.verify_signature( @@ -693,7 +693,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body try: infra.crypto.verify_signature( @@ -717,7 +717,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body r = c.post( "/app/verifySignature", @@ -729,7 +729,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == False, r.body + assert r.body.json() is False, r.body curves = [ec.SECP256R1, ec.SECP256K1, ec.SECP384R1] for curve in curves: @@ -747,7 +747,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body key_priv_pem, key_pub_pem = infra.crypto.generate_eddsa_keypair() algorithm = {"name": "EdDSA"} @@ -763,7 +763,7 @@ def test_npm_app(network, args): }, ) assert r.status_code == http.HTTPStatus.OK, r.status_code - assert r.body.json() == True, r.body + assert r.body.json() is True, r.body r = c.post( "/app/digest", diff --git a/tests/lts_compatibility.py b/tests/lts_compatibility.py index 6274504d62d2..91298349739d 100644 --- a/tests/lts_compatibility.py +++ b/tests/lts_compatibility.py @@ -475,6 +475,10 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): # Note: dicts are ordered from Python3.7 lts_releases[None] = None + ledger_dir = None + committed_ledger_dirs = None + snapshots_dir = None + jwt_issuer = infra.jwt_issuer.JwtIssuer( "https://localhost", refresh_interval=args.jwt_key_refresh_interval_s ) @@ -524,6 +528,7 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): network = infra.network.Network( **network_args, existing_network=network ) + network.start_in_recovery( args, ledger_dir, diff --git a/tests/raft_scenarios_runner.py b/tests/raft_scenarios_runner.py index 5951398419bf..f70bcb2c8cff 100644 --- a/tests/raft_scenarios_runner.py +++ b/tests/raft_scenarios_runner.py @@ -42,10 +42,10 @@ def separate_log_lines(text): if line.startswith(""): mermaid.append(line[len("") :]) elif '"raft_trace"' in line: - l = json.loads(line) - if "msg" in l: - if "configurations" in l["msg"]: - for config in l["msg"]["configurations"]: + line_ = json.loads(line) + if "msg" in line_: + if "configurations" in line_["msg"]: + for config in line_["msg"]["configurations"]: nodes.update(config["nodes"].keys()) log.append(line) return ( diff --git a/tests/reconfiguration.py b/tests/reconfiguration.py index 4f7372e05920..9215c828819a 100644 --- a/tests/reconfiguration.py +++ b/tests/reconfiguration.py @@ -160,13 +160,13 @@ def test_ignore_first_sigterm(network, args): with new_node.client() as c: r = c.get("/node/state") - assert r.body.json()["stop_notice"] == False, r + assert r.body.json()["stop_notice"] is False, r new_node.sigterm() with new_node.client() as c: r = c.get("/node/state") - assert r.body.json()["stop_notice"] == True, r + assert r.body.json()["stop_notice"] is True, r primary, _ = network.find_primary() network.retire_node(primary, new_node) diff --git a/tests/recovery.py b/tests/recovery.py index f8d254f9ac86..250154536346 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -390,8 +390,10 @@ def test_persistence_old_snapshot(network, args): # ledger directory (note: they used to be marked as ".ignored" by the new node) current_ledger_dir, committed_ledger_dirs = old_primary.get_ledger() for committed_ledger_dir in committed_ledger_dirs: - for l in os.listdir(committed_ledger_dir): - shutil.copy(os.path.join(committed_ledger_dir, l), current_ledger_dir) + for ledger_file_path in os.listdir(committed_ledger_dir): + shutil.copy( + os.path.join(committed_ledger_dir, ledger_file_path), current_ledger_dir + ) # Capture latest committed TxID on primary so we can check later that the # entire ledger has been fully recovered From f0f8915436e96368012167fa62e70f4c5a7aa9e1 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 19 May 2023 13:03:52 +0100 Subject: [PATCH 008/135] [release/4.x] Cherry pick: Timeout idle node-to-node channels (#5266) (#5284) --- .threading_canary | 2 +- src/consensus/aft/test/logging_stub.h | 3 + src/enclave/enclave.h | 6 + src/node/node_state.h | 7 ++ src/node/node_to_node.h | 3 + src/node/node_to_node_channel_manager.h | 100 ++++++++++++---- src/node/test/channels.cpp | 147 ++++++++++++++++++++++++ 7 files changed, 244 insertions(+), 24 deletions(-) diff --git a/.threading_canary b/.threading_canary index a286117986a1..bc32cd175b55 100644 --- a/.threading_canary +++ b/.threading_canary @@ -1 +1 @@ -This looks like a "job" for Threading Canary!!!!! +THIS looks like a job for Threading Canary!! diff --git a/src/consensus/aft/test/logging_stub.h b/src/consensus/aft/test/logging_stub.h index 047ef4853d8d..08fd1c42b238 100644 --- a/src/consensus/aft/test/logging_stub.h +++ b/src/consensus/aft/test/logging_stub.h @@ -253,6 +253,9 @@ namespace aft } void set_message_limit(size_t message_limit) override {} + void set_idle_timeout(std::chrono::milliseconds idle_timeout) override {} + + void tick(std::chrono::milliseconds elapsed) override {} bool recv_authenticated_with_load( const ccf::NodeId& from, const uint8_t*& data, size_t& size) override diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index 2de95d87b964..b1e7524e199e 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -214,6 +214,12 @@ namespace ccf node->set_n2n_message_limit(ccf_config_.node_to_node_message_limit); + // If we haven't heard from a node for multiple elections, then cleanup + // their node-to-node channel + const auto idle_timeout = + std::chrono::milliseconds(ccf_config_.consensus.election_timeout) * 4; + node->set_n2n_idle_timeout(idle_timeout); + ccf::NodeCreateInfo r; try { diff --git a/src/node/node_state.h b/src/node/node_state.h index 9b714e433d18..0a7467a1962a 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -1559,6 +1559,8 @@ namespace ccf const auto tx_id = consensus->get_committed_txid(); indexer->update_strategies(elapsed, {tx_id.first, tx_id.second}); } + + n2n_channels->tick(elapsed); } void tick_end() @@ -2596,6 +2598,11 @@ namespace ccf n2n_channels->set_message_limit(message_limit); } + void set_n2n_idle_timeout(std::chrono::milliseconds idle_timeout) + { + n2n_channels->set_idle_timeout(idle_timeout); + } + virtual const StartupConfig& get_node_config() const override { return config; diff --git a/src/node/node_to_node.h b/src/node/node_to_node.h index bb1b42499b26..6e2af37cbc4b 100644 --- a/src/node/node_to_node.h +++ b/src/node/node_to_node.h @@ -141,5 +141,8 @@ namespace ccf size_t size) = 0; virtual void set_message_limit(size_t message_limit) = 0; + virtual void set_idle_timeout(std::chrono::milliseconds idle_timeout) = 0; + + virtual void tick(std::chrono::milliseconds elapsed) = 0; }; } diff --git a/src/node/node_to_node_channel_manager.h b/src/node/node_to_node_channel_manager.h index 1e1b62de5bbf..414f0b09b5f2 100644 --- a/src/node/node_to_node_channel_manager.h +++ b/src/node/node_to_node_channel_manager.h @@ -14,7 +14,13 @@ namespace ccf ringbuffer::AbstractWriterFactory& writer_factory; ringbuffer::WriterPtr to_host; - std::unordered_map> channels; + struct ChannelInfo + { + std::shared_ptr channel; + std::chrono::milliseconds idle_time; + }; + + std::unordered_map channels; ccf::pal::Mutex lock; //< Protects access to channels map struct ThisNode @@ -36,6 +42,10 @@ namespace ccf std::nullopt; #endif + // This is set during node startup, using a value derived from the run-time + // configuration. Before that, no timeout applies + std::optional idle_timeout = std::nullopt; + std::shared_ptr get_channel(const NodeId& peer_id) { CCF_ASSERT_FMT( @@ -55,21 +65,23 @@ namespace ccf auto search = channels.find(peer_id); if (search != channels.end()) { - return search->second; + auto& channel_info = search->second; + channel_info.idle_time = std::chrono::milliseconds(0); + return channel_info.channel; } // Create channel - channels.try_emplace( + auto channel = std::make_shared( + writer_factory, + this_node->service_cert, + this_node->node_kp, + this_node->endorsed_node_cert.value(), + this_node->node_id, peer_id, - std::make_shared( - writer_factory, - this_node->service_cert, - this_node->node_kp, - this_node->endorsed_node_cert.value(), - this_node->node_id, - peer_id, - message_limit.value())); - return channels.at(peer_id); + message_limit.value()); + auto info = ChannelInfo{channel, std::chrono::milliseconds(0)}; + channels.try_emplace(peer_id, info); + return channel; } public: @@ -116,6 +128,41 @@ namespace ccf message_limit = message_limit_; } + void set_idle_timeout(std::chrono::milliseconds idle_timeout_) override + { + idle_timeout = idle_timeout_; + } + + void tick(std::chrono::milliseconds elapsed) override + { + std::lock_guard guard(lock); + + if (idle_timeout.has_value()) + { + // Close idle channels + auto it = channels.begin(); + while (it != channels.end()) + { + const auto idle_time = it->second.idle_time += elapsed; + if (idle_time < idle_timeout.value()) + { + ++it; + } + else + { + LOG_DEBUG_FMT( + "Closing idle channel to node {}. Was idle for {}, threshold for " + "closure is {}", + it->first, + idle_time, + idle_timeout.value()); + it->second.channel->close_channel(); + it = channels.erase(it); + } + } + } + } + virtual void associate_node_address( const NodeId& peer_id, const std::string& peer_hostname, @@ -129,22 +176,12 @@ namespace ccf peer_service); } - void close_channel(const NodeId& peer_id) override - { - get_channel(peer_id)->close_channel(); - } - bool have_channel(const ccf::NodeId& nid) override { std::lock_guard guard(lock); return channels.find(nid) != channels.end(); } - bool channel_open(const NodeId& peer_id) - { - return get_channel(peer_id)->channel_open(); - } - bool send_authenticated( const NodeId& to, NodeMsgType type, @@ -232,10 +269,27 @@ namespace ccf return get_channel(from)->recv_key_exchange_message(data, size); } - // NB: Only used by tests! + // NB: Following methods are only used by tests! bool recv_channel_message(const NodeId& from, std::vector&& body) { return recv_channel_message(from, body.data(), body.size()); } + + void close_channel(const NodeId& peer_id) override + { + std::lock_guard guard(lock); + + auto search = channels.find(peer_id); + if (search != channels.end()) + { + search->second.channel->close_channel(); + channels.erase(search); + } + } + + bool channel_open(const NodeId& peer_id) + { + return get_channel(peer_id)->channel_open(); + } }; } diff --git a/src/node/test/channels.cpp b/src/node/test/channels.cpp index e5a871501f2e..cba3ee51004c 100644 --- a/src/node/test/channels.cpp +++ b/src/node/test/channels.cpp @@ -1490,3 +1490,150 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Key rotation") REQUIRE(received_by_1 == expected_received_by_1); REQUIRE(received_by_2 == expected_received_by_2); } + +TEST_CASE_FIXTURE(IORingbuffersFixture, "Timeout idle channels") +{ + auto network_kp = crypto::make_key_pair(default_curve); + auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); + + auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_cert = + generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); + + auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_cert = + generate_endorsed_cert(channel2_kp, "CN=Node2", network_kp, service_cert); + + const auto idle_timeout = std::chrono::milliseconds(10); + const auto not_quite_idle = 2 * idle_timeout / 3; + + auto channels1 = NodeToNodeChannelManager(wf1); + channels1.initialize(nid1, service_cert, channel1_kp, channel1_cert); + channels1.set_idle_timeout(idle_timeout); + + auto channels2 = NodeToNodeChannelManager(wf2); + channels2.initialize(nid2, service_cert, channel2_kp, channel2_cert); + channels2.set_idle_timeout(idle_timeout); + + MsgType msg; + msg.fill(0x42); + + { + INFO("Idle channels are destroyed"); + REQUIRE_FALSE(channels1.have_channel(nid2)); + REQUIRE(channels1.send_authenticated( + nid2, NodeMsgType::consensus_msg, msg.begin(), msg.size())); + + REQUIRE_FALSE(channels2.have_channel(nid1)); + REQUIRE(channels2.send_authenticated( + nid1, NodeMsgType::consensus_msg, msg.begin(), msg.size())); + + REQUIRE(channels1.have_channel(nid2)); + REQUIRE(channels2.have_channel(nid1)); + + channels1.tick(not_quite_idle); + REQUIRE(channels1.have_channel(nid2)); + REQUIRE(channels2.have_channel(nid1)); + + channels1.tick(not_quite_idle); + REQUIRE_FALSE(channels1.have_channel(nid2)); + REQUIRE(channels2.have_channel(nid1)); + + channels2.tick(idle_timeout); + REQUIRE_FALSE(channels1.have_channel(nid2)); + REQUIRE_FALSE(channels2.have_channel(nid1)); + + // Flush previous messages + read_outbound_msgs(eio1); + read_outbound_msgs(eio2); + } + + // Send some messages from 1 to 2. Confirm that those keep the channel (on + // both ends) from being destroyed + bool handshake_complete = false; + + for (size_t i = 0; i < 20; ++i) + { + REQUIRE(channels1.send_authenticated( + nid2, NodeMsgType::consensus_msg, msg.begin(), msg.size())); + + auto msgs = read_outbound_msgs(eio1); + for (const auto& msg : msgs) + { + switch (msg.type) + { + case NodeMsgType::channel_msg: + { + channels2.recv_channel_message(msg.from, msg.data()); + break; + } + case NodeMsgType::consensus_msg: + { + auto hdr = msg.authenticated_hdr; + const auto* data = msg.payload.data(); + auto size = msg.payload.size(); + + REQUIRE(channels2.recv_authenticated( + msg.from, {hdr.data(), hdr.size()}, data, size)); + break; + } + default: + { + REQUIRE(false); + } + } + } + + if (!handshake_complete) + { + // Deliver any responses from 2 to 1, to complete handshake + msgs = read_outbound_msgs(eio2); + if (msgs.empty()) + { + handshake_complete = true; + } + else + { + for (const auto& msg : msgs) + { + switch (msg.type) + { + case NodeMsgType::channel_msg: + { + channels1.recv_channel_message(msg.from, msg.data()); + break; + } + default: + { + REQUIRE(false); + } + } + } + } + } + + { + INFO("Sends preserve channels"); + REQUIRE(channels1.have_channel(nid2)); + } + + { + INFO("Receives preserve channels"); + REQUIRE(channels2.have_channel(nid1)); + } + + channels1.tick(not_quite_idle); + channels2.tick(not_quite_idle); + } + + REQUIRE(handshake_complete); + + { + INFO("After comms, channels may still close due to idleness"); + channels1.tick(not_quite_idle); + REQUIRE_FALSE(channels1.have_channel(nid2)); + + channels2.tick(not_quite_idle); + REQUIRE_FALSE(channels2.have_channel(nid1)); + } +} From ffb08474f42feb1e96be19e55fbce9fcc72b9a8e Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Wed, 31 May 2023 16:48:30 +0100 Subject: [PATCH 009/135] [release/4.x] Cherry pick: Handle large snapshots (#5273) (#5312) --- .daily_canary | 2 +- src/consensus/ledger_enclave_types.h | 18 ++- src/ds/serializer.h | 21 ++++ src/enclave/enclave.h | 11 ++ src/host/main.cpp | 4 +- src/host/snapshots.h | 125 +++++++++++--------- src/host/test/ledger.cpp | 12 +- src/kv/kv_types.h | 4 +- src/kv/raw_serialise.h | 4 +- src/kv/snapshot.h | 4 +- src/kv/untyped_map.h | 2 +- src/node/node_state.h | 5 + src/node/snapshotter.h | 165 ++++++++++++++++++++------- src/node/test/snapshotter.cpp | 137 +++++++++++++++++----- tests/config.jinja | 2 +- tests/e2e_operations.py | 52 +++++++++ tests/infra/e2e_args.py | 6 + tests/infra/network.py | 1 + 18 files changed, 422 insertions(+), 153 deletions(-) diff --git a/.daily_canary b/.daily_canary index 6f567a4f2aeb..332142186300 100644 --- a/.daily_canary +++ b/.daily_canary @@ -1,4 +1,4 @@ --- ___ ___ - (- -) (o o) | Y & +-- + (- -) (o =) | Y & +-- ( V ) z x z O +---=---' /--x-m- /--n-m---xXx--/--yY------ diff --git a/src/consensus/ledger_enclave_types.h b/src/consensus/ledger_enclave_types.h index 7430ecf36711..1a45b7cb8d0f 100644 --- a/src/consensus/ledger_enclave_types.h +++ b/src/consensus/ledger_enclave_types.h @@ -4,6 +4,8 @@ #include "ds/ring_buffer_types.h" +#include + namespace consensus { using Index = uint64_t; @@ -31,9 +33,12 @@ namespace consensus DEFINE_RINGBUFFER_MSG_TYPE(ledger_init), DEFINE_RINGBUFFER_MSG_TYPE(ledger_open), - /// Create and commit a snapshot. Enclave -> Host - DEFINE_RINGBUFFER_MSG_TYPE(snapshot), + /// Ask for host memory allocation and commit a snapshot. Enclave -> Host + DEFINE_RINGBUFFER_MSG_TYPE(snapshot_allocate), DEFINE_RINGBUFFER_MSG_TYPE(snapshot_commit), + + /// Host -> Enclave + DEFINE_RINGBUFFER_MSG_TYPE(snapshot_allocated), }; } @@ -65,10 +70,15 @@ DECLARE_RINGBUFFER_MESSAGE_PAYLOAD( DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(consensus::ledger_commit, consensus::Index); DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(consensus::ledger_open); DECLARE_RINGBUFFER_MESSAGE_PAYLOAD( - consensus::snapshot, + consensus::snapshot_allocate, consensus::Index /* snapshot idx */, consensus::Index /* evidence idx */, - std::vector); + size_t /* size to allocate */, + uint32_t /* unique request id */); +DECLARE_RINGBUFFER_MESSAGE_PAYLOAD( + consensus::snapshot_allocated, + std::span, /* span to host-allocated memory for snapshot */ + uint32_t /* unique request id */); DECLARE_RINGBUFFER_MESSAGE_PAYLOAD( consensus::snapshot_commit, consensus::Index /* snapshot idx */, diff --git a/src/ds/serializer.h b/src/ds/serializer.h index c072f2af5237..7e7aed5ae0ce 100644 --- a/src/ds/serializer.h +++ b/src/ds/serializer.h @@ -6,6 +6,7 @@ #include "serialized.h" #include +#include #include #include #include @@ -256,6 +257,17 @@ namespace serializer return std::tuple_cat(std::make_tuple(cs), std::make_tuple(bfs)); } + /// Overload for std::span of bytes + /// Note that the memory region that the span points to is not copied + /// to the ring buffer (only the pointer and size are). + static auto serialize_value(std::span s) + { + auto csd = std::make_shared>( + reinterpret_cast(s.data())); + auto css = std::make_shared>(s.size()); + return std::tuple_cat(std::make_tuple(csd), std::make_tuple(css)); + } + /// Overload for strings (length-prefixed) static auto serialize_value(const std::string& s) { @@ -331,6 +343,15 @@ namespace serializer return serialized::read(data, size, prefixed_size); } + /// Overload for std::span of bytes + static std::span deserialize_value( + const uint8_t*& data, size_t& size, const Tag>&) + { + const auto ptr = serialized::read(data, size); + const auto s = serialized::read(data, size); + return {reinterpret_cast(ptr), s}; + } + /// Overload for strings static std::string deserialize_value( const uint8_t*& data, size_t& size, const Tag&) diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index b1e7524e199e..ed925a4d7578 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -403,6 +403,17 @@ namespace ccf } }); + DISPATCHER_SET_MESSAGE_HANDLER( + bp, + consensus::snapshot_allocated, + [this](const uint8_t* data, size_t size) { + const auto [snapshot_span, generation_count] = + ringbuffer::read_message( + data, size); + + node->write_snapshot(snapshot_span, generation_count); + }); + rpcsessions->register_message_handlers(bp.get_dispatcher()); // Maximum number of inbound ringbuffer messages which will be diff --git a/src/host/main.cpp b/src/host/main.cpp index 9da84a92adf9..303683ecdd00 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -299,7 +299,9 @@ int main(int argc, char** argv) ledger.register_message_handlers(bp.get_dispatcher()); asynchost::SnapshotManager snapshots( - config.snapshots.directory, config.snapshots.read_only_directory); + config.snapshots.directory, + writer_factory, + config.snapshots.read_only_directory); snapshots.register_message_handlers(bp.get_dispatcher()); // handle LFS-related messages from the enclave diff --git a/src/host/snapshots.h b/src/host/snapshots.h index bbf5118ab626..1be5fdf7ea7b 100644 --- a/src/host/snapshots.h +++ b/src/host/snapshots.h @@ -172,13 +172,24 @@ namespace asynchost class SnapshotManager { private: + ringbuffer::WriterPtr to_enclave; + const fs::path snapshot_dir; const std::optional read_snapshot_dir = std::nullopt; + struct PendingSnapshot + { + consensus::Index evidence_idx; + std::shared_ptr> snapshot; + }; + std::map pending_snapshots; + public: SnapshotManager( const std::string& snapshot_dir_, + ringbuffer::AbstractWriterFactory& writer_factory, const std::optional& read_snapshot_dir_ = std::nullopt) : + to_enclave(writer_factory.create_writer_to_inside()), snapshot_dir(snapshot_dir_), read_snapshot_dir(read_snapshot_dir_) { @@ -208,43 +219,18 @@ namespace asynchost return snapshot_dir; } - void write_snapshot( + std::shared_ptr> add_pending_snapshot( consensus::Index idx, consensus::Index evidence_idx, - const uint8_t* snapshot_data, - size_t snapshot_size) + size_t requested_size) { - TimeBoundLogger log_if_slow(fmt::format( - "Writing snapshot - idx={}, evidence_idx={}, size={}", - idx, - evidence_idx, - snapshot_size)); - - auto snapshot_file_name = fmt::format( - "{}{}{}{}{}", - snapshot_file_prefix, - snapshot_idx_delimiter, - idx, - snapshot_idx_delimiter, - evidence_idx); - auto full_snapshot_path = snapshot_dir / snapshot_file_name; - - if (fs::exists(full_snapshot_path)) - { - LOG_FAIL_FMT( - "Cannot write snapshot at {} since file already exists: {}", - idx, - full_snapshot_path); - return; - } + auto snapshot = std::make_shared>(requested_size); + pending_snapshots.emplace(idx, PendingSnapshot{evidence_idx, snapshot}); - LOG_INFO_FMT( - "Writing new snapshot to {} [{}]", snapshot_file_name, snapshot_size); + LOG_DEBUG_FMT( + "Added pending snapshot {} [{} bytes]", idx, requested_size); - std::ofstream snapshot_file( - full_snapshot_path, std::ios::out | std::ios::binary); - snapshot_file.write( - reinterpret_cast(snapshot_data), snapshot_size); + return snapshot; } void commit_snapshot( @@ -257,32 +243,47 @@ namespace asynchost try { - // Find previously-generated snapshot for snapshot_idx and rename file, - // also appending receipt to it - for (auto const& f : fs::directory_iterator(snapshot_dir)) + for (auto it = pending_snapshots.begin(); it != pending_snapshots.end(); + it++) { - auto file_name = f.path().filename().string(); - if ( - !is_snapshot_file_committed(file_name) && - get_snapshot_idx_from_file_name(file_name) == snapshot_idx) + if (snapshot_idx == it->first) { + // e.g. snapshot_100_105.committed + auto file_name = fmt::format( + "{}{}{}{}{}{}", + snapshot_file_prefix, + snapshot_idx_delimiter, + it->first, + snapshot_idx_delimiter, + it->second.evidence_idx, + snapshot_committed_suffix); auto full_snapshot_path = snapshot_dir / file_name; - const auto committed_file_name = - fmt::format("{}{}", file_name, snapshot_committed_suffix); - LOG_INFO_FMT( - "Committing snapshot file \"{}\" [{}]", - committed_file_name, - receipt_size); - - // Append receipt to snapshot file - std::ofstream snapshot_file( - full_snapshot_path, std::ios::app | std::ios::binary); - snapshot_file.write( - reinterpret_cast(receipt_data), receipt_size); - - fs::rename( - snapshot_dir / file_name, snapshot_dir / committed_file_name); + if (fs::exists(full_snapshot_path)) + { + // In the case that a file with this name already exists, keep + // existing file and drop pending snapshot + LOG_FAIL_FMT( + "Cannot write snapshot as file already exists: {}", file_name); + } + else + { + std::ofstream snapshot_file( + full_snapshot_path, std::ios::app | std::ios::binary); + const auto& snapshot = it->second.snapshot; + snapshot_file.write( + reinterpret_cast(snapshot->data()), + snapshot->size()); + snapshot_file.write( + reinterpret_cast(receipt_data), receipt_size); + + LOG_INFO_FMT( + "New snapshot file written to {} [{} bytes]", + file_name, + snapshot_file.tellp()); + } + + pending_snapshots.erase(it); return; } @@ -336,10 +337,22 @@ namespace asynchost messaging::Dispatcher& disp) { DISPATCHER_SET_MESSAGE_HANDLER( - disp, consensus::snapshot, [this](const uint8_t* data, size_t size) { + disp, + consensus::snapshot_allocate, + [this](const uint8_t* data, size_t size) { auto idx = serialized::read(data, size); auto evidence_idx = serialized::read(data, size); - write_snapshot(idx, evidence_idx, data, size); + auto requested_size = serialized::read(data, size); + auto generation_count = serialized::read(data, size); + + auto snapshot = + add_pending_snapshot(idx, evidence_idx, requested_size); + + RINGBUFFER_WRITE_MESSAGE( + consensus::snapshot_allocated, + to_enclave, + std::span{snapshot->data(), snapshot->size()}, + generation_count); }); DISPATCHER_SET_MESSAGE_HANDLER( diff --git a/src/host/test/ledger.cpp b/src/host/test/ledger.cpp index 11a4bda55186..f362d1047113 100644 --- a/src/host/test/ledger.cpp +++ b/src/host/test/ledger.cpp @@ -1302,7 +1302,7 @@ TEST_CASE("Generate and commit snapshots" * doctest::test_suite("snapshot")) auto snap_ro_dir = AutoDeleteFolder(snapshot_dir_read_only); fs::create_directory(snapshot_dir_read_only); - SnapshotManager snapshots(snapshot_dir, snapshot_dir_read_only); + SnapshotManager snapshots(snapshot_dir, wf, snapshot_dir_read_only); size_t snapshot_interval = 5; size_t snapshot_count = 5; @@ -1314,8 +1314,7 @@ TEST_CASE("Generate and commit snapshots" * doctest::test_suite("snapshot")) i += snapshot_interval) { // Note: Evidence is assumed to be at snapshot idx + 1 - snapshots.write_snapshot( - i, i + 1, dummy_snapshot.data(), dummy_snapshot.size()); + snapshots.add_pending_snapshot(i, i + 1, dummy_snapshot.size()); } REQUIRE_FALSE(snapshots.find_latest_committed_snapshot().has_value()); @@ -1358,11 +1357,8 @@ TEST_CASE("Generate and commit snapshots" * doctest::test_suite("snapshot")) INFO("Commit and retrieve new snapshot"); { size_t new_snapshot_idx = last_snapshot_idx + 1; - snapshots.write_snapshot( - new_snapshot_idx, - new_snapshot_idx + 1, - dummy_snapshot.data(), - dummy_snapshot.size()); + snapshots.add_pending_snapshot( + new_snapshot_idx, new_snapshot_idx + 1, dummy_snapshot.size()); snapshots.commit_snapshot( new_snapshot_idx, dummy_receipt.data(), dummy_receipt.size()); diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index 8560eb843492..e6b8f912a291 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -618,7 +618,7 @@ namespace kv public: virtual ~Snapshot() = default; virtual void serialise(KvStoreSerialiser& s) = 0; - virtual SecurityDomain get_security_domain() = 0; + virtual SecurityDomain get_security_domain() const = 0; }; using GetName::GetName; @@ -689,7 +689,7 @@ namespace kv virtual ~AbstractSnapshot() = default; virtual Version get_version() const = 0; virtual std::vector serialise( - std::shared_ptr encryptor) = 0; + const std::shared_ptr& encryptor) = 0; }; virtual ~AbstractStore() {} diff --git a/src/kv/raw_serialise.h b/src/kv/raw_serialise.h index 9f57ccb3d8e2..585d12e5f55f 100644 --- a/src/kv/raw_serialise.h +++ b/src/kv/raw_serialise.h @@ -15,7 +15,7 @@ namespace kv { private: // Avoid heap allocations for transactions which only touch a limited number - // keys of keys in a few maps + // of keys in a few maps using WriterData = llvm_vecsmall::SmallVector; WriterData buf; @@ -240,6 +240,4 @@ namespace kv } }; - using KvStoreSerialiser = GenericSerialiseWrapper; - using KvStoreDeserialiser = GenericDeserialiseWrapper; } diff --git a/src/kv/snapshot.h b/src/kv/snapshot.h index b170a6947624..d50302fb1f7e 100644 --- a/src/kv/snapshot.h +++ b/src/kv/snapshot.h @@ -33,13 +33,13 @@ namespace kv view_history = std::move(view_history_); } - Version get_version() const + Version get_version() const override { return version; } std::vector serialise( - std::shared_ptr encryptor) + const std::shared_ptr& encryptor) override { // Set the execution dependency for the snapshot to be the version // previous to said snapshot to ensure that the correct snapshot is diff --git a/src/kv/untyped_map.h b/src/kv/untyped_map.h index 4c1612ce2c8a..ed494b36e5bd 100644 --- a/src/kv/untyped_map.h +++ b/src/kv/untyped_map.h @@ -322,7 +322,7 @@ namespace kv::untyped s.serialise_raw(ret); } - SecurityDomain get_security_domain() override + SecurityDomain get_security_domain() const override { return security_domain; } diff --git a/src/node/node_state.h b/src/node/node_state.h index 0a7467a1962a..c983528b3bd9 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -2655,5 +2655,10 @@ namespace ccf }); client->send_request(std::move(req)); } + + void write_snapshot(std::span snapshot_buf, size_t request_id) + { + snapshotter->write_snapshot(snapshot_buf, request_id); + } }; } diff --git a/src/node/snapshotter.h b/src/node/snapshotter.h index 7bc2ce7f61fe..d667f6ab93ce 100644 --- a/src/node/snapshotter.h +++ b/src/node/snapshotter.h @@ -4,6 +4,7 @@ #include "ccf/ccf_assert.h" #include "ccf/ds/logger.h" +#include "ccf/pal/enclave.h" #include "ccf/pal/locking.h" #include "consensus/ledger_enclave_types.h" #include "ds/thread_messaging.h" @@ -21,10 +22,14 @@ namespace ccf class Snapshotter : public std::enable_shared_from_this, public kv::AbstractSnapshotter { - public: + private: static constexpr auto max_tx_interval = std::numeric_limits::max(); - private: + // Maximum number of pending snapshots allowed at a given time. No more + // snapshots are emitted when this threshold is reached and until pending + // snapshots are flushed on commit. + static constexpr auto max_pending_snapshots_count = 5; + ringbuffer::AbstractWriterFactory& writer_factory; ccf::pal::Mutex lock; @@ -36,9 +41,15 @@ namespace ccf struct SnapshotInfo { + kv::Version version; crypto::Sha256Hash write_set_digest; std::string commit_evidence; crypto::Sha256Hash snapshot_digest; + std::vector serialised_snapshot; + + // Prevents the receipt from being passed to the host (on commit) in case + // host has not yet allocated memory for the snapshot. + bool is_stored = false; std::optional evidence_idx = std::nullopt; @@ -51,7 +62,7 @@ namespace ccf }; // Queue of pending snapshots that have been generated, but are not yet // committed - std::map pending_snapshots; + std::map pending_snapshots; // Initial snapshot index static constexpr consensus::Index initial_snapshot_idx = 0; @@ -72,25 +83,6 @@ namespace ccf }; std::deque next_snapshot_indices; - void record_snapshot( - consensus::Index idx, - consensus::Index evidence_idx, - const std::vector& serialised_snapshot) - { - auto to_host = writer_factory.create_writer_to_outside(); - size_t max_message_size = to_host->get_max_message_size(); - if (serialised_snapshot.size() > max_message_size) - { - LOG_FAIL_FMT( - "Could not write snapshot of size {} > max ring buffer msg size {}", - serialised_snapshot.size(), - max_message_size); - return; - } - RINGBUFFER_WRITE_MESSAGE( - consensus::snapshot, to_host, idx, evidence_idx, serialised_snapshot); - } - void commit_snapshot( consensus::Index snapshot_idx, const std::vector& serialised_receipt) @@ -106,19 +98,32 @@ namespace ccf { std::shared_ptr self; std::unique_ptr snapshot; + uint32_t generation_count; }; static void snapshot_cb(std::unique_ptr> msg) { - msg->data.self->snapshot_(std::move(msg->data.snapshot)); + msg->data.self->snapshot_( + std::move(msg->data.snapshot), msg->data.generation_count); } void snapshot_( - std::unique_ptr snapshot) + std::unique_ptr snapshot, + uint32_t generation_count) { + if (pending_snapshots.size() >= max_pending_snapshots_count) + { + LOG_FAIL_FMT( + "Skipping new snapshot generation as {} snapshots are already " + "pending", + pending_snapshots.size()); + return; + } + auto snapshot_version = snapshot->get_version(); auto serialised_snapshot = store->serialise_snapshot(std::move(snapshot)); + auto serialised_snapshot_size = serialised_snapshot.size(); auto tx = store->create_tx(); auto evidence = tx.rw(Tables::SNAPSHOT_EVIDENCE); @@ -144,7 +149,8 @@ namespace ccf // transaction is committed. To allow for such scenario, the evidence // seqno is recorded via `record_snapshot_evidence_idx()` on a hook rather // than here. - pending_snapshots[snapshot_version] = {}; + pending_snapshots[generation_count] = {}; + pending_snapshots[generation_count].version = snapshot_version; auto rc = tx.commit(cd, false, nullptr, capture_ws_digest_and_commit_evidence); @@ -157,17 +163,27 @@ namespace ccf return; } - pending_snapshots[snapshot_version].commit_evidence = commit_evidence; - pending_snapshots[snapshot_version].write_set_digest = ws_digest; - pending_snapshots[snapshot_version].snapshot_digest = cd.value(); - auto evidence_version = tx.commit_version(); - record_snapshot(snapshot_version, evidence_version, serialised_snapshot); + pending_snapshots[generation_count].commit_evidence = commit_evidence; + pending_snapshots[generation_count].write_set_digest = ws_digest; + pending_snapshots[generation_count].snapshot_digest = cd.value(); + pending_snapshots[generation_count].serialised_snapshot = + std::move(serialised_snapshot); + + auto to_host = writer_factory.create_writer_to_outside(); + RINGBUFFER_WRITE_MESSAGE( + consensus::snapshot_allocate, + to_host, + snapshot_version, + evidence_version, + serialised_snapshot_size, + generation_count); LOG_DEBUG_FMT( - "Snapshot successfully generated for seqno {}, with evidence seqno {}: " - "{}, ws digest: {}", + "Request to allocate snapshot [{} bytes] for seqno {}, with evidence " + "seqno {}: {}, ws digest: {}", + serialised_snapshot_size, snapshot_version, evidence_version, cd.value(), @@ -184,11 +200,10 @@ namespace ccf for (auto it = pending_snapshots.begin(); it != pending_snapshots.end();) { - auto& snapshot_idx = it->first; auto& snapshot_info = it->second; if ( - snapshot_info.evidence_idx.has_value() && + snapshot_info.is_stored && snapshot_info.evidence_idx.has_value() && idx > snapshot_info.evidence_idx.value()) { auto serialised_receipt = build_and_serialise_receipt( @@ -201,7 +216,7 @@ namespace ccf snapshot_info.commit_evidence, std::move(snapshot_info.snapshot_digest)); - commit_snapshot(snapshot_idx, serialised_receipt); + commit_snapshot(snapshot_info.version, serialised_receipt); it = pending_snapshots.erase(it); } else @@ -257,6 +272,64 @@ namespace ccf next_snapshot_indices.push_back({last_snapshot_idx, false, true}); } + bool write_snapshot( + std::span snapshot_buf, uint32_t generation_count) + { + std::lock_guard guard(lock); + + auto search = pending_snapshots.find(generation_count); + if (search == pending_snapshots.end()) + { + LOG_FAIL_FMT( + "Could not find pending snapshot to write for generation count {}", + generation_count); + return false; + } + + auto& pending_snapshot = search->second; + if (snapshot_buf.size() != pending_snapshot.serialised_snapshot.size()) + { + // Unreliable host: allocated snapshot buffer is not of expected + // size. The pending snapshot is discarded to reduce enclave memory + // usage. + LOG_FAIL_FMT( + "Host allocated snapshot buffer [{} bytes] is not of expected " + "size [{} bytes]. Discarding snapshot for seqno {}", + snapshot_buf.size(), + pending_snapshot.serialised_snapshot.size(), + pending_snapshot.version); + pending_snapshots.erase(search); + return false; + } + else if (!ccf::pal::is_outside_enclave( + snapshot_buf.data(), snapshot_buf.size())) + { + // Sanitise host-allocated buffer. Note that buffer alignment is not + // checked as the buffer is only written to and never read. + LOG_FAIL_FMT( + "Host allocated snapshot buffer is not outside enclave memory. " + "Discarding snapshot for seqno {}", + pending_snapshot.version); + pending_snapshots.erase(search); + return false; + } + + ccf::pal::speculation_barrier(); + + std::copy( + pending_snapshot.serialised_snapshot.begin(), + pending_snapshot.serialised_snapshot.end(), + snapshot_buf.begin()); + pending_snapshot.is_stored = true; + + LOG_DEBUG_FMT( + "Successfully copied snapshot at seqno {} to host memory [{} " + "bytes]", + pending_snapshot.version, + pending_snapshot.serialised_snapshot.size()); + return true; + } + bool record_committable(consensus::Index idx) override { // Returns true if the committable idx will require the generation of a @@ -306,7 +379,7 @@ namespace ccf { std::lock_guard guard(lock); - for (auto& [snapshot_idx, pending_snapshot] : pending_snapshots) + for (auto& [_, pending_snapshot] : pending_snapshots) { if ( pending_snapshot.evidence_idx.has_value() && @@ -316,7 +389,7 @@ namespace ccf LOG_TRACE_FMT( "Recording signature at {} for snapshot {} with evidence at {}", idx, - snapshot_idx, + pending_snapshot.version, pending_snapshot.evidence_idx.value()); pending_snapshot.node_id = node_id; @@ -331,7 +404,7 @@ namespace ccf { std::lock_guard guard(lock); - for (auto& [snapshot_idx, pending_snapshot] : pending_snapshots) + for (auto& [_, pending_snapshot] : pending_snapshots) { if ( pending_snapshot.evidence_idx.has_value() && @@ -342,7 +415,7 @@ namespace ccf "Recording serialised tree at {} for snapshot {} with evidence at " "{}", idx, - snapshot_idx, + pending_snapshot.version, pending_snapshot.evidence_idx.value()); pending_snapshot.tree = tree; @@ -355,12 +428,14 @@ namespace ccf { std::lock_guard guard(lock); - for (auto& [snapshot_idx, pending_snapshot] : pending_snapshots) + for (auto& [_, pending_snapshot] : pending_snapshots) { - if (snapshot_idx == snapshot.version) + if (pending_snapshot.version == snapshot.version) { LOG_TRACE_FMT( - "Recording evidence idx at {} for snapshot {}", idx, snapshot_idx); + "Recording evidence idx at {} for snapshot {}", + idx, + pending_snapshot.version); pending_snapshot.evidence_idx = idx; } @@ -369,12 +444,14 @@ namespace ccf void schedule_snapshot(consensus::Index idx) { + static uint32_t generation_count = 0; auto msg = std::make_unique>(&snapshot_cb); msg->data.self = shared_from_this(); msg->data.snapshot = store->snapshot_unsafe_maps(idx); - static uint32_t generation_count = 0; + msg->data.generation_count = generation_count++; + auto& tm = threading::ThreadMessaging::instance(); - tm.add_task(tm.get_execution_thread(generation_count++), std::move(msg)); + tm.add_task(tm.get_execution_thread(generation_count), std::move(msg)); } void commit(consensus::Index idx, bool generate_snapshot) override diff --git a/src/node/test/snapshotter.cpp b/src/node/test/snapshotter.cpp index 57b8c730f547..8ea03f113185 100644 --- a/src/node/test/snapshotter.cpp +++ b/src/node/test/snapshotter.cpp @@ -32,7 +32,7 @@ auto read_ringbuffer_out(ringbuffer::Circuit& circuit) -1, [&idx](ringbuffer::Message m, const uint8_t* data, size_t size) { switch (m) { - case consensus::snapshot: + case consensus::snapshot_allocate: case consensus::snapshot_commit: { auto idx_ = serialized::read(data, size); @@ -49,18 +49,24 @@ auto read_ringbuffer_out(ringbuffer::Circuit& circuit) return idx; } -auto read_snapshot_out(ringbuffer::Circuit& circuit) +auto read_snapshot_allocate_out(ringbuffer::Circuit& circuit) { - std::optional> snapshot = std::nullopt; + std::optional> + snapshot_allocate_out = std::nullopt; circuit.read_from_inside().read( - -1, [&snapshot](ringbuffer::Message m, const uint8_t* data, size_t size) { + -1, + [&snapshot_allocate_out]( + ringbuffer::Message m, const uint8_t* data, size_t size) { switch (m) { - case consensus::snapshot: + case consensus::snapshot_allocate: { + auto idx = serialized::read(data, size); serialized::read(data, size); - serialized::read(data, size); - snapshot = std::vector(data, data + size); + auto requested_size = serialized::read(data, size); + auto generation_count = serialized::read(data, size); + + snapshot_allocate_out = {idx, requested_size, generation_count}; break; } case consensus::snapshot_commit: @@ -75,7 +81,7 @@ auto read_snapshot_out(ringbuffer::Circuit& circuit) } }); - return snapshot; + return snapshot_allocate_out; } void issue_transactions(ccf::NetworkState& network, size_t tx_count) @@ -170,9 +176,9 @@ TEST_CASE("Regular snapshotting") REQUIRE(read_ringbuffer_out(eio) == std::nullopt); } - INFO("Generate first snapshot"); + INFO("Malicious host"); { - REQUIRE(record_signature(history, snapshotter, snapshot_tx_interval)); + REQUIRE(record_signature(history, snapshotter, snapshot_idx)); // Note: even if commit_idx > snapshot_tx_interval, the snapshot is // generated at snapshot_idx @@ -181,17 +187,66 @@ TEST_CASE("Regular snapshotting") threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - REQUIRE( - read_ringbuffer_out(eio) == rb_msg({consensus::snapshot, snapshot_idx})); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + + // Incorrect generation count + { + auto snapshot = std::vector(snapshot_size); + REQUIRE_FALSE(snapshotter->write_snapshot(snapshot, snapshot_count + 1)); + } + + // Incorrect size + { + auto snapshot = std::vector(snapshot_size + 1); + REQUIRE_FALSE(snapshotter->write_snapshot(snapshot, snapshot_count)); + } + + // Even if snapshot is now valid, pending snapshot was previously + // discarded because of incorrect size + { + auto snapshot = std::vector(snapshot_size); + REQUIRE_FALSE(snapshotter->write_snapshot(snapshot, snapshot_count)); + } } - INFO("Commit first snapshot"); + INFO("Generate first snapshot"); { + issue_transactions(network, snapshot_tx_interval); + snapshot_idx = 2 * snapshot_idx; + REQUIRE(record_signature(history, snapshotter, snapshot_idx)); + + // Note: even if commit_idx > snapshot_tx_interval, the snapshot is + // generated at snapshot_idx + commit_idx = snapshot_idx + 1; + snapshotter->commit(commit_idx, true); + + threading::ThreadMessaging::instance().run_one(); + REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + + // Commit before snapshot is stored has no effect issue_transactions(network, 1); record_snapshot_evidence(snapshotter, snapshot_idx, snapshot_evidence_idx); - // Signature after evidence is recorded commit_idx = snapshot_idx + 2; REQUIRE_FALSE(record_signature(history, snapshotter, commit_idx)); + snapshotter->commit(commit_idx, true); + REQUIRE(read_ringbuffer_out(eio) == std::nullopt); + + // Correct size + auto snapshot = std::vector(snapshot_size, 0x00); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); + // Snapshot is successfully populated + REQUIRE(snapshot != std::vector(snapshot_size, 0x00)); + } + + INFO("Commit first snapshot"); + { snapshotter->commit(commit_idx, true); REQUIRE( read_ringbuffer_out(eio) == @@ -200,7 +255,7 @@ TEST_CASE("Regular snapshotting") INFO("Subsequent commit before next snapshot idx has no effect"); { - commit_idx = snapshot_tx_interval + 2; + commit_idx = snapshot_idx + 2; snapshotter->commit(commit_idx, true); threading::ThreadMessaging::instance().run_one(); REQUIRE(read_ringbuffer_out(eio) == std::nullopt); @@ -210,7 +265,7 @@ TEST_CASE("Regular snapshotting") INFO("Generate second snapshot"); { - snapshot_idx = snapshot_tx_interval * 2; + snapshot_idx = snapshot_tx_interval * 3; snapshot_evidence_idx = snapshot_idx + 1; REQUIRE(record_signature(history, snapshotter, snapshot_idx)); // Note: Commit exactly on snapshot idx @@ -219,8 +274,12 @@ TEST_CASE("Regular snapshotting") threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - REQUIRE( - read_ringbuffer_out(eio) == rb_msg({consensus::snapshot, snapshot_idx})); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + auto snapshot = std::vector(snapshot_size); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); } INFO("Commit second snapshot"); @@ -228,7 +287,7 @@ TEST_CASE("Regular snapshotting") issue_transactions(network, 1); record_snapshot_evidence(snapshotter, snapshot_idx, snapshot_evidence_idx); // Signature after evidence is recorded - commit_idx = snapshot_tx_interval * 2 + 2; + commit_idx = snapshot_idx + 2; REQUIRE_FALSE(record_signature(history, snapshotter, commit_idx)); snapshotter->commit(commit_idx, true); @@ -273,9 +332,13 @@ TEST_CASE("Rollback before snapshot is committed") threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - REQUIRE( - read_ringbuffer_out(eio) == - rb_msg({consensus::snapshot, snapshot_tx_interval})); + + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + auto snapshot = std::vector(snapshot_size); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); } INFO("Rollback evidence and commit past it"); @@ -304,8 +367,13 @@ TEST_CASE("Rollback before snapshot is committed") threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - REQUIRE( - read_ringbuffer_out(eio) == rb_msg({consensus::snapshot, snapshot_idx})); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx_, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + REQUIRE(snapshot_idx == snapshot_idx_); + auto snapshot = std::vector(snapshot_size); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); // Commit evidence issue_transactions(network, 1); @@ -330,8 +398,13 @@ TEST_CASE("Rollback before snapshot is committed") threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - REQUIRE( - read_ringbuffer_out(eio) == rb_msg({consensus::snapshot, snapshot_idx})); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx_, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + REQUIRE(snapshot_idx == snapshot_idx_); + auto snapshot = std::vector(snapshot_size); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); REQUIRE(!network.tables->flag_enabled( kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE)); @@ -413,9 +486,13 @@ TEST_CASE("Rekey ledger while snapshot is in progress") { threading::ThreadMessaging::instance().run_one(); REQUIRE(read_latest_snapshot_evidence(network.tables) == snapshot_idx); - auto snapshot = read_snapshot_out(eio); - REQUIRE(snapshot.has_value()); - REQUIRE(!snapshot->empty()); + auto snapshot_allocate_msg = read_snapshot_allocate_out(eio); + REQUIRE(snapshot_allocate_msg.has_value()); + auto [snapshot_idx_, snapshot_size, snapshot_count] = + snapshot_allocate_msg.value(); + REQUIRE(snapshot_idx == snapshot_idx_); + auto snapshot = std::vector(snapshot_size); + REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); // Snapshot can be deserialised to backup store ccf::NetworkState backup_network; @@ -434,7 +511,7 @@ TEST_CASE("Rekey ledger while snapshot is in progress") std::vector view_history; REQUIRE( backup_network.tables->deserialise_snapshot( - snapshot->data(), snapshot->size(), hooks, &view_history) == + snapshot.data(), snapshot.size(), hooks, &view_history) == kv::ApplyResult::PASS); } } diff --git a/tests/config.jinja b/tests/config.jinja index b97483aaad3a..b9ed8f94ead6 100644 --- a/tests/config.jinja +++ b/tests/config.jinja @@ -97,7 +97,7 @@ "worker_threads": {{ worker_threads }}, "memory": { "circuit_size": "4MB", - "max_msg_size": "16MB", + "max_msg_size": "{{ max_msg_size_bytes }}", "max_fragment_size": "64KB" }, "ignore_first_sigterm": {{ ignore_first_sigterm|tojson }}{% if node_to_node_message_limit %}, diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index f4ff012206a8..19230d56b791 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -150,6 +150,55 @@ def test_forced_snapshot(network, args): raise RuntimeError("Could not find matching snapshot file") +# https://github.com/microsoft/CCF/issues/1858 +@reqs.description("Generate snapshot larger than ring buffer max message size") +def test_large_snapshot(network, args): + primary, _ = network.find_primary() + + # Submit some dummy transactions + entry_size = 10000 # Lower bound on serialised write set size + iterations = int(args.max_msg_size_bytes) // entry_size + LOG.debug(f"Recording {iterations} large entries") + with primary.client(identity="user0") as c: + for idx in range(iterations): + c.post( + "/app/log/public?scope=test_large_snapshot", + body={"id": idx, "msg": "X" * entry_size}, + log_capture=[], + ) + + # Submit a proposal to force a snapshot at the following signature + proposal_body, careful_vote = network.consortium.make_proposal( + "trigger_snapshot", node_id=primary.node_id + ) + proposal = network.consortium.get_any_active_member().propose( + primary, proposal_body + ) + proposal = network.consortium.vote_using_majority( + primary, + proposal, + careful_vote, + ) + + # Check that there is at least a snapshot larger than args.max_msg_size_bytes + snapshots_dir = network.get_committed_snapshots(primary) + extra_data_size_bytes = 10000 # Upper bound on additional snapshot data (e.g. receipt) that is passed separately from the snapshot + for s in os.listdir(snapshots_dir): + snapshot_size = os.stat(os.path.join(snapshots_dir, s)).st_size + if snapshot_size > int(args.max_msg_size_bytes) + extra_data_size_bytes: + # Make sure that large snapshot can be parsed + snapshot = ccf.ledger.Snapshot(os.path.join(snapshots_dir, s)) + assert snapshot.get_len() == snapshot_size + LOG.info( + f"Found snapshot [{snapshot_size}] larger than ring buffer max msg size {args.max_msg_size_bytes}" + ) + return network + + raise RuntimeError( + f"Could not find any snapshot file larger than {args.max_msg_size_bytes}" + ) + + def split_all_ledger_files_in_dir(input_dir, output_dir): # A ledger file can only be split at a seqno that contains a signature # (so that all files end on a signature that verifies their integrity). @@ -211,6 +260,8 @@ def run_file_operations(args): json.dump(service_data, ntf) ntf.flush() + args.max_msg_size_bytes = f"{100 * 1024}" # 100KB + with tempfile.TemporaryDirectory() as tmp_dir: txs = app.LoggingTxs("user0") with infra.network.network( @@ -234,6 +285,7 @@ def run_file_operations(args): test_parse_snapshot_file(network, args) test_forced_ledger_chunk(network, args) test_forced_snapshot(network, args) + test_large_snapshot(network, args) primary, _ = network.find_primary() # Scoped transactions are not handled by historical range queries diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index d920a547c78d..3c79b56762ec 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -402,6 +402,12 @@ def cli_args(add=lambda x: None, parser=None, accept_unknown=False): type=int, default=1, ) + parser.add_argument( + "--max-msg-size-bytes", + help="Maximum message size (bytes) allowed on the ring buffer", + type=str, + default="16MB", + ) add(parser) diff --git a/tests/infra/network.py b/tests/infra/network.py index 7af7cb6c9a3d..9eb847fbb727 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -194,6 +194,7 @@ class Network: "snp_endorsements_servers", "node_to_node_message_limit", "tick_ms", + "max_msg_size_bytes", ] # Maximum delay (seconds) for updates to propagate from the primary to backups From 831964504de44869713b33ac8d19b1b6da360131 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:18:05 +0100 Subject: [PATCH 010/135] [release/4.x] Cherry pick: Fix SNP CI pipeline (#5310) (#5322) --- .snpcc_canary | 2 +- tests/code_update.py | 4 +++- tests/external_executor/external_executor.py | 5 ++++- tests/governance.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.snpcc_canary b/.snpcc_canary index 076a53ff62ab..34ebaf23d036 100644 --- a/.snpcc_canary +++ b/.snpcc_canary @@ -1,4 +1,4 @@ ___ ___ ___ (. =) Y (9 3) (* *) Y - O / O | / + O / . | / /-xXx--//-----x=x--/-xXx--/---x---->xxxx diff --git a/tests/code_update.py b/tests/code_update.py index 5c20a32ac2b6..bfc882690b15 100644 --- a/tests/code_update.py +++ b/tests/code_update.py @@ -524,7 +524,9 @@ def run(args): test_add_node_with_bad_code(network, args) # NB: Assumes the current nodes are still using args.package, so must run before test_proposal_invalidation test_proposal_invalidation(network, args) - test_update_all_nodes(network, args) + + if not snp.IS_SNP: + test_update_all_nodes(network, args) # Run again at the end to confirm current nodes are acceptable test_verify_quotes(network, args) diff --git a/tests/external_executor/external_executor.py b/tests/external_executor/external_executor.py index 462bc0290cc9..0ee97a040e90 100644 --- a/tests/external_executor/external_executor.py +++ b/tests/external_executor/external_executor.py @@ -5,6 +5,7 @@ import infra.interfaces import suite.test_requirements as reqs import queue +from infra.snp import IS_SNP from executors.logging_app.logging_app import LoggingExecutor @@ -524,7 +525,9 @@ def run(args): ) as network: network.start_and_open(args) - network = test_executor_registration(network, args) + if not IS_SNP: # UNKNOWN RPC failures + network = test_executor_registration(network, args) + network = test_parallel_executors(network, args) network = test_streaming(network, args) network = test_async_streaming(network, args) diff --git a/tests/governance.py b/tests/governance.py index f67f7e96e0c9..f565e764c2f0 100644 --- a/tests/governance.py +++ b/tests/governance.py @@ -50,7 +50,7 @@ def test_consensus_status(network, args): @reqs.description("Test quotes") @reqs.supports_methods("/node/quotes/self", "/node/quotes") def test_quote(network, args): - if args.enclave_platform == "virtual": + if args.enclave_platform != "sgx": LOG.warning("Quote test can only run in real enclaves, skipping") return network From bcfa6bb551d4d1fb0c407d82b51e284a98e2336f Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:02:30 +0100 Subject: [PATCH 011/135] Remove non-clang15 SNP/Virtual containers (#5316) --- .azure-pipelines-quictls.yml | 2 +- .azure-pipelines-templates/daily-matrix.yml | 13 +++++-------- .daily.yml | 4 ---- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci-checks.yml | 2 +- .github/workflows/ci-containers.yml | 6 ------ 6 files changed, 8 insertions(+), 21 deletions(-) diff --git a/.azure-pipelines-quictls.yml b/.azure-pipelines-quictls.yml index 31078339b9dc..42d3f5ba0c38 100644 --- a/.azure-pipelines-quictls.yml +++ b/.azure-pipelines-quictls.yml @@ -17,7 +17,7 @@ parameters: jobs: - job: build_quictls - container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual + container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-clang15 pool: 1es-dv4-focal strategy: diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index 12c4e524da4e..e32eec161154 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -3,9 +3,6 @@ parameters: Virtual: container: virtual pool: 1es-dv4-focal - VirtualClang15: - container: virtualclang15 - pool: 1es-dv4-focal SGX: container: sgx pool: 1es-DC16s_v3-focal @@ -42,7 +39,7 @@ parameters: jobs: - job: "Make" displayName: "Make generator" - ${{ insert }}: "${{ parameters.env.VirtualClang15 }}" + ${{ insert }}: "${{ parameters.env.Virtual }}" dependsOn: configure steps: - checkout: self @@ -58,7 +55,7 @@ jobs: - template: common.yml parameters: target: Virtual - env: "${{ parameters.env.VirtualClang15 }}" + env: "${{ parameters.env.Virtual }}" fetch_quictls: debug cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.SAN.cmake_args }} ${{ parameters.build.QUICTLS.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "Instrumented" @@ -70,7 +67,7 @@ jobs: - template: common.yml parameters: target: Virtual - env: "${{ parameters.env.VirtualClang15 }}" + env: "${{ parameters.env.Virtual}}" cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "ScanBuild" artifact_name: "Virtual_ScanBuild" @@ -99,8 +96,8 @@ jobs: - template: common.yml parameters: - target: VirtualClang15 - env: ${{ parameters.env.VirtualClang15 }} + target: Virtual + env: ${{ parameters.env.Virtual }} cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DCLIENT_PROTOCOLS_TEST=ON -DSHUFFLE_SUITE=ON" suffix: "Release" artifact_name: "Virtual_Release" diff --git a/.daily.yml b/.daily.yml index 23b519b321ea..cb15c22a6754 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,10 +25,6 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual - options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - - - container: virtualclang15 image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec8448c5c6c6..b1e7ef63dda3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual", + "image": "ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 2582ef300800..58def75f64e0 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual + container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.github/workflows/ci-containers.yml b/.github/workflows/ci-containers.yml index 21b0ce9a7b62..f89282bf3578 100644 --- a/.github/workflows/ci-containers.yml +++ b/.github/workflows/ci-containers.yml @@ -24,15 +24,9 @@ jobs: - name: Build CCF CI sgx container run: docker build -f docker/ccf_ci . --build-arg="platform=sgx" -t $ACR_REGISTRY/ccf/ci:${{steps.tref.outputs.tag}}-sgx - - name: Build CCF CI snp container - run: docker build -f docker/ccf_ci . --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:${{steps.tref.outputs.tag}}-snp - - name: Build CCF CI snp clang 15 container run: docker build -f docker/ccf_ci . --build-arg="platform=snp" --build-arg="clang_version=15" -t $ACR_REGISTRY/ccf/ci:${{steps.tref.outputs.tag}}-snp-clang15 - - name: Build CCF CI virtual container - run: docker build -f docker/ccf_ci . --build-arg="platform=virtual" -t $ACR_REGISTRY/ccf/ci:${{steps.tref.outputs.tag}}-virtual - - name: Build CCF CI virtual clang 15 container run: docker build -f docker/ccf_ci . --build-arg="platform=virtual" --build-arg="clang_version=15" -t $ACR_REGISTRY/ccf/ci:${{steps.tref.outputs.tag}}-virtual-clang15 From 3f8f4bc6214902f984dcd16c4e756c763145bc1d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 9 Jun 2023 15:27:02 +0000 Subject: [PATCH 012/135] Update ADO virtual pools --- .azure-pipelines-quictls.yml | 2 +- .azure-pipelines-templates/daily-matrix.yml | 4 ++-- .azure-pipelines-templates/matrix.yml | 4 ++-- .multi-thread.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines-quictls.yml b/.azure-pipelines-quictls.yml index 42d3f5ba0c38..abe469be4e5e 100644 --- a/.azure-pipelines-quictls.yml +++ b/.azure-pipelines-quictls.yml @@ -18,7 +18,7 @@ parameters: jobs: - job: build_quictls container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-clang15 - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub strategy: matrix: diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index e32eec161154..a6b536f7ec26 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -2,13 +2,13 @@ parameters: env: Virtual: container: virtual - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub SGX: container: sgx pool: 1es-DC16s_v3-focal SNPCC: container: snp - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub build: common: diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index a19ff93979c1..b4328a8dcc44 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -8,13 +8,13 @@ parameters: vmImage: ubuntu-20.04 Virtual: container: virtual - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub SGX: container: sgx pool: 1es-DC16s_v3-focal SNPCC: container: snp - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub build: common: diff --git a/.multi-thread.yml b/.multi-thread.yml index 27dbe5fc4056..19d96e67c6f9 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -27,7 +27,7 @@ jobs: target: Virtual env: container: virtual - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub cmake_args: "-DCOMPILE_TARGET=virtual -DWORKER_THREADS=2" cmake_env: "CC=`which clang-15` CXX=`which clang++-15`" suffix: "MultiThread" @@ -40,7 +40,7 @@ jobs: target: Virtual env: container: virtual - pool: 1es-dv4-focal + pool: ado-virtual-ccf-sub cmake_args: "-DCOMPILE_TARGET=virtual -DWORKER_THREADS=2 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DTSAN=ON -DLVI_MITIGATIONS=OFF" cmake_env: "CC=`which clang-15` CXX=`which clang++-15`" suffix: "MultiThreadTsan" From 18fd847273a2cfa516ab8c296071887e2e078c1a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 9 Jun 2023 15:16:22 +0000 Subject: [PATCH 013/135] Update SGX pools to new subscription --- .azure-pipelines-templates/daily-matrix.yml | 2 +- .azure-pipelines-templates/matrix.yml | 2 +- .azure-pipelines-templates/stress-matrix.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index a6b536f7ec26..4bc27077cc86 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -5,7 +5,7 @@ parameters: pool: ado-virtual-ccf-sub SGX: container: sgx - pool: 1es-DC16s_v3-focal + pool: ado-sgx-ccf-sub SNPCC: container: snp pool: ado-virtual-ccf-sub diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index b4328a8dcc44..4cdf0ce6413c 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -11,7 +11,7 @@ parameters: pool: ado-virtual-ccf-sub SGX: container: sgx - pool: 1es-DC16s_v3-focal + pool: ado-sgx-ccf-sub SNPCC: container: snp pool: ado-virtual-ccf-sub diff --git a/.azure-pipelines-templates/stress-matrix.yml b/.azure-pipelines-templates/stress-matrix.yml index b96b1d0a675f..017ec216516f 100644 --- a/.azure-pipelines-templates/stress-matrix.yml +++ b/.azure-pipelines-templates/stress-matrix.yml @@ -4,7 +4,7 @@ jobs: target: SGX env: container: sgx - pool: 1es-DC16s_v3-focal + pool: ado-sgx-ccf-sub cmake_args: "-DCOMPILE_TARGET=sgx" suffix: "StressTest" artifact_name: "StressTest" From f3390f2821d0e3d262bda927af67c2796607def6 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 9 Jun 2023 16:03:24 +0100 Subject: [PATCH 014/135] Update to CI pool in new subscription --- .github/workflows/build-ci-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml index 092cdc0654f4..b0e7c5d0213c 100644 --- a/.github/workflows/build-ci-container.yml +++ b/.github/workflows/build-ci-container.yml @@ -27,7 +27,7 @@ env: jobs: build: name: "Build and Publish Pre-built SNP Container" - runs-on: [self-hosted, 1ES.Pool=ccf-ci-github] + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] steps: - uses: actions/checkout@v3 with: From 7c3ad5847ef9c8c94d16ae4abcd6b49fc7f31d6b Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 12 Jun 2023 10:51:42 +0000 Subject: [PATCH 015/135] Migrate to new RG --- .azure-pipelines-templates/deploy_attestation_container.yml | 4 ++-- .azure-pipelines-templates/test_attestation_container.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines-templates/deploy_attestation_container.yml b/.azure-pipelines-templates/deploy_attestation_container.yml index b0f3f81fdbc2..95bc405f8e23 100644 --- a/.azure-pipelines-templates/deploy_attestation_container.yml +++ b/.azure-pipelines-templates/deploy_attestation_container.yml @@ -27,7 +27,7 @@ jobs: pip install -r ./scripts/azure_deployment/requirements.txt python3.8 scripts/azure_deployment/arm_template.py deploy aci \ --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) \ - --resource-group attestationContainer \ + --resource-group attestation-container \ --aci-type dynamic-agent \ --deployment-name ci-$(Build.BuildNumber) \ --attestation-container-e2e \ @@ -59,7 +59,7 @@ jobs: pip install -r ./scripts/azure_deployment/requirements.txt python3.8 scripts/azure_deployment/arm_template.py remove aci \ --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) \ - --resource-group attestationContainer \ + --resource-group attestation-container \ --aci-type dynamic-agent \ --deployment-name ci-$(Build.BuildNumber) name: cleanup_aci diff --git a/.azure-pipelines-templates/test_attestation_container.yml b/.azure-pipelines-templates/test_attestation_container.yml index bc8d6d07686d..9e461ddd4973 100644 --- a/.azure-pipelines-templates/test_attestation_container.yml +++ b/.azure-pipelines-templates/test_attestation_container.yml @@ -31,13 +31,13 @@ jobs: # See https://github.com/Azure/azure-cli/issues/13352 for rationale for use of "script return..." - script: | set -ex - script --return -c "az container exec --resource-group attestationContainer --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-attestation-container --exec-command 'attest.test --testdata-dir /testdata'" /dev/null - script --return -c "az container exec --resource-group attestationContainer --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-attestation-container --exec-command 'uvm.test'" /dev/null + script --return -c "az container exec --resource-group attestation-container --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-attestation-container --exec-command 'attest.test --testdata-dir /testdata'" /dev/null + script --return -c "az container exec --resource-group attestation-container --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-attestation-container --exec-command 'uvm.test'" /dev/null name: run_unit_test displayName: "Unit Test in Attestation Container Instance Deployed to ACIs" - script: | set -ex - script --return -c "az container exec --resource-group attestationContainer --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-dummy-business-logic-container --exec-command 'attestation-container.test -addr /mnt/uds/sock -test.v'" /dev/null + script --return -c "az container exec --resource-group attestation-container --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-dummy-business-logic-container --exec-command 'attestation-container.test -addr /mnt/uds/sock -test.v'" /dev/null name: test_attestation_container displayName: "Test attestation container with dummy business logic container Instance Deployed to ACIs" From f8b027a0d060d7f34612ba1aaf1b9d93a0d48077 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 15 Jun 2023 09:30:27 +0100 Subject: [PATCH 016/135] [release/4.x] Cherry pick: LTS compatibility: no longer run 1.x and 2.x nodes (#5349) (#5350) --- tests/infra/github.py | 19 +++-- tests/lts_compatibility.py | 69 +++++++++++++----- .../eol_service/common/member0_cert.pem | 12 +++ .../eol_service/common/member0_enc_privk.pem | 27 +++++++ .../eol_service/common/member0_enc_pubk.pem | 9 +++ .../eol_service/common/member0_privk.pem | 9 +++ .../eol_service/common/member1_cert.pem | 12 +++ .../eol_service/common/member1_enc_privk.pem | 27 +++++++ .../eol_service/common/member1_enc_pubk.pem | 9 +++ .../eol_service/common/member1_privk.pem | 9 +++ .../eol_service/common/member2_cert.pem | 12 +++ .../eol_service/common/member2_enc_privk.pem | 27 +++++++ .../eol_service/common/member2_enc_pubk.pem | 9 +++ .../eol_service/common/member2_privk.pem | 9 +++ .../eol_service/common/service_cert.pem | 12 +++ .../eol_service/common/user0_cert.pem | 12 +++ .../eol_service/common/user0_privk.pem | 9 +++ .../eol_service/ledger/ledger_1-2.committed | Bin 0 -> 35231 bytes .../eol_service/ledger/ledger_12-18.committed | Bin 0 -> 14962 bytes .../eol_service/ledger/ledger_19-23.committed | Bin 0 -> 5977 bytes .../eol_service/ledger/ledger_24-28.committed | Bin 0 -> 9254 bytes .../eol_service/ledger/ledger_29-33.committed | Bin 0 -> 10146 bytes .../eol_service/ledger/ledger_3-11.committed | Bin 0 -> 20079 bytes .../eol_service/ledger/ledger_34-35.committed | Bin 0 -> 1087 bytes .../eol_service/ledger/ledger_36-37.committed | Bin 0 -> 8086 bytes .../eol_service/ledger/ledger_38-47.committed | Bin 0 -> 147626 bytes .../eol_service/ledger/ledger_48-52.committed | Bin 0 -> 11592 bytes .../eol_service/ledger/ledger_53-57.committed | Bin 0 -> 5406 bytes .../eol_service/ledger/ledger_58-63.committed | Bin 0 -> 9270 bytes tests/testdata/eol_service/ledger/ledger_64 | Bin 0 -> 3219 bytes .../snapshots/snapshot_63_64.committed | Bin 0 -> 78794 bytes 31 files changed, 267 insertions(+), 25 deletions(-) create mode 100644 tests/testdata/eol_service/common/member0_cert.pem create mode 100644 tests/testdata/eol_service/common/member0_enc_privk.pem create mode 100644 tests/testdata/eol_service/common/member0_enc_pubk.pem create mode 100644 tests/testdata/eol_service/common/member0_privk.pem create mode 100644 tests/testdata/eol_service/common/member1_cert.pem create mode 100644 tests/testdata/eol_service/common/member1_enc_privk.pem create mode 100644 tests/testdata/eol_service/common/member1_enc_pubk.pem create mode 100644 tests/testdata/eol_service/common/member1_privk.pem create mode 100644 tests/testdata/eol_service/common/member2_cert.pem create mode 100644 tests/testdata/eol_service/common/member2_enc_privk.pem create mode 100644 tests/testdata/eol_service/common/member2_enc_pubk.pem create mode 100644 tests/testdata/eol_service/common/member2_privk.pem create mode 100644 tests/testdata/eol_service/common/service_cert.pem create mode 100644 tests/testdata/eol_service/common/user0_cert.pem create mode 100644 tests/testdata/eol_service/common/user0_privk.pem create mode 100644 tests/testdata/eol_service/ledger/ledger_1-2.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_12-18.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_19-23.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_24-28.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_29-33.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_3-11.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_34-35.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_36-37.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_38-47.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_48-52.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_53-57.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_58-63.committed create mode 100644 tests/testdata/eol_service/ledger/ledger_64 create mode 100644 tests/testdata/eol_service/snapshots/snapshot_63_64.committed diff --git a/tests/infra/github.py b/tests/infra/github.py index f6b9c98741be..de3a575dc06c 100644 --- a/tests/infra/github.py +++ b/tests/infra/github.py @@ -35,6 +35,12 @@ BACKPORT_BRANCH_PREFIX = "backport/" # Automatically added by backport CLI +# To be updated when major versions are end of life. We expect these +# versions to no longer be run in the recovery LTS compatibility test +# and corresponding ledgers should be copied to the testdata/ directory +# instead. +END_OF_LIFE_MAJOR_VERSIONS = [1, 2] + # Note: Releases are identified by tag since releases are not necessarily named, but all # releases are tagged @@ -244,7 +250,7 @@ def get_tags_for_release_branch(self, release_branch_name=None): ) return self.get_tags_for_major_version(major_version) - def get_lts_releases(self, branch): + def get_supported_lts_releases(self, branch): """ Returns a dict of all release major versions to the the latest release tag on that branch. Only release branches older than `branch` are included. @@ -262,7 +268,8 @@ def get_lts_releases(self, branch): tag = self.get_latest_tag_for_major_version(major_version) if tag is None: break - releases[major_version] = tag + if major_version not in END_OF_LIFE_MAJOR_VERSIONS: + releases[major_version] = tag major_version += 1 return releases @@ -517,10 +524,10 @@ def exp(prev=None, same=None): # All releases so far LOG.info(f"Finding all latest releases so far for local: {e.local_branch}") - lts_releases = repo.get_lts_releases(e.local_branch) + lts_releases = repo.get_supported_lts_releases(e.local_branch) if is_release_branch(e.local_branch) and lts_releases: - assert len(lts_releases) == get_major_version_from_release_branch_name( - e.local_branch - ) + assert len(lts_releases) + len( + END_OF_LIFE_MAJOR_VERSIONS + ) == get_major_version_from_release_branch_name(e.local_branch) LOG.success(f"Successfully verified scenario of size {len(test_scenario)}") diff --git a/tests/lts_compatibility.py b/tests/lts_compatibility.py index 91298349739d..089a01aee20f 100644 --- a/tests/lts_compatibility.py +++ b/tests/lts_compatibility.py @@ -18,9 +18,11 @@ from e2e_logging import test_random_receipts from governance import test_all_nodes_cert_renewal, test_service_cert_renewal from infra.snp import IS_SNP +from distutils.dir_util import copy_tree from loguru import logger as LOG + # Assumption: # By default, this assumes that the local checkout is not a non-release branch (e.g. main) # that is older than the latest release branch. This is to simplify the test, and assume @@ -464,8 +466,7 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): LOG.info("Use snapshot: {}", use_snapshot) repo = infra.github.Repository() - lts_releases = repo.get_lts_releases(local_branch) - has_pre_2_rc7_ledger = False + lts_releases = repo.get_supported_lts_releases(local_branch) LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}") @@ -516,13 +517,51 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): kwargs["reconfiguration_type"] = "OneTransaction" if idx == 0: - LOG.info(f"Starting new service (version: {version})") + LOG.info( + f"Recovering end-of-life service from files (version: {version})" + ) + # First, recover end-of-life services from files + expected_recovery_count = len( + infra.github.END_OF_LIFE_MAJOR_VERSIONS + ) + service_dir = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "testdata", + "eol_service", + ) + new_common = infra.network.get_common_folder_name( + args.workspace, args.label + ) + copy_tree(os.path.join(service_dir, "common"), new_common) + + new_ledger = os.path.join(new_common, "ledger") + copy_tree(os.path.join(service_dir, "ledger"), new_ledger) + + if use_snapshot: + new_snapshots = os.path.join(new_common, "snapshots") + copy_tree(os.path.join(service_dir, "snapshots"), new_snapshots) + network = infra.network.Network(**network_args) - network.start_and_open( + + args.previous_service_identity_file = os.path.join( + service_dir, "common", "service_cert.pem" + ) + + network.start_in_recovery( args, - set_authenticate_session=update_gov_authn(version), + ledger_dir=new_ledger, + committed_ledger_dirs=[new_ledger], + snapshots_dir=new_snapshots if use_snapshot else None, + common_dir=new_common, **kwargs, ) + + network.recover( + args, expected_recovery_count=expected_recovery_count + ) + + primary, _ = network.find_primary() + jwt_issuer.register(network) else: LOG.info(f"Recovering service (new version: {version})") network = infra.network.Network( @@ -586,28 +625,20 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): version, ) + snapshots_dir = ( + network.get_committed_snapshots(primary) if use_snapshot else None + ) + + network.save_service_identity(args) # We accept ledger chunk file differences during upgrades # from 1.x to 2.x post rc7 ledger. This is necessary because # the ledger files may not be chunked at the same interval # between those versions (see https://github.com/microsoft/ccf/issues/3613; # 1.x ledgers do not contain the header flags to synchronize ledger chunks). # This can go once 2.0 is released. - current_version_past_2_rc7 = primary.version_after("ccf-2.0.0-rc7") - has_pre_2_rc7_ledger = ( - not current_version_past_2_rc7 or has_pre_2_rc7_ledger - ) - is_ledger_chunk_breaking = ( - has_pre_2_rc7_ledger and current_version_past_2_rc7 - ) - - snapshots_dir = ( - network.get_committed_snapshots(primary) if use_snapshot else None - ) - - network.save_service_identity(args) network.stop_all_nodes( skip_verification=True, - accept_ledger_diff=is_ledger_chunk_breaking, + accept_ledger_diff=True, ) ledger_dir, committed_ledger_dirs = primary.get_ledger() diff --git a/tests/testdata/eol_service/common/member0_cert.pem b/tests/testdata/eol_service/common/member0_cert.pem new file mode 100644 index 000000000000..a35c298412ee --- /dev/null +++ b/tests/testdata/eol_service/common/member0_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBtTCCATygAwIBAgIUaJxS1rUh8VMs1VLiVkaKNbpOubYwCgYIKoZIzj0EAwMw +EjEQMA4GA1UEAwwHbWVtYmVyMDAeFw0yMzA2MTIwMDAwMDZaFw0yNDA2MTEwMDAw +MDZaMBIxEDAOBgNVBAMMB21lbWJlcjAwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATu +RLXmi9HK2TPyAg2/DRlrnPCGpmQNSsIDJhC1p4kfycVwd7u+CRCK2AVmpLE61HGy +8exj5KwtsZf5V+vHsd46610SWUzvXLColkxnrv7JnmMZ/XkGSIlAP6iRSd9JAwij +UzBRMB0GA1UdDgQWBBRHSq25h+78+Au5y/fZMvQJi8ChTjAfBgNVHSMEGDAWgBRH +Sq25h+78+Au5y/fZMvQJi8ChTjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMD +A2cAMGQCMHcL5n64uU69B87FnAakpoLIH8UTXOjVKhmMX5d67V7qz/0AphWJ4N+T +eZA6AnhrfQIwbYWhImA2envP5Nup4Fz/Yi87fLLi/jKNhOUXBcbu3zC+WiR7k7x9 +AOFMu/1b7jGq +-----END CERTIFICATE----- diff --git a/tests/testdata/eol_service/common/member0_enc_privk.pem b/tests/testdata/eol_service/common/member0_enc_privk.pem new file mode 100644 index 000000000000..17a2de9e585a --- /dev/null +++ b/tests/testdata/eol_service/common/member0_enc_privk.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA6NfN71CKrUX3gtFMDsR6DGrQZfF8vpSLtSpmFcjyQdKkDRga +B0uE5rAfcMdTKSqPEIaqgDmvjBnrQEjMoytdUclf419VunUly7Bzju3N9bDtMvd2 +LqD3X8rzNUquP+Gz3tI5nxMwxp0571tlf8Oa95QwRm5DF2ovXdmjCD3F8OCVnF64 +8V0gh15+s1Hq1ehHKTWu7XRyiZJQo2ksm/3fmj94TTCOmNzhfJxsfOF246cuj9sH +Zmjzy4p8dXDQcopbQV+CjRSmzj48k3lt7TA+6CT5OHt3Mx3DL/cNu+w6Oe+YJdnw +n7k+tXS+w3tJHSFRBjWrq9A9CaKupM2/y25RzwIDAQABAoIBAGnVDQlldmLjlmij +cQqVS1QxiJ+Nb8HACvBBYT0tpM8+SsqY1dvS+4lPLDs6nhXr/jLowr6deakYbtE9 +RaY0o6n0+dyDDJg/fvpiLWQduqudy3KK//CZwVaNDJaAZ0sp73Or4fzn4BemhxrA +VBYKdnlppbedjmkqHnVt4qhwS3P8ApLfCQmInCi3Dl5uUr+omGSOV36GbVerDWYu +z6wZXLRfhNVCREiCgdEFnseAp+G9KcmwQmEt46hfUlLZ/8aqlrfJB31ctRlIQZM0 +Y1UisfH5pyYloytV8T3ytGkg+OyYBC+wc5FSUaXqdtb7AAGnY3pd3u0VEYCR/a/w +8RYoBSECgYEA9K0v3SfqDO9PZt08ESWI9K7NpGnOaG0NS3PiovuRA2k/nQOrgWZk +Bi6GpURMhPcgocXumBsNNZVY1YKeCMj7G1DkJVB3lljWVCIzYIPHW4zltCbG/amT +UtCnCH9I4vHgxF0GrSNFOQBmLJsyKsWtW5K02xU/V1a6VON4OfcjmAsCgYEA855r +XI40f0iNrKNopxHwFIWxVLDWPHRhxoHSC9pxh0RivyklIOe/ngzyaCBrj7C8fYkb +Usi57UDb/2oSY0cnhlJRoz2b1wh9gY8Yac1S9sX0WrGMPkOk8CZaAPD2yfJ8wX+z +kzlM15tO4eXa2SL3MpdqLyHBahFX829S+FgmU80CgYA3uJ0241kk+vb9ORLt4Ltw +5cJFWCc+lt85OLn7XgknWre2/Vf4jyXmVoApcQVwHoua6+WrVDXM5yck8Ksm2wZ8 +sQ5UBSBG6BJfdK54o2oqSkshnNcie5Tmfkgb4kjqDiWe8oSLt4WTNqkOLYXeQch6 +G03+urcDH+gtXuRJZyP/AwKBgQDMa5ILCAhlBaxbwIKALj0V+1jhh8E82StyNlza +NWIt7uQtI4lUIyJWrOZ7GTkoSVTgwbt7q5LX0iTIHWlwhLRVsZ1vLLMnrKgjnr8N +iIylJVX1BCZ52Kj+GjBLueZbbm3gsBcu4lXoVRnL0+Pfb8edXBBtnBV/uy3hxvgI +C2+dwQKBgQCmSqtTUStD3pMIeQkr6jynlrLMI/8HyhN453ZBGGyQujI0AXddCT5u +E6GQL35FBmpfnTwQOeM/mIpjnHzESCGey2luBmD1bX12+Zv4e1qkK8SvUuWYZtT8 +hDsA+eGK67NB9AJQUvB2dZ4vJ+nYu6u/jy9MQloQ9Qqtc60quwEoNA== +-----END RSA PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/member0_enc_pubk.pem b/tests/testdata/eol_service/common/member0_enc_pubk.pem new file mode 100644 index 000000000000..76ee3a983b78 --- /dev/null +++ b/tests/testdata/eol_service/common/member0_enc_pubk.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6NfN71CKrUX3gtFMDsR6 +DGrQZfF8vpSLtSpmFcjyQdKkDRgaB0uE5rAfcMdTKSqPEIaqgDmvjBnrQEjMoytd +Uclf419VunUly7Bzju3N9bDtMvd2LqD3X8rzNUquP+Gz3tI5nxMwxp0571tlf8Oa +95QwRm5DF2ovXdmjCD3F8OCVnF648V0gh15+s1Hq1ehHKTWu7XRyiZJQo2ksm/3f +mj94TTCOmNzhfJxsfOF246cuj9sHZmjzy4p8dXDQcopbQV+CjRSmzj48k3lt7TA+ +6CT5OHt3Mx3DL/cNu+w6Oe+YJdnwn7k+tXS+w3tJHSFRBjWrq9A9CaKupM2/y25R +zwIDAQAB +-----END PUBLIC KEY----- diff --git a/tests/testdata/eol_service/common/member0_privk.pem b/tests/testdata/eol_service/common/member0_privk.pem new file mode 100644 index 000000000000..0b323a2ede9f --- /dev/null +++ b/tests/testdata/eol_service/common/member0_privk.pem @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDDNpyB4+k1FwfMhgIotbx6gY9U0sMrsu97rPoUGTg9n6uGoUpDEMc9b +7fFWp7EyzSmgBwYFK4EEACKhZANiAATuRLXmi9HK2TPyAg2/DRlrnPCGpmQNSsID +JhC1p4kfycVwd7u+CRCK2AVmpLE61HGy8exj5KwtsZf5V+vHsd46610SWUzvXLCo +lkxnrv7JnmMZ/XkGSIlAP6iRSd9JAwg= +-----END EC PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/member1_cert.pem b/tests/testdata/eol_service/common/member1_cert.pem new file mode 100644 index 000000000000..3325ea06bb35 --- /dev/null +++ b/tests/testdata/eol_service/common/member1_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBtTCCATygAwIBAgIUInY8vcgZhYyoPDKDBh/tZqC4O5QwCgYIKoZIzj0EAwMw +EjEQMA4GA1UEAwwHbWVtYmVyMTAeFw0yMzA2MTIwMDAwMDdaFw0yNDA2MTEwMDAw +MDdaMBIxEDAOBgNVBAMMB21lbWJlcjEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAS+ +iZM4Emi0YAwH6lUCxqHUGRzdLYq7/m2CGftYXPdG81mDzVjIHkC9KrKxwD05AeLw +mODZWYxO3KxiDwBqZdwRM2hDTWEmpCEZBhd+4FzfdFjCiOtUoVm8x2F8Stc3xLCj +UzBRMB0GA1UdDgQWBBQGZP/xIEGWCLYyJBZVbfuEXLL98DAfBgNVHSMEGDAWgBQG +ZP/xIEGWCLYyJBZVbfuEXLL98DAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMD +A2cAMGQCMA71vYE2mvrz4L6crOk7Y2OPsKVXCvpHAPKhkU4U3BVQb7+lNXotMJ4Z +a+d3C7pyNgIwag9ZHXmQ+MsiyVReF4GnjcLxBhPgx7mQ5U/Dnn5THn+wHqLfdhnC +IBsztdxDrldv +-----END CERTIFICATE----- diff --git a/tests/testdata/eol_service/common/member1_enc_privk.pem b/tests/testdata/eol_service/common/member1_enc_privk.pem new file mode 100644 index 000000000000..40abeeb39a33 --- /dev/null +++ b/tests/testdata/eol_service/common/member1_enc_privk.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuxk6tutcV4ygF2Jm+idDTZRBoUMu0KcGrzDuXZmRmM3nHF2W +69W8JLxK1UGGytb/h9zf1LzG12Aw8F+uFCw1C170fnwAtR2ak3mmeseZij3qn5b9 +rTqv8WgEAOzW2lcSG8fFt6G7RiSxOzDG4/zqhBW9NJOt72afExdzaQn9eqT9zNvK +fdKKlEJX5DPlXdaMjXCpiM0IN0eSoL/1hr8JecfOQNivB2huPJNCOHl3uJxkpf+3 +UYgHLMv5D3OvOGyX7bt6JDzg4UhtPogWT/h3uHVhMdvrJ9smPcvdo5pjhL8l2SHw +uhSeeqSAXHWadgzyhQOgnXInDSjkC4Omol0CtQIDAQABAoIBAFdoNPbpw60fBFeI +YCVznEys+3A0K7XMoiHMUT+nE874OoUU8AHjS086LBeu0iMHCD/XnzYKrzZFOPfD +K1c/Q9S5Gq3QjqAOYX2OMolfoEpmHKtL5d088p9sAZrRCprcXJhf3u2oAA6H68iE +RBaUvJbSdR8P8w1Juv03UnJ6+VGG4FszaLDH5exg+eCFLoCzDPqp1ZtL9wrlhYo1 +IFKcJSfLcgzabmPm03A+wbPicvIbNkf9Jf3DpOlNm5SHtCQ9pqiBa9vHPYRs7Jr5 +gSGhR1qNCkPOLwV5Rc4nse2t0TEW6i/RofT79PuqO+wQHdkXgQCZq4XIQN5b4qsx +8Cf4R0ECgYEA5RwJ/nm1eAHg1P5lYTYuAm26Mn8x1w/x0d/i9c6GUGgexQz44OjP +vh47cKmYPJgQYIbKTcFSZ3XHcVP92H1SyTkOOreWNfc+EA4/0IC/qk20uKt+sdkH +faK2h1VJr46Zw+keS3KqTIwqA9nzozr08umClm6s2uu1q9fbCGWj3dECgYEA0Q7n +Uev6TNK4884xlwLDbI1XF8zqJUPTZ2fzh0J0xrcaBj+iePTJrKO9GatoU7aZRiQ/ +OGyeQJz3utxij2rO7qc7IxpQ2jnDLvWeN+L8FzUelUUZlPONOFnRlZLBUsUOOCII +kGD1w0ovIx2teH7vWD7UU4qjj+tlWQlFErY6G6UCgYBs48KsO9RP0TS2hqJQbZu4 +c8ZHMcTSXjWkdjA8e4fCt3nIVb4DxyLVTTEa/r/oLK278tQ72GH94jCEYFMJjabH +FGL67T5rL7NDlkImKsrtejkme/ufM3Mn9ymhcJuw4KmcPzOy0DC4aPWfFvuA3QiG +Ww1ByiBFfDnADrt/Zi9XwQKBgDLTOb+77B1TOa3kWkAJZvN0JVFe+ynWJIZGSnl+ +2Z8aEtSkEd7wPDuhjy63cmSvxvG7jDcuiQbSYpPmUnvljquMsp7lHfUACAjZo/6Y +S4tk2auWohKJZ7Lke0Su6hsyVzOkaoqXWAiDd5RE+dCKo6vDkqFA2mT56KjNzSIw +dU4VAoGBAIxbR0Kn+xVjwtaOvXfWz/geNBTzTlXXQpwgSVEdzQbabvnFFFCLogft +RPp3xrqioKk+4BzanGylmhvOuBhFj0+y2BhBsDFeqwvr0hMPGbJyl3gxcgV04JJR +YUSQtN0QD4PIUQMmyGUf7bN1PqnobBxTGFQSuvyR3BlsUQ8j+iUa +-----END RSA PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/member1_enc_pubk.pem b/tests/testdata/eol_service/common/member1_enc_pubk.pem new file mode 100644 index 000000000000..fb20eefbda87 --- /dev/null +++ b/tests/testdata/eol_service/common/member1_enc_pubk.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuxk6tutcV4ygF2Jm+idD +TZRBoUMu0KcGrzDuXZmRmM3nHF2W69W8JLxK1UGGytb/h9zf1LzG12Aw8F+uFCw1 +C170fnwAtR2ak3mmeseZij3qn5b9rTqv8WgEAOzW2lcSG8fFt6G7RiSxOzDG4/zq +hBW9NJOt72afExdzaQn9eqT9zNvKfdKKlEJX5DPlXdaMjXCpiM0IN0eSoL/1hr8J +ecfOQNivB2huPJNCOHl3uJxkpf+3UYgHLMv5D3OvOGyX7bt6JDzg4UhtPogWT/h3 +uHVhMdvrJ9smPcvdo5pjhL8l2SHwuhSeeqSAXHWadgzyhQOgnXInDSjkC4Omol0C +tQIDAQAB +-----END PUBLIC KEY----- diff --git a/tests/testdata/eol_service/common/member1_privk.pem b/tests/testdata/eol_service/common/member1_privk.pem new file mode 100644 index 000000000000..9ef740350c22 --- /dev/null +++ b/tests/testdata/eol_service/common/member1_privk.pem @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAAdTWprEQGjaDQJ7GvVuG7swTI3LqejWROO6M2RaooB869saMgHr3N +fJmGAEmpTeCgBwYFK4EEACKhZANiAAS+iZM4Emi0YAwH6lUCxqHUGRzdLYq7/m2C +GftYXPdG81mDzVjIHkC9KrKxwD05AeLwmODZWYxO3KxiDwBqZdwRM2hDTWEmpCEZ +Bhd+4FzfdFjCiOtUoVm8x2F8Stc3xLA= +-----END EC PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/member2_cert.pem b/tests/testdata/eol_service/common/member2_cert.pem new file mode 100644 index 000000000000..afa6afe6c61e --- /dev/null +++ b/tests/testdata/eol_service/common/member2_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBtTCCATygAwIBAgIUc421QWJz501P0gn9heCBSzQMqOowCgYIKoZIzj0EAwMw +EjEQMA4GA1UEAwwHbWVtYmVyMjAeFw0yMzA2MTIwMDAwMDdaFw0yNDA2MTEwMDAw +MDdaMBIxEDAOBgNVBAMMB21lbWJlcjIwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQW +YCozNG6eaHw1F8nTL6AQ6DMHG47HHSnWOnP85N3H7DWgi8+Ub+8nmenmIuhE2rZH +h5AkYgPaAm+Dv7AtJC9vDd6pS7X2CsNTRVjZv880U8jWEvDptzlvH/4IQsffTT+j +UzBRMB0GA1UdDgQWBBT5Hl9uX56N30vKtQtTQc8JrnzwpDAfBgNVHSMEGDAWgBT5 +Hl9uX56N30vKtQtTQc8JrnzwpDAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMD +A2cAMGQCMBW2QQjYVPkrNHLr7AzGXdIwue2Ezdt/II3h+Zag5mmnRoq2EQCfxYJ/ +aByZdxjqggIwMI0Kl6El5l5JqBLnNo6DlsS6oaMA8hwvuWsRFFlMvom6jpZLn33X +cpG5+DFKPygU +-----END CERTIFICATE----- diff --git a/tests/testdata/eol_service/common/member2_enc_privk.pem b/tests/testdata/eol_service/common/member2_enc_privk.pem new file mode 100644 index 000000000000..b80ad13e0f73 --- /dev/null +++ b/tests/testdata/eol_service/common/member2_enc_privk.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA5gsjbOfKdsD5Pwt7DB4rWMWROEFZKRmOuXqUgHZY+sJt2NQG +EAuWkmDphoVHF5+YS4M10WNHH4bvfuJHoHHnRCZXFXdDvxCF39kCrQunlWGdGnQM +6gtGtoU/+fL7g1p6xIZypIvGSrwyCOvxWaSSIlhL8dFVuSNJ3WD5Sc51J5o09MZ+ +AE7LX7E6aYJnLZbW3NfnfN+JQdJVhlJWXqSt5c96zAYThP4p2Q2raQhecQkFNXDc +Bw10YKXPI3SmfRD0POdZ38ynEFNF2AlI9KS7ccYhgMsMm5fkxy7jSWUOlVH/bSvl +xpOnZzvTuflktfihxTbSqR158vQZyId431VFbwIDAQABAoIBAAb6Z4SyM04O5ICp +FH51m9MBizotoUrB37Z4pXCQJnrU0wg0xkpOT675CZxvBo0M5k3EGSnU3R9ztGgg +7RNM4TsL/rBEAeYV1jnIF3CwXdEGOQHIa2veE7MNr7chS8fcQsUbXvbm1okF1G64 +2YVinJw2GXeoZfVLwL3aZzV58cvVrvGe2XFRrsUKY8vbYFXuppaWvDxcIFmDqe7p +gHN8qJRD5PfWp94WBxVv71JGdwmHGJu3sDf7QQQmOnqLDB+uFFKxSrsMhReFKDiH +xfJo8RrlqbtTYSvlJU4HczdZMIwW7EZ48TvMi4iVLxEzM/ip8rzMIIiFXz/q4rsf +p2Yvq4ECgYEA/FzydI93Vv61ReOxrnQG0UT0X1jhC5lptRCboL22ZF/uIWiLlgTw +h/wbFMFDf/3bYUfpJQE4raSRZaeWN7iCphaZQRioyYHQY/Y26ZMkvyPQd1KwNZO+ +mEPZ15RMB5e0mqafjh1ld8T7yk6yLwYFkcc0r6j2gqo3kkA5KmEsy5ECgYEA6VvY +yxfX9VDd+N6Edp/wRtmmYyusgAWjaxLjmrdzr+gzmtboPgxtzbb1EPZZC8dg39TE +gPsVCL48xApaOeCaJudBjST7q5Z2jbR+m+FisBeEx2m2VVI9Emn+3YEFd6F7Wcdr +7fjKWC5DLCNjY4wpOVxqh1wbqdq/R2cIccwOgP8CgYBQy5k8RCpbCdUTeSCLz83W +9hKe7B+wzP8Q3y5UOijI3gWpYIVAFHjFsKsi6/UmZynzbUhxdoCk50k9mHf5gvMh +JZ8PPkvbp9oGBu/xvBBJubemXCzaGmEs8pwt7uagj6tcz/fJUhQIUmKilKtrbqwd +zf9KNfgmQUTlZ/ZlYRMsYQKBgB6KT7Acub24TKSicpGX0vliyLzX3WaP1lhQC4uX +EMPnzsEn+RMNy10N4brLktHQ0SPaE7GRqqoPQ6KgfDhw5ZvLzi9rtKg7M97NzXgX +IjRLj6wNcLhv3+BGEP3qQc3rS1pvl/d4BPtPaqkfJIXFtSt1pdd7PX+YSkpM+LTz +8L3/AoGBAPQUpLzBo+Ars8Zx2oXVqbKkvWkYxqbMyQ/QddcaBT5wwQg7v16qet45 +tta+AwbL5MQi/PKzjbTfyw7EPSb/DBMdj99jcsTNE9uuVfVxUy3ku9+nbdJMo6lp +RhycM1iuNgohsqylrnnTVe15UGWxW/zuNfaRdHvA7aA6cRbHX79b +-----END RSA PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/member2_enc_pubk.pem b/tests/testdata/eol_service/common/member2_enc_pubk.pem new file mode 100644 index 000000000000..2a934c56b2b5 --- /dev/null +++ b/tests/testdata/eol_service/common/member2_enc_pubk.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5gsjbOfKdsD5Pwt7DB4r +WMWROEFZKRmOuXqUgHZY+sJt2NQGEAuWkmDphoVHF5+YS4M10WNHH4bvfuJHoHHn +RCZXFXdDvxCF39kCrQunlWGdGnQM6gtGtoU/+fL7g1p6xIZypIvGSrwyCOvxWaSS +IlhL8dFVuSNJ3WD5Sc51J5o09MZ+AE7LX7E6aYJnLZbW3NfnfN+JQdJVhlJWXqSt +5c96zAYThP4p2Q2raQhecQkFNXDcBw10YKXPI3SmfRD0POdZ38ynEFNF2AlI9KS7 +ccYhgMsMm5fkxy7jSWUOlVH/bSvlxpOnZzvTuflktfihxTbSqR158vQZyId431VF +bwIDAQAB +-----END PUBLIC KEY----- diff --git a/tests/testdata/eol_service/common/member2_privk.pem b/tests/testdata/eol_service/common/member2_privk.pem new file mode 100644 index 000000000000..300c6cfd7a0c --- /dev/null +++ b/tests/testdata/eol_service/common/member2_privk.pem @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDCEIE9M83dvWkSTKzgofEIXEPMeaqtPqC6ulVAzgKNGbR0bbxP+KsnX +ZdZ7qLDZ8V2gBwYFK4EEACKhZANiAAQWYCozNG6eaHw1F8nTL6AQ6DMHG47HHSnW +OnP85N3H7DWgi8+Ub+8nmenmIuhE2rZHh5AkYgPaAm+Dv7AtJC9vDd6pS7X2CsNT +RVjZv880U8jWEvDptzlvH/4IQsffTT8= +-----END EC PRIVATE KEY----- diff --git a/tests/testdata/eol_service/common/service_cert.pem b/tests/testdata/eol_service/common/service_cert.pem new file mode 100644 index 000000000000..95bcfdae477a --- /dev/null +++ b/tests/testdata/eol_service/common/service_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBtzCCAT2gAwIBAgIQdAtmiSLQ5X1YVsXBHw3rbzAKBggqhkjOPQQDAzAWMRQw +EgYDVQQDDAtDQ0YgTmV0d29yazAeFw0yMzA2MTQwODMzNTdaFw0yMzA2MTUwODMz +NTZaMBYxFDASBgNVBAMMC0NDRiBOZXR3b3JrMHYwEAYHKoZIzj0CAQYFK4EEACID +YgAEJF1XvISMiXVbU6HDlHlaMwHRvFEfy62SkR3C4fDKreJdD2sF+weBNvjvxgBB +rX36H9XPmf9rth4Jj6YIrgOmzBRlLkmdmQMUPJR8o/Br/UF7AaQnclu+U+YM3bOs +rOF1o1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBRgJ4QHTEs9LSFdf9T8qmdv +vVAUvDAfBgNVHSMEGDAWgBRgJ4QHTEs9LSFdf9T8qmdvvVAUvDAKBggqhkjOPQQD +AwNoADBlAjEAiowoDdNYSne22niGAoLKKwSkcgAbNnRs1ELg/R8rrRJme5KHfo4w +Ueuv2OhJEBMpAjBlM9FCA5Jcqw2vUAl9HmPwD5dEjuUKf8c9kww7S0L3X3StPO4i +9S93Rt1gfVVBAqA= +-----END CERTIFICATE----- diff --git a/tests/testdata/eol_service/common/user0_cert.pem b/tests/testdata/eol_service/common/user0_cert.pem new file mode 100644 index 000000000000..68f6743ed767 --- /dev/null +++ b/tests/testdata/eol_service/common/user0_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBsjCCATigAwIBAgIUebgi0+Xlgn8xfFzCN7CunSZjN3gwCgYIKoZIzj0EAwMw +EDEOMAwGA1UEAwwFdXNlcjAwHhcNMjMwNjEyMDAwMDE0WhcNMjQwNjExMDAwMDE0 +WjAQMQ4wDAYDVQQDDAV1c2VyMDB2MBAGByqGSM49AgEGBSuBBAAiA2IABAFnQift +fYzNLKcV5nWtHC+gV6xtXSox37uu/+0FL/II5rCtnMZ3EMQYHLt2z9c0H1gToQRp +WP2OZHw6Fdp9tDTexEHkwjzflsOpd3no0fWOBZMtCBSeaT1vScnpz+yuYqNTMFEw +HQYDVR0OBBYEFN381PX7/mhtB1eOvf2lJNEYDgvFMB8GA1UdIwQYMBaAFN381PX7 +/mhtB1eOvf2lJNEYDgvFMA8GA1UdEwEB/wQFMAMBAf8wCgYIKoZIzj0EAwMDaAAw +ZQIwQEZGwkh+CuzxDLMC9w3e6a/8hy5SXfe9a1Lr4dEWy0pGjdb8uykD5LxDVGSr +oHC0AjEA4CknVcj2yhKMG/AB0nihcyJl9RwjcG+bb1JZ+BXuqPzL6Vfk3P8Ss/CB +Ely3H4sM +-----END CERTIFICATE----- diff --git a/tests/testdata/eol_service/common/user0_privk.pem b/tests/testdata/eol_service/common/user0_privk.pem new file mode 100644 index 000000000000..cbc0c62c33df --- /dev/null +++ b/tests/testdata/eol_service/common/user0_privk.pem @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDBnO3cr5Lq/5hm3/00dNVUwCPpbzfMBSB00Aa7RvZBYAoypyGRwjM7X +qWCCFW4nCXGgBwYFK4EEACKhZANiAAQBZ0In7X2MzSynFeZ1rRwvoFesbV0qMd+7 +rv/tBS/yCOawrZzGdxDEGBy7ds/XNB9YE6EEaVj9jmR8OhXafbQ03sRB5MI835bD +qXd56NH1jgWTLQgUnmk9b0nJ6c/srmI= +-----END EC PRIVATE KEY----- diff --git a/tests/testdata/eol_service/ledger/ledger_1-2.committed b/tests/testdata/eol_service/ledger/ledger_1-2.committed new file mode 100644 index 0000000000000000000000000000000000000000..9553c2c3e3c5cfc373fad71631cd663c22a29ebe GIT binary patch literal 35231 zcmeHw$&MsjmQYrqmRiv)T7nidlk7!3bw!rv3!kT?UR9aT^E`!TWW_VL!QFf?^Dr|X zvtx%ESg=DvY+%zLzyh)8C8E`97O1x@kdWvl3$~nduZ`~J5s@!nRfE*2R~7DN_nv$1 zna{a{|C@jLhWz`{n}7W$_~GBG%8&oY{(pZZ{ri9Rqw25!=kNcBzJBv#_`4=8;Dhzh zeD~%bnjhAi$x`Ou^SpFBRkydis_2GnYz$ddtftmq3&ihQ56r)YOg~)nOHS7rmdDP_ zr+bBA2smxS8>)7*<(9I*8R9#JTd$Xgx3mSr7&BSFwO%scGSlqzAksf60rd zdjQcXzMMJt(wP|o>td>Dd z;-l$Q;wJ!ER0MhdO~%HX(L@+h5qC^SoaXoI8;a2V^{;PkKK%N9@RvXN7efTx`baM~?@SFMgCpWkF`u-ZByd`rW)M2KYx>@Tm5+HuN;tYOvqlvVE zEHO7<(f2NLY3hz()jO=kyruzZzx(hM%0|fN$atsanqHk7wM;1_^aEjpQLVvs&>Qh4 z^p5!^?4mJ_L|Xya`)f|qxC5YO2|M8^iw|#Qo&2SEy|r7Q#9JJ;xl2g##H=>D!AwMk zQy2o+WG=%H^qv@Tsf)}B2C`y^Q&Bs`p|M#_L=BKA)A!#K4HNpah^UfD>92q!NE>^u zFV7VE5^8?^3I9AXlZ}B=PQO}!pBy3g5v;tjSy_U$5^>hLSGEXG%s02P!Y?<1sGri~ z6{spA0~me%$prO0Rv8s!&K%_(vs8EQn3+7Conr>C7$h7K+CTaulaxQlE}ucisV0Ky z)j+mDcE0`uon7cLL}qY!pOJn+@cN9Q0Jbv_0(Fh#!=9R`OhJQ9&0opNJ7&f0&*=bf z&at~)lHSbE8F>8}I9dvX41=EXYO_Md$qMQaSVhIq9hsu!0i-UE{|m(V`;O1;t?{|N z-}pX6Yp9BN!)a6991}$tzj`pVADmmaGih#V>RQ#ovI?qbIuV{1*WgQ(1K67B25ACJ zL8%3^_Vp(O{&ausP{{|&yaJ)bp)6R;^%(S1n}2_;8=9<4@2@T16VnSmyaxdSqtDdM zQb6Np%tU4;?!OTgUKOAN@{nV?B{mXIFoHZ4b>o)yfo{!)i(QmN@bvG0k6;QDHi5S5MX-%Zn5d<<<@;1Ly1VGkS=NlIz?AKQK%S{( z1Lu$IhX0Vy3lXHxV0JT^pP8J)$JB8{8%^X@O%MfqXGS*|uMx+Z*+`P8-K5YtpCO4s z>Q?MP_bmi3b?O$fAl-(AFie2~zFcSp;t(RzfPwR3`{DljB#^fJ2}r4SAYsb`ZSeqe z((kW7-b$*L=OEa-d3eVx-ZJ0*!io`aWR2TVq{@n;OS&BCvMBOuRdf(~>XKWx;hFRQSQx>}78s-^(1p@Oqx2Gi@b4S(oVwNTNH?QPOSKj`R?maG~gqzOsWEvnHy@CKFIHw`EKzM zEg!W+2m+#EFrW~*II*NN5;VVE$N~=Im|5>QAHM!1KYhejHXl#=8;95QHx3v2@8kS4 z(=>;NM&9mbe1H=m0K)G`D0E6CpYCpo&EGMTL`)DhLNpVY55+)aZO24Aykg3L7Ysjt z`eEKy|pk{s#YL^2Kt4$>uw~a-p1MdwKHo9@5!sCg8ND9Kf|HPs=S?lQH-VxHzJualDg3G@_aBm$f zh$OqsJ`n?IN73*DOJF(}Gyv?I4=6L_|4>P7gebcU6)?huAjFIOM?yMFzYsH@Xw4p)U=DEoXnZ!0 zw+>XChVu==zAU7iQ;gFwa&u#Z(WC<5I(easD-O~HUMJx^BpoWMC8L78*s}&Nsbj!l zX)bnLF^0-;TNUgAz?L>T)*iuCLbNLLvI^0{Rs|EWf>eQ63wZ5d=YV?F2)H;y`@SVq zqU9`)w9r`POXTFlSW-%ZtwTvcn`Z^~$euINqQgHxm6gNxH2Q$b7=CfR^TGfYu}h#U zvF<=ph_`8^$~dtr3B)1Z-ic5-L-Q&^C;{d}HXwVEbwjsk`UI4*K0l`t#1fj3on@fF zLG@I~7VSAZbcdOe9!`*Gb%JPy>b(WbD}R?XgSG}`1d@_y9Ic9lI8xR%(wR!wV>*~R zUA^O@i6yNZT4El6NdSbgB$BbyMI83BE?XFT0fCU)355Wq3q$nG!ZQyCN2>r$M>Ok( zpWN?XqT6LGP6_Ej!9z+qS1&QWaB*rb({h5gBWct%Q3?@{3nu*&)|u%988eF^uWrp| zs3g#gFvs=iLMJQp07YU+VdTU^y+kkcN|>LWQs+4_^Rv9JIl2hrl{s5gy7E+kgo)G& z{1C+Atp?@vTit3%Q~`5HhjhMP!g=l1Q51HVkZafr>E>(XoZ}a^V!Ej09?w^BiJh<~ z^h~JY#?%efvNeFfsTg5~xuEgP1Fh9S3wx|vO;rt-{JgBo?m0RnMe4X_?4vV&k^!(> z5^<&~R1?f-$k%^q;tGl(4^+aw#D&>qKeukt+|v?Y2=5${qtBq|z34GqWHCAGH6J#7 z2fD>hrz@zUJ1KjaO_uVaVmM#~_ZX(=*GVC%Oh$M{_RXawEJT(CakYjBMKLTQxXLF; z6{^j2ViYh5_g-B>abUoL6RBrWB!PYoP^Yp&3>;RyNC58OMA49St1x5(u?&lCYcX>E z%wm``%*R_OD|v|QyknSOGKRJh-A~>#5(kT3pmokuq$(^ML9|IHh{Z%K_>;+#k|Eao z%$JZzV8T!19t=5M-~rH%RGnc-9O52acw=;^Dneih`~v5Vi%A%-Vj0o}6RORkOw4h6+|pl!eGwo|+orSB*5s!`uTJT7^PqcoY(! zWWe3PE$wz`(Hi&}SwAl>-d}g=^hv}nPd6H=9>O=Q@Z$nDP@1eXxUB@DNOCv)K>tyh zq;oPSe?cuMp57_Y-j|Ditb_|@-@0i)h(wjhs|UJOI{t8&aEz=x#>TRO<9p@kiM zGqFQZl=?oXKfN^zFc$J&>YJVjUwXJ|nUhN<#InU^uzSHAzXZh_X1)+ayT)*Y!9AOo zMzfQQFGJdj7QP=8SXF^tIItJtA5ObM81z!2DkmF?mAsPFp}Cs_PqUgk>2W z7V@y9KoZOtl`i{@JA8Om zl7U>o-geFW*^(5t2H((kcd%uif zM6mpfgAux&?G-tB<+o034eS?V(pgyk`LxhMQd*yBP-E*SieyeUwyuetnK8|l&5xfg#x$}ud;1z=^PPLfuM|#`XrB$h zl8a0IU-F=2K=fO`ElQvHP8)->(z^ZT3piz)9IsBh=RtsAeh1{mjAIg*pXeTlgCV9P6A zJvfJpLTGR8#V>YN<`zOzh|SHW#G?9vf;|YDYJ%exOiBe6gQ+o7F(We(IAibXygvPb-0EJ7^IJ!UUfiF{^kPUd+Y znHZik$s&cd1Ozj^*H8Bi6e#Ra>`ys!fx)0ip6`X zH62mHlA=q7fEK79zNx2KlyRIbt@*%r0_96QuCuS+Cg$s$-)~@gw7Lbef~wPRU{01b ze{Rj~WLExVu;*FDDLPn6Re3N)@4-+Xa8u7;`;08ay42J4R5XgOlt2vi=K2Gi8T`J} zAS*X?+oGk=^qVbr?)-zka`Fxs6?_nVO2a*io79Z2K^5dZ0Ja+4!v3cXl=3~h>YUmj zg=95C*yMN_4nsk?oeQ{P_eNw2nQW`|g7Mv5i%RW`OZ$4~1^Yhfm``*|ucv}`-uYJH zR&ZAEWLvWb0%`IR7zY(dz`(QP+)q(43FHCgG1@+KhuxzzYkh{XP|~BVyp%X6y+b*R z)MiIK3Au(sxj{CPJG;)E88|IG=)0K-M@Zkno-YcBFtkawuiK1tgp7fMc(|$9#@qq# zG;;R%BG=ulvj*|b*AH1ZSV7OWg0J!*4tpbAatH*RskkK#0!2T}tpQI_0;Gs0Qc_1i zV_rgnAVi0Q<4+&IbFtMsM?}RtKXVYbE}eIQs_etFvKV=Cnw(I?O=27#8(QZ5f{IXR zRwI9SaxdwF`)fp6CGI0EOVNp|FD^#IT6~$HlLB+=qUn3sJChdkOSA>uu8? zFhlU3822=IGADc)L68w#HUfPHKl-@AwW+fa0bgAOj>;>vkBBd~+;MnD2R|z!EbPC8 zsLwF3!JFjwgi+`bxiuW?@NC+2)52`w&mjWHmqTW^hnXx0qH;EO@DFe%UgQ+qBzg+K zXLu9Nt0#8~l1vs4m@5x!6*uUL`q{9K^EvJ9M#u}~1%%wD0*Fk8J`g+MgOF|R&W(jz z!6*>qn2*B%;q@Zu>&raRKT#@3*uJtXEA`m7D4RDoD{??=MO+z4;w=59w!#n9)6DZb z+LZYzobxe_1AX%JQ?luTp6!Gq6cm9oNi*X-#M6Vg{rq*2GO@RGVBF3RCihaZVm1Qh za~WA`uPuB20tCC67X~Z`FSgd2w77?(9wKwggBoZzt^~%x@fPIA|EL}%hO4Dn-nNh6 z5n8z|zKxRS0LSSOKnmB>_9A3QODJt9`z}V6E?uB_<6aT3%$+3Rri+BTA4o(Q#8XPR zp&kNCB9AWa*!~n6!(j}Qxpqk#?~ia3!)i`}`E)Qxp?|zR=`;I82!aLxaCn>`1|Qwc zJyDvA0v~?UF^_w!FAY*~+XD$uD4<(I?ZZhKEz9ZR4V=fMane~pLi7!~`cN01B9R;c zLb3DO1{cS@FFZ`Q_=ruMar_qbu!Wm3IHE_)adm)VOQX3HMqaQybPB%J0&|h>d!%cB zW6##sR-xafJwuVO+3d?wOuu>V0oS`9sO7SwmoGY*b)HH>{0Hf!&x*>WtqUeT7#4WY zb7cNAqzcCd1tjG@{Or;>KU41-@T?_vcQn&B$oSYn%~E4ZQ>VF~BGWdWzECalTXZ+z z&LuCm?0x+vR&X|#WRo6UH^T~F0Wfvi3-z}r13x4SU(1-!Bd!@Nn7|z_Jx5eh)Y=&j zxSQ_FHZ(ZvmK2sZNQQMwa|8!3kUZ~3HHe;7tDuYBU1vBuTV+qjd9`i&%&uQTU$UzR9Ia!3~u%7eE3&CDA zbQ+unYc?IdZrw(mj(Dq2_YZJio4h(rz}y+A7X+QyS$}$?8MWG{3Azuyu8~hr_m*@h zui&WXM)a%bcu|9cIk>mUp^9r1Z%e;S$&o!WoFgH~*zbISk}?pYNQyviL3TMWN)tOj zvYi?}cg7@`!;J*4(w!-p`0xV%@zn|W2*e8BP2quf4#b6!%^$Z$oxsI5UlEW#nsU)cR57B z5l(-6@`?pC0Z(h+AxNf6&D05O25ubSXUvUriJ67QX})dQIoJIKa3SC2DV+@D#t&Hl z7~aw=`(Wc97#VV@AxOvROwQ2oEI8xg96GNPv|x1`E>r;e@8CW?`^Ln}*t*esmH`&- zpr-*w z#0v6!RKp{<{=J<*&5r9Xs;dVaBJAbkSC?ur7lJrEtmTCOmhEJ6fS$LBtHuJ7^SSc- z9qe++vgG~`Gi;=1g{a`7NyZA-DH2aVeYQksGumsTN$zZ_iy4R0Z!R)Yo}iVrxQL%! z4QMPz9n{Gy+85G!vPNX~cXdSuVV4skA(J3ltL%&xxxEz++=A*>;SwM?M268DltJtz zB?7<&)HPQ+KJDToMf8Q+E_o4ly7*-X76zXF7sA;2&Vbuu5OX^>u(5LH0^>Qr%xU=M z2UaLck0Czb30+(Q08frpeJYJ#lqpi=yn%vcSiK(mm0tN(2FA@UOV}tTF7~<`ISjHb z18wh+(aQNOZaH22a=8;e!DbOt;5@9M+kZR|KV#e1cN9gaXVDK&PIcN-lRlGUYd|0k zYRCjHE30^Jh=CIjRMk$O!c5rt6|N zIT0f>Tczwq)|}WobBm+nMck|h&QL1)nnPdgp5q872oJbX2`>H-aQX5FFct{u;q>k` zh=b{4x4F2)kINVT5D$frF6byJqoRuPSzsx6T2H8?-38+3zSmvrl5FV}uHB*F7qYq1fcr<0(b1f14IOK$qo z=*!B0=>UGV#G-+V@!q7-X3cZ9+4PnO?EfkA>|r?78*1@Px$^3OX&l``^*mq94#b+8 zu9AXxK)jM_e`8=w1A|u`UOeAun}!Fd$$o}upbCtg+^XK8N8&v@FaMe~5mEden)ngr z?OwqR6Pn>%cxc_m2RARSH-@PTcrwPiOA<&}={6g;q`GFdB6bfOiTDE+7~blTXzT0*Y)P?cLWjTShlB>2|~>5OmJ8`$vfw&}3D$%WMu-g^x@Qz-JVC)WJ< z1Q*`O;2-E+sN~`lbb3&SzE|K6N*=#(A31R0Lq|KkHOr092@dbUB~j-}F|+e8)81G} znYSyGr!-h+K2ZeUn+&$@O2@04;U_a7NaQt{T#IeDm_v{WKjeW)^;^dNk&shqgL4Av zatpWFnpd=1p)ol@NbY*HswGqc=$ZEc+^|HhXaO<6`ylmOx`K0avn3m=wzv57t%)TO zbN7K5dP2nl@Vbk?EpH04OZN`Zkv>r-P~@-;V1(x1FI%-QQ$^9lwzgdXDn4@owhLXK zd#@1za#_cUo0H#pL1m(3(m`VJq9l6rzpY`n*adMoCFU>S(o|!1D=k&{5gHVct>_w? z2{I8Ul{4ky?EX9u8U$~@#%=h-Sdtf~>?c0}bjEycXd#m4p`erykKokKzYoL%bd6;XAeS*iY4V_q6AT(4*Yj*{S?!MpK~(V zdW%=bSWT_Jf6M&vEAubMb1_axh>=h#A%-L&%}=|#JU*F(!+a_(rK7?th7iM1DI`e%E+oYxlS!0| zio7VrQo@8wPo#+$Pw*+2i4f)BO^D;e;YobLCnPSJNQ%klW?|0n%|9|fT)5;6w_*Io ziT$te$GzYmxeVO@KJlx1xtuY2*(}>TOxayI!%oY6uCnijwf-zMsO#ZDO&%<`YGbl) zZYFoT?DVc&RmbJyJd|g5_1#r|o^RLLSdk6);lZvn84irQ)!}4xi=8vnza9&y@H19NWxH8-om6uV*6R%ZM&h$TG8JGq@k3A= z*SGD8oXXC6b5_D>mb&$PF~<(4(Dv$?#x2qqz_U^+pXsqVwp|L=S+$tWJ{G(6SQ_Y+ zyJ90eTQ9b=^%7r;D~Z^qpGap?$%4Xii?v!Sms0)Ss5u{0XRG=sE+mqJ_2ySZB;UOGQZ@g+!d+JGQd@pHp4}bPR<4@M z%!0=FF&k^f+n*uaJwM?V{DfP0M!54Y2zNJdC6DW|{7MepvAa@Y+0X7DOZ{T!DAevA zlfhLaTa=8uQA;SM!mHeIFfW%D*>qK_?svIRoE2-kt5q{M9^UPn(du5#?J|#JVb`fg zX1U%lzglPWKc4>U(5Olce^-l1^4BAahR>$9V(ge zU?OeuqgpMU%AE>#yLk0XsKHr3A%?)eWQ+~pU9yFI+R%c{plF(GoLUAT}^dbI@GPUPyPVk}uIb(LXL zX{F+gXepT+PUTdfKMAB1_>HMmc{9sLv~lTb7H1cC(-z0B0=aFHH7eQkHYX(3-Q*~e z)f>IeU_RcaQlWloKFn`(YvZ`wmV&WzTbHC>FW@KKy?AMv-i+djMl`go8f~N3=2I0- zIqueH!rhBsJ=6Fzg*!8hwA=H$L2IEkN;NIX9*ZNPyxWM8{82E1<#Kcu7<1G3YNd44 z$4I`NmG*a);1!oSjD`LDaSFm+FNdnjM1C1x#w(ARn$l1cxuxDss9c>*&34<(Q1289 z%lcMbCFbjKO^HTFSNysd59A8f)?wOzPPk3O{pK6Wv-S6f4lMtR`NN~G zSf|y`)XG_=n!h6tNvKnvvyIGj0l|b^Oz%P&ww*7qY%|NYQ!Iu;*=ZF%=h;LsZ)Tv2O}r9x`E?$(U%dR5@(hqh2%6WZkC8Y}U2p;zrbw(@1} zahh9g=NUz7=jV0xUA^SN8SYejSP@!^TodHMy%aIH}Gmo-0Ju?MPV7v$<#?)yxi*LL!zLgr>7_JfMe5 zk703Es`iGPUc6Z{qV;_=R}1ouO<mpMJ0q;Rlw9&3=)9 zAHC%VvBR_wsjLFBkh|)QI~lcK--N1sQ9I@~qw%V#)hp9?!Uk&$*#e*>k z&eBIITss!S5q6g<1U7~2E_{^@Cqt65V~tLPTSQlIzEB*?^XQ`zpQN?k<2E&%=Go?P z7+Lb&VoE9)iDI%NclXU>t{4j*AFpPaVY*Ri8p#MJ<@dsoYb$B-v6ntJwpB@}R+st8 zD4uIAM-Yk3N7=Pp50x7sv8&dC;hC1IT#3BYY&Ya>CNkT!DvfNjw2W>l`^8!cMEiHs zQmwv?=c3JRvv?RKCq|-@J5FQ$nbA_G!(MO}z1oxpv%0X=DrtSy;*Zcdd^S;=+h3kbSa>N1qQGTN_}i7)l6Kd?;1r}})-l&XTB zi??=0GM9;ISHt?S)65sf)y}H989nx=rSV-ruNaX=yO?J;!^JAMo~eUUAs)Ev#_HkF zuu&?-CR=G!DXFEBa@EO>M}?7)+wQZ4XnK*=+8bp#EDA-XT~ADnA{^=o2BcbY8eS*% zKa%Svzl6ee`xpf2PR-z z*VyP#(O}J=fcW7m8?PGVOBLkVK`iq9q50t?1i(c%`0Z>TdRspW{wuSiz)j*pC>$4) zP)rggd?>+l(U2I8MTBrfiiG&IkP=hzSTvTD;$bcpg_2M#G)YHq`8@=J8osGs~Ev}f%@{7nep{9F^ob9K|Y4vcotL_@{tbVr|Z)%<4IK5M+ooer1 zi6qhnn{KP~&`3`$`08x7-w4xvWZP_S>*Z)E(ky1cNcA>+zn|R+g}Z63);^SK@osPv zJVq1uiW-jXMme@(7A5lRoMpY0k8XFDZ^&aQTv^J)=(crC#KMxXn839YJr*{3dTSqD zqi6IMVrm{}SJKseJ20Gx;~KY&_4Y?fr&1@M&*s?TY}W~vM-8Y0pB6vt+DB#`(uRX_ zAk^+|#bq;oCkNNvMI5YMMAt*9$IPHQmot-lWtE(z!D^R-wP~@I56hwbVmwG?3?4#g z{!keY#jPGz1Vi4GAIGqgFdrn#ca!95w9Jf;jaKMhNe9x2L%%d0tp)D5Ee5iaW=k;o znL9qnKlkf<%JnoClD?3Vqv>}gh+{vj;57Pqd!QE+D0Uo%pQADzFFlT)4QEC z<7VSxUOtSu!8kb0-R(;IX-K|T%DL*}-7y*4ONy9GbSEQ?7v@4m@A0ydn{kKAwwvfB z_`p6KSS&Pgni>S#nN>F$u8zCq=;QHT*{#RT-XcEj&H{4sm=f1BS<>?IZobQfquI^l z-73&{Y)+;X$VHcZA!te+T^QG4C%4{4>$0KQI{J4R2EpjdP_4`jCO#&>o(%-iR zi0~fPJK(1h;d3+p&@GN0wm1Umgzul;U2ALp(rEDNWHfq39`rQ#w_HD&KH<$jHb0zr z74LmyR%7e0Qs zk1vIiu;o|!t~BEt^*I#XkNG_<#185Nk1l2o zv;xge^TkYelgY3wC>0wivzcNbAIF0xWr@3)q#=;~hSM%H3J#qUZx(t8fe4!eTN`nI;(y4Oa44!GO_ zBaYefu5j0$ihOD`y}K(c`Gp$jspD{JSj|4htHr|pPSHcHYy*ZnmipV=BX{Ij2pK}< zNGjDRmAA)S93AbZl`NKoeOK=;2K~(Oanu_pkGXkirtJog{a|Q)Fy^anS?BxtZfaF( z+$)uYJ`rPqE-Uty&20HwmytP-l0Y$R-NXe}*r(>`=07mi$)%h2FN2W-ac8SpmSS3e z-&h~QmZtvb&0i7y_`Uz!`?&wJzxyA#-~Y?NkN@_k8&hfC;NQP+f5DCxlz(DE@k7Y| zwXKUtohs^oFu|g=SgmApKi&2F5V~vfU@MC|@*R$|%wXXae)XIe!HRvM`wjus(Y6Dm zFO7-Ou)&Xu0i<+1oPw%ONSH`MWCBZ=QcQ?P!jW)XiXTq7NUk4^b=Wbd2Bh^isY;JxA>S_V*>Yq!XeE=K$Go!5ls00zREZTz zk%GSo^wk(Z$(@8Z_4F`i_bGhNz%|e=Yltrb=6*^pw{q|cxk8&eo+w8ypj5HXU zD4MVBKb91o%0bI+K|WZ*=Y8t)?Z5x0KVSXxzx=Ns_@Do;Zwr6%p*#P3fBJv^UH=b% Z=l6dvru~cm_6L7ub;0re^iSTr`M(SVRJs5F literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_12-18.committed b/tests/testdata/eol_service/ledger/ledger_12-18.committed new file mode 100644 index 0000000000000000000000000000000000000000..c1f830c77a3bc7c5b65fe6f1e6ad0935c9c7bc65 GIT binary patch literal 14962 zcmeHO36LaNS?*mV+yfTyKoP;RVReQcr}M1J92}%7ud1xdtj@bCx1r6mGApk-vd(5z zLQq7+B}9%zKo?gdbj2eCK@FTfSAAJsHPbUY1G_r|Vz?dA zk=ZZb```cn@BjbzzwdvopV?adT(kABtLOV4_tm>!^s1NQFZk5wknk}d`l-hpecrkj zJU_U8-`X@@zizmB{Txi}ICRc9j(<8#b8nPd@#%4D4aPy1A9&Ny^-{;%5ZOGhS-)Jj zX;?oy>*wKiV2y+AGutFdqK@ymWw(raXod7_+<_djLY8fUAPKK*!H|PNC|UL$9P(Tk zg%I5GFx&MY#D(GQQ`=K-G!C`C37R6|5k_YL*AOmX^dgJXgw`})kIcT4Zz0rZ=jq>g1oZ1F67xRuh z%bktF+ymdUfd>{d+&*)7>#g6h=JEmoR?Gj|ht281(|>-~N!9?|Jz* zPV-qBFD_F2lP|;hZV-<1Y!o_H{Ln3^5aRs!%Pv9c0S5wKABq88K=!x=J?-lY0<72Z z!a@Gq4{KG<&wMZo^I${lnj`TH_u9&Zp7ARW zne#z5{sy|nrGnDNtB*0i()w#2Chyxt`^i#C)aTV8B>ZJwLG^V#bLn}i9- zClSlFe9Us)isQIGL=qV4R2&lb@v?_wu;V!>;^R0DVWf-WwhhA$f%_!tPKwPA{2Xpa z-r^uEc8x0~39L3L=!k8R0qV-<=BOZ=ulJ5dJab}U8*`Y5fW*k zwS|q=rVhf6odvNJ4L0nmVkD>yNLa>+z%|hBGMaY{J!sRtg~>@+ci97fYnXsE7A7r5 zZB7gYl}&`D9GQt8cKc$|HrPPy_Ykws$1Q{pY;Hbv>q67y;?(SjP&=%t3d7_=Uqkj3 zGE}+2L_+w9rw2_PjXARu2Xc~Oy3R9(HmACJEE}3R(USr1)f)?)Zp<_&BB?~2+Nm7d zsg&ECJ~2@_Yll@#sGdE0q=<LdWL2vnAs=~H8>Nt z6A`iua?+@qQiQ=fK)G7_2Ww)*`?63wosUA1#P)Mav zG5a{|fk_2UqXdbj5Twl-RjjRhzB?ErR<3uG#F!+OQjJI`G@NpcsO>QW+IL2SMia60 zE}ir>B3?FjAQz$phUUBWcsZ@oRMz%U5w%>jF*UUE($DJLOu|44((}cT>JEd}p3>&} zXm#k2&4xU#s)m5ifJ)9KWS4P)qg3bt8P&!GZalKwtz(JOcYpg)+a^^i7290s9{8gy zStUk={)ZE9v^Y_uN#L)OrNbLaB_~VAc?l#{i3d`+(%I7b>*-bB)8j!L<|np)X#3!|HO1PXQ^~6n7R1PgHbjsrQWQ>j7Hh$#5R>pVz-oLdC3n`xAeoI z`@^L$Ev+~&=DYQV8!k`>*kQAgdm^z0r%IuFT>fO-1MGqir-9l|fNzDg1*@wMfZIvx zdSDW7J?b38ho!=%&y)^JUcu9u(zckD#uFzffuOQoy0vs`L8;(=E#dhAfb^M_!1lyf zGbJAtlbt*oa-8#Z#(OE|3l4v<;QFXf!T3g{518&D@|TY$Uco%z#G4PY5zz5E313;( zI{&_uErl*n<=3~qLN9A3i_UBD3Im9yw}8-DEY%n!=fz50mW}gi&xM>EBYZKZ4}#VO zDb$68kBoohjVE!w7W07<6oM5UP8YUu8i-Qlb)a`4+Ac!0f0!MUC zhwkaT@DXs#J8|;lsmt~&TnKzzO>vP6d<}~}5@nSEJ$y+n*4+Q5Ko_2pFO=!Xgde)G zTo{KQuzr_!)D1J2(C^jd<>D1M$I^Pc+0I7vTXCG_0F*e~wj2xOuw@T{h_VOyF6r17>=G8{puR_<2#Ep0UKw)z3IQ-5$U-0z ztH3S^5?&IpQDjvYh3RRQd!-X2Yqqx*?^NkXP;b;Iwy{n0J@N~XCtY#&d&NB53gDa& z0BHvW?^j7Lg?Lv(wt^1eZ==Ga1A_;nKIHqr1|T0tz%pAXi1t24xVA;wzU|>6gftW3RgLrnMm!cEqfDfR|0<2DE$Xy3;XJKMzxez0?BvmUr0%a|7M2iEO=b?F5J0 zPL?ELelYfiX;wgsr#3Cq(3@?aK~Aj@OwTJJSSaM77yvvG9|7>FOLzp1p;*PoVGBdS@eqS-5^-&hK%7F^qb$n-PcY=e71u%t z4@FUgun7+XZ#L{$FkG=b$RjF*OZrY3f?e1nNDT38$Oa1ufzG&HpsO1NVCT;l5+zXf zZo^a?a@$h7K9UY~P0BOg%O!u==x0m|QFK0wlxB)nmeX*@S}=sESsBN;(=OG4)oNp= zOz<6PG2x@U!we=c=@V2Vh-^4e+bjf8opFLL7Mxu%Ss0JT8aA4x)J|_Q;uU09Y<4-o z+j=R^8PX+y+jF*%kGuzsQh#`34oA<8>D~BWUs3+4{^(Eq+dG390c=`Lj@6P9b{86m=>6b6B`LlPv?Q{H_`JcT073KH7 z_uk)o+TVTW^4VW~8u};q{q3*s``7CEZ7+S(54<(~oc**n-2Ap1Z+g*nKmE*S-Tv;y zN8SaWyp#R>b+35w@56t3Tl{P9c;89v9bfq3b8s%c{SUw6tMPCA=g+;}c=R8&-hcOt zU$qhInyuekxzTGsdtLwR&37H>m8ZV#{V%`kKR5VR;cze6INGLh<8YTb$vsFg<4WWI zyVu9RV_?RWk_3iy6~K(yAhHa%06)qNR%}QLv=G%U*;=}>8Z!Vq5Lqu}C#L4&T1Usa zhS`!-QB)uTG4<|JLF33uct}Emt^|D8SwhV&LQY%&GPo2xHVr&=xn8+JGjnk%Tpnx) zx~M#`5xnrjp0NTTs=Xs1q6)x>YG@~HZYqs304A1ZNA|l23x4&5%aWmM;E~N~VXImI z7f1cO9X@pypnrYbT(@0J2Pz5nxN~f z)NxTI;p=TpQ14d`!vZ~3Z$G}6-Pm~!JRg?MDU}R0{3`CiY@q%9~ z0_~a%_{0JD<8m^itE$G=WQdvTnqC8$Q?na6ILM7U9!G`(gw0ef=A|eeN*Yd?NIti? zd>}7lRABgmUk=c%Ng7E@I)h0X%qtpCn@|q^W?5iOV>#7Or!LGCRn9#+P~^G^d$DNR zw7DlkIP5T@Xlwbft44E28ia!yUfx|o`w&1!Rxk=rKSLj*OT0Ib&% z5`3zu>_FGehN`);(idAB^n;ZfZ~uHu?cArZLi7HW1m6U{e)f04MhKls5&&HFTO! z1lAntDnFAo(=xRn6ScT+#nocnIu@+M-9#2GmF?=B7}%PLSTTbe9dl3Yo2g;U=W-I` zEdU#K(^}c1BgkIXCMLv&!jkW{4ZJBSIU_=50_=0-skH>q?dwZ-1m+s(NrpC{k>(`KF~T3US&G|>hw0J)qr;AIX&O@x_u z;aZ?DaRZFQgxFj-=pb*?awyj2mdy;Nwi>hmUDXDTCNe?aO1qeivZ98tHEYR=UDbgk zV;;(}E*O#A6d*BG`ZBG@ZsHjduCsiaf&9^bOsnp5hL~aq|w{N>>5dhMeII$cBBDWss zD;P;sP^97$7277svSlN`{B1{+4*{YAPN+o%9vsR_x2({YqOthdPOSvw`6ZAM=Le3p zs;M8?051il{3D#E7(D6~BR6A?jRNPq)v%)x2PSfUtN zZLnbHRhd=mekwE?R8CV=)m-$dvqlZ9wPw@9mY>iuRB6TYmhIR13a+(s9BO4VTI%x6 z%xnZxN32RrYd@_?)fr1OicqU^Raj%Hvl(x5WFHhNGEi)Qgc{bN)y7hwyK|vmoe6Yx zCdS#OZ2@>V-}<#FH5#=>zbe*x(J+WYj+{X?zyrIVR;z8QDiKv6O^xa`!8=n$V|G#* zjG+dW%n6yEPL@hHE`x&4WIzc?vu}~`kY6@IU8Aq>gKTPt6~|n8YR4Kpr$v`2!$fTe zB0-u-aH#2Zfs=TY91R1Uo?Mq~L z+Af!S-FV1`i`@ll50-Nh%b>JZwNa!)rhIjWuCRW!7cC%vKXr^s6A(g-JxXfORjFDNst`vhLr!VfC@nFy{gj$%)hY%oNungB(yR?b zWPuJ9LrRq{whIkrb<@+6i0+Duojn9=v}+E$V@Gwm%W`dM%I~LKyVl2Q2;b{A1~bfq z7L>WSw01@$ID(Gm4LYs#EZEtz>WzB4Hb?Z)c$pBGjpfal&k~}!pDKxfV|AxLC9%ac6KX3F+4@d(%i+eI@|>m(j9w&8}hOWaz8xO zX7k-9KMXMoR%T;P6XYdjLQzv%43@g!7>wAX=X|Z|*;Heu&VyZPKXr#oa_VPpPgX6r z6>99B*Tn_bp20M);VooRTjI1hR+1(>3(L`jNr2iU3rfWi(QcXZ{S<|Vc73c5hJDhX zP2td2@?D~w)^$+791hjae!6mEMx-y{*xTz+Qz;oRZ&s$hCo8Y$Ct2tM3C zkLf)CEV>E@vJF_&2Dg)r14VCNMe2J$d(GK*eg5^g|Jz^R_&x1=)YosW2o&em+t;yt zYt#6xac;dt5&kNu=_;IC7i0tq?lS?=P*XJn)6f%B)o_q8!6Mz8i8Ls*_Zi&e%)Ypc zP;sfp!m^5+h8UVg7ZvFU1Lx6&h;%U#;ls8XAzh=fc%%pqawKZcq02_xZ+7&NfRoiu zTT8R>Y4yC7F%Xv+gTZyi2>BK$ z3w$%9*ppZH8qT6Gh;W{H@CEkDE8$W)>_&$UY#g*rIn9MM6oL+ zd3DgYN21s_<|$S~E;iXk=B8U|vz7r>&AL3d$$14_Oc0!eWW5Rr;U;fB zUu3=7R=>S=r(^Zj0;|d6@#zaek;~RxH~%Ja2}I=I*QK^c@Q+9Ek4Ny2|9|+$`K|gF zR*_(l-}wRmXa4(F|NQaxnS)z?;*FbsP%2X7qdOzuWz)Dxm9OrNkUzNGjSz4lVtq&C z|I&>R2VAJ~V6TD@BtXQ=1PU%+mBHnaic>}bG+r)ymWBH+1c5tQF7Be03i$5=a2X5y zkDg@_Zl$Ostgpln7Znvo)I4YHj-7K>4aWupfu2O*+Py#uf;(0mCW!lDICt@#a(B3J zqE=89<_Op5v0=xlj)`hfRHz9Emkh{$!0j#NT}h+eku;hHO91sXgpi!r6ic>Qkq_GO zh^}^U!I&>3p)#9Q$}N+_dPQCQ8+9w>3mfY^dTwy0o1Xj^AN-xeCw!DUMen?cIdjk8 z#M7;KZJ+^>HRee%vX{OFyxH_wpId-mDa zJpZj9_&4OA{*-vZ-M2)q@2l_m@CS@1d=mZHpC3H+EjRtc;o(d1cYpNh_ulrhKfdFM z$*XTPe{9)&#*h8$-`xM1yPx^I>tAotW3PSeZ@t2M&#OQ2GvD=TnX-ZLW~$ilt7|@36}#gKw?BW5u!xMfyAh25TY>#6I9-!kO)@K&g?D6osZmV?hiZJ zt(vdC`ueNt`hMT9zT{am<7eB<3*)vv`>8JBoSePyCa4s zJ0^l%leXK_EfXvNVC*rS0C|pP?eQJ0>$lxt3H42g9Dt@qrtP*#=cK7%(l#dTTBNP{ z?Z`qz<2e#(3=1@lM;eJ(!yp+^CmD*+c}TOINFs`&X@O-0kt28m@D$`Zh7mMg%#Cp#cv>)?hirG9*BN zrVN&42}Xo0tLr2Q1lAB~_?OT({H#SfXt-oTyh+Du5^s_n+L4QL68Al{f`8;PSqn52 z3(%zirc*)E$A&r`>jNs~gF&~W)nTa5Nhw(>RfwTnZHmYy z7}d_U%S37!EVoBmrsm05R;og)G=jZqJy=#!GF?b#AIsm8a^et3k1Cy4? zbJ6hT{VK@1X%|sm%#XzAHFIB3sz$^F4o<8cRgG5VOjV^X_HXR#uU;|dvl|>yy z3u+;|8e7qb<*3ypm29cmrzi&rGTo90lrzeWmSxuyshmBkX}hD?sg3wOdp!nb4hzAq z=XPCRv;4{MMss)&LK5(Vz|suQ=tx8&Y0wO35IFionh^+yF$6GxLNrA|*5DwmvAQ7A ztcH-lL6-XK#PZrHv$GKm1Q=iZ$inPL@Xy)VP#8WDN3#)@Mtl@3Ef0bv)AxJGi_A^O zHLn%Un(E^j6Ae749Ryu}VSe6nfo8Q`KUiRBf+aU5W92aZ4b#Gy8@@!dtj&W!TLQ?# zvC(s&g_fqS1Yx}3vCZPCKjBh(fxF}(!$TqZ&IVo&O`P33Zo~gqoU-v5&ixmkVaaq1 zcjF^WI$QlMoawiS=y(s@0B`Bq-nq#7SwP;FL8KpO0sfqkVi!4wPU5zid*2#5aLw!N zzJrh6_TGJ0IJZ2ttn5Ey;#G*&*N-)DXDZy9Orik|ClC%%guo%ffFjT}5(*jvXak8f zB{C#FuN(ml9_L1b&?rJ=coK>@+lZn;^xm9Fj@9JWWRjuSDbx#eA<-fWqY^ErvsSTQ zV+-}>NKUABA)(lEwS)Vufttv6WU4?kM_H>dl7?zHW|v0Is$v!sSf`${)M|@T6SZNU z%3^F@?ZZqVr=~4OZOBB?j8!YiWKdYHQHvGPtfsrY5|!;CrJYk~D~++bUAA3DQL@Q; zZ8hFhta81k_7uCDMHy+RB&0!&=!kJ!?(0t3(w$O(vACzwWf$iLrVzr`IHevTQ<=^B zs;|?fF`}!D_CTkzgp^iE-IfWhF7~91+K1^{3+LiCBoF1yrx9nUT0R`lJ;8X+?SjO23KQaYo2C1Z8T5 zm^FINI7Q6%-2lyg(9;HslSDN)JB29;UO2lp>!G0MIb+8jB2lX+{v*DC2WFEti>#q1(e8*wGK+F#^`O=WjI76o*lbf_G+9CHBXw5w!7R;p>5&Il}=XKH=x+N!{EWs zVSE&mzCG)d?x>7(auuIB3f~*7+r_uNDK$JgGkgOfT<3$4BhtXg5n142)kNP{9pi0|$q&$EaoLU0Ok4q43Nu|u#J6By!h^UKYa1Wv|KhzM3IOT` z2uAj9^{L-B^ZB=g5%v0WR}OaGci^|upKpKgH(vCCDRzYC(Hv{w&Qv(f(&vRU@v{k< z4%O+*mZM?DfsP7KhZpGXnyhUt0$kfRgC!rWIBr;5o104dCK^N*sJU?o55XnJ#RWi2 zhKOT)a8`C)7+Qjw&O(A@5sz;_PzMADG@3v(LqU=%GB}8bjlP8$;+K z1$87)fCve>>jv18ArPEKu^d73oJKPm0<4HHOZc|WXoP^P01Tv9foFID7aBQ{rwy8= z7=d9JouNdL3CZ>P22k`Np52V&8pt3z%=9$|*R%3*Q*TytW>u)>i@PPF94qp1CYjPY zz^RzgR58f1jbXK%NKsZ@^=kHLn29miMxw<9@mj*!-E3M$Cfyvh>ba_{idL>N%(sPA z)M6H0uTXPGxp@wxsfJX|>&~K;Us0{1)XgMA@;~_;{|w4E&O_j#f4fxs-1@Pa{(sw~ zbJ~;TGhh6gckYv{lJ}pzf3WKhH$V3C3;uN3Gyd7<-6Q_s;FZ@_?%o?ak@(^Mm#(@~ zzxvzPKJ&>Z?zpl-ZhLk6fo~*F-T%n1xwr2#AK!J__kQ=w9S?8M#y*z0@yA!)@W|6k zr+o1fcfHKreZx#?ZYKAh?dPBV*je{gFaOTT=TJA@wm;F|+5Bkl)8C(?uIt_E+*1Dg z8Qbf>c=0p$-2ck6*Ie+v{TJSO+fRG1U3}5DmoLjc)H?84;zLh-)xG3Fv-f-XrkmgS zx65C+_~Qp&IscEpx|O*7N4e);xc-EFuYODZdUxm5FMZ&f58d-Hb?WJ*GZs!&?)~x? ZnCXJU$T$H z@8#b6)%Sh(yI=18?(e&|4m^Bj`7=B7>~eY9gCDtn|Fgc^+K{b^TTc0B?(bF}&#Z#w zwn=?vN_^jh@Zm|>?3%u7t+%Yoac&UW?ZEJl>jXx#<21r0yXOKsu^UV& z%F?9VH18N4XMX)WW>7JMv`7rW%!rBN5y~*4I6!PGW>Yc5ikfI8Y9f|NnFfvnA`-DJ z15_duAfq@DCFj@8_w07ZZ3Oe{G5q9dce67O?LV%cvDN!vc*>J!oOSIVk6xX;jyz<- zFdq30%N(#WB~C{^Eq57twb47yaXVq7J)CIRQA~&W@d?GG&=^voFasDB!T=y@S)hVY zF#@-u7DZObh)oixWm`B_AxRP;;7~}@L{Te7Rwx{_CO+FV@7cp87eFVyRh7_5^|2ly zFEFs)wp*}~O=Wzmr1};vhJa&-I;J97xly$^Y6KGTo{mvi4yT7EPWLQ~?O6S$XJ}x| zA4#)9S}Y<%K`v8LO+@mnBgjey7ZDOK%R*Mi)j(kea-J&>3=th1qBOh+K^NH0khBLGMVr_pkK&@7|W2#%s_z}`w&=-VaU z&Be!hS7ghjOkfS6Y^sx;(3W)?C1C_KbvW)bU3UXYw*#mP$GT>O3yW7B1J>Lb2xSng-yC zmn2PT=1jDiF;aR>_EO_89`})0Gd}-RxANr@|$TjkX(%m@gnCyezYZ* zK;H0FQdT6$i!!2|DH)uQm$jw<5VBP)`GUl-`1m>RgO;eWf+9;er0Ej;Qxu62RlTjV z)&R~*MMMNzL3~_dS-t3mJ}P#iQbgH%50U@+<^rl!N7R7G@@9s#dBGy|RCSyi+&H7nqmnvepjtcoMa*Bt{9 z5@snQ14UJ{h}GkL)lrm8Ij?{~u1UP$(~>DWks^#LlAQ4j25?%5=8LjWR$!_58NZCP zeF3NYw#=psrhzDm*i&ml#0u0}F%f#EtQ&mY=QT#{NQwaGkxGlIM9W?#FmSeEV16oJ zvic&NpBAD+Cs1&{!}*|)eWOSzSxjn!Ku<{=87X5S=@XeA=>M9@vjAbmu3G^DB;YWH-R;9YF093}LoWW6~;j@J@;^!2VMHGe# zZLLY?6`wU31P6idL)$}(^GC&`kTix=4tjEw*L*UIl?N);Od3h2WOE8(F|4GZEGh>c zY9#eu5zTbVT6L&kVNJj~Jp)lwa>}pEB_GCaY9M%im&Yw!Lg|!>f|{y=hJfm-g#jtg zG!dN@a24s3Qmsk=RdR{}3+KEy(imM;F>eHYpSBbN${ z7Sn71430wqj5<=Vvdq?0oGqIQF;G~iY}C?2I|Ve^bJ9Y|!QdJvEmiOGs;}j_Dpu5_ zYz{#UeFTlD?${5e@voX@kzHf@GQA?cny<8LJGGO4%C}rXJr(hU@$S}ft8i1}IWfT} zvkn*}diffi%_b^{$c4SAUWu2jg))#Mf1S0f!g zU2S^vJFwFf2)+jXG~114qht7;9mr==c`X2np*C*g7>bgVMMQBEwJN3&jS!ekkTx2H zAP2E+GKLdm1&LJ8Jb3pN#hdn;o0~VR5MsN|{Q9{KusJt3CXPSz zfl;?1b-rVVOSOKuv|{ZNbZX!R)g_3XI`iwM%tpI9RxuTZ;%=wYwcBF_P8c>j>lYXO zhGqCpqZ6(taD+s62t(#t-+Q$|aP#&&ZM z+(l4;5!A7fC`lYkSW)U&GYX7jV}=#MZ~$SjVg9A?fV80SichA!jmvpHoldlqi9}p4 zR^$CN4Yy~dASz-ZO(7+w7xt`ZJQ@{@e1@2k#adbB+9u)mX4>jj@KLUqX9C?f{`Yf)fQb&w!$DuYiO~ms(PuAa)BadDJ)sC zYgHeV;w{;wGH8+np_@o`dsSZ>nU-EvdW|8H9!i--j*%8ZUulF@wZkt|XmVu6wNaGu zqh7p#Cbc%-LJCr|60awugp|u9*)ChG*gT=Rd>J7_8pu5~nr|s>D%MSPsVEnV^+H3g zR|FzOR12LRiv_(Tq7+(oCfc@$zQT`pOT$ah^>(q#C=1*orWhh!qPVI+{~N#l_$FIR?KHxIH-V^|aORyr$+VIf~1*5Xu}@%bo}s7LBex0AvX z-eMLcaLOGnR8Xy!aAMlPjy2a zSKu_wQZy&3DxInr%Y>lV*K!mai*;oq)5^46yu>P+i;K-vOMoe+rMlf#awNxXDJkJ< zEuPB;3AUI|)qNSq0w0tkVI$B-F+JTfMo!GI{gL7$L6_(mifa?`Lb{o8;ND|!NsWvc zsHL?^bpb7uN;M1{^i?61A_qO7Q7tt<%WBMtq(Nqp*Q+q#Ph!z@v7Y#<_Q|`nc0QR> zSIY7CiF@q`(;t|G-!}@=OKwnU{HF5!g;#6}$0!0dY&(L0N;F1ADu{t06cI%M1y2Gf zxZ8Edw!zOXS^is9`SxV_!x+J9%1D>_Auls%UgHsAB)GgL)XQ}S7i!{Amth_+!@5=` zc~12OxjGV(YMmcJd{)mmyv#$na-WwigikVLx#qYzEaR9QTpKAqt8;#!mtel^(sGeu zLcXS8g(Bt3T(c`e3}>rOR>ge|=EjaxZxE^qa~hZnmQ^3-h>XDJ}14o#OvU?$=EI>@2j4g76Yk+Ee>jY!%w-&s9|%U*IP*1 zX0or1q?x?oWFHC^! zy18kXN`s^4HqEu|u-gvi!gkl5J9+;9GKzg>udxrr>_vO5ayDLnX8DU7Pu%k2+@TZ4 zj3b4$%mFJ?;xu|q%l};zd#eYFuM@@Is>=V5V(;)MCMI244FMyUbGRXMaZ(T@*$caN zLS8hwl)@^`qLcSC<(!Ws`-)N7NFb-WkC;^6lzDHMT>mwOT{XcL$mUg-Bb#T> z{mSZ9v#VAebi(cj9Dh#uiR|ybzwOqizoW-4ESdWt`|k1XV@^w*_QJlivwQD8gB*bF zhwQV*YGt?8`|J+7_4*K5d+@#(jzmxl8Gj=BN)wwlUq@o|?EJ23XLp}jy?J*38EDB~ zt2WQh&Rl|A^v6HsnLDqz?4?8A|EIzw3%9-aX>9M0eEr(9K7aLl)?HVBWb@-k9k8c< z=i*&=J@X6fg1f~PIQ*AOf4`;uODFxv%ztYqJ$3zUuk?a{+CKlQ1I1OVXJ=-2^$>JC^w8`~ zxQ-4_q*-kPtwuH zzxLiw9(E;l?^nNe)X%9O>=oZTzuy1a>U(!PciYo$R*U?L3XgAJ0-v-z?Mt^U|*v5MPmJl*-S#~qD!TiZ@_pG0txqJ0^ z0aZ2}eEQLA&wJmE#$kWF8h^C?{>iEu&&NxbIbdZ;Gq%)&uC*L0mDp%L=6ImxBmnJGcsV> zG0Z}srgozdjx!Q~qXvS+V4R5|Q42Exj-aLqkHTXZ9wRL)TA|DeJOf9`7(gSyu2?8; z#*7#qi6A&`KuuEz#CBdEz{FBaNCAsx=|q{RH+ffBjFyG4oy%(;C)sLN>l6yZeosn; zVbxk(C=>|Z)od#wEuxV;Qtb5feu7Vhpw!Nkdo7j>Tc!rD`CAOgFPe+0ml;x}l9JES z%@Bt-6|P&C;FToT3tCxE9JN%JO?1*Lx%TAe*n!PnCzS7)4`JJW)9;NS(^X zGY4BA{M;+zkW&2!(~cKn@7 zH~g}{bP{s=zdje;ctX*{)M0X?uVemA9>7=kNDZDXRg`yJ1o-M zpUPfoq*;V;**tzVqI{0rG1o#?!>>7yHG&)@LK z_isJt9xVH%tq+_ci7zgU&OQArx7_T!d}2O({d?{`zk6Bm4e8}I@6zsj?$bBk_TtlD zIPyc69)0zth%Eq5Yup&{Fe1I{qhSzp>%DyZ@8kcl9BcJa*Gno6gSM zP=?b&CZ-LmSEKV2zY z*gW*?=a0Dj&KvK-_CH8mvwnZ|=5x;>Cd+YVys}n7A3lD-%*>T*W@b*9o0<6^C~3@I literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_29-33.committed b/tests/testdata/eol_service/ledger/ledger_29-33.committed new file mode 100644 index 0000000000000000000000000000000000000000..8c88eacae9a2eefb4ca87b65581c9017bf86063d GIT binary patch literal 10146 zcmeHNd(7-;S^svyK+bAdaM>U%c>Gp&+01dL_i0x*Y}@H=X4+1#)0R(;z0alXT-uq= z%*lz!Me(*sBx)cAREUa*upr7JtP%cjWrGMBg2Eq0O(crO6-)#ZeCIne-!8{pkbFnw z5BtZo^S)1?zJ2>X&+~hJ&->urN2kY4M~|PL|LP}R^6cL4Jh-{-br0Ni{4)Q$UVN?l z=w@*K#Qyr|(DDs{ z{^Hq(hW**upRXJb{c?DG-!bgBNXSQ!kRXGIfD$}HViF+;5{)qgYqfk7CLtJulU6GV zQIeow1cPzhr-C@f!#Fv}JiPhsd!Z-2c{*L)I~E2QuQCt@6f!#G5d1jlg*qaqv+0vHZSJfYC&m%rnhqASOfcyoCa zfk`^AN-#afVnbbXnuY?kjaYi+Ar#U>nN5H) zs}SMKu%1o)L9~-6Bwmp;1KKF2ONF*wgb?g{ZE54e%rq2m!74l>5Ioi#N3d;0kdpI; z()RQKQEVlLO&`%${ub)G2(^p^dl|8C=?(BA5>i5Hhpd8sO$2pQ$nup&Al-hE=TwlZ^1k3=i&CB6z*jEuA+Zn2Df>k!^d| zj_SGYn)!O{Pk7W2hGTOA&rK&%{fUj6mJVcLPB$6H7ZlYT%=8Gt=ejdgbdEF+v#ED3CwC%#KsOUS`Y-O7GLP#zlGW?csItk zxQ$`)K*n2IgDj7LUN6fCbF%KaunPxS1R0#X@L(5-McuU!3X``H92w+7<~+FvWgW%o zv?Bpm24jUcZQe7uH6#~P+}$#gJcz6S=8bSSUYOpP@Jx!fp5Tcz9&x(mIYZtMD%3A# zc_1qNP|&tPPS#!)^|frKM4ZA}h&{C1JYpfrG8Cta*s;CP!Y$MC_}%;(Jn!k?u`)-o zS_URzf^F3{uMOJaDi{nQZOmsT>g962?wS+Z(}f7p!Mv#4>GzlzPNWYkm~|~)Fvl_` z+l;FKy*&h;Mx3{2AbxT@BFu2#^6)>l{Dn@M& z&N>D{`Gzx_%bb1sJ%N5^H{L(d`JAMMm^ zww#2By@~Q{?Aa6)z|<-P9ziPLDA7*6Ksic*@ymkC${Nt!vU;`|%6!vqWqK>n`qFya zU4>~@arL~dmAz?Pz^k#OIdMJFNH&~f&?@A*CPEC%Qyp_!RsGH&*9<&eD^5ri_Jr*r zxFSKgY4Tx@^7Vq{l{l<<^_AFe_g1UEvB<+>1XA> z`|Namu<%1DLIaXQk;q3PJU}A~56PIqKx`mL%EtoG6ZkEhz*-TBMFa`qXp|s95MV7b zLBG5UWZ<}R?1vQyg5~jj7hiGZxbX8BTpgEjb$MCFi{ncdkNriu>%DRu#*1p#`2)Ly z;RH#!$O8W&@e)g$+-d7cAxjcdj#HCOCa$d(Er5l+pi zrq93_`iSa@HNi(Ss%o3@niD7Wc$;LUI*ZU^3MB)T@#KnSjMz8fl@S)RZDX@^=ZY!w z9GC}D1Ebd=m0?_tS5Ok}JFR9utQa^}R|x`KCg--(6)!X7ZmuG!)?AtK>4WS^@s!f^ zHEFKqm}af)lnyKeVnma5U1@SEgtgBwl~Q$E~V zU=cUiKr>++1>|xwY2&?h+ZK$)Nih{!h#qq^#!d=593z`yR~DKy0~MoTxank+UXMee z0dP#mYcS4o-rx2JJDH$;(kPoO1J*;fK}D>TPm0qe#wqQmSa!!(uU-O6^ss)bgMR`F^S#+R$<<>@Rgj<5dknGqh4>(d$G?jiPyd9_unb~>+in* zd-no6H}P|)65!&{cw_LcANm015-|diL<}Vn6$U;Wkvr1xFOX#1xf4`I!ql!@K8Id1LVIh8hCYGLaDpQ%!l}3EakO+anp#0KAbg&zwMV z?!wAA1yNZa$=lAv%{dfcVn{(WOc69hZPT!f=m2eWM($WcRl(i%kj^j=)~iUAyB>Ee@o(>)ij zTwKJ}a#38Y7R&hJ)#Lw7-*xAA?`D8~)qmV^3;eR*T7BgW(tH2nAHH~TKP@}|@U`dd z{~Q``jPE*cV*M1p>%1!e-*G_|GGISWmvnq~`uioete+WL;B^>({1<>SApKe@F zi2vT%y1kZ`fAg%o+U~M);*gc=wCtK&k6|dxo1o%k)NhJRs9M)&f;B(pERfeT(jg=7 z0d7-}I6RBQXcgw=J~QWPd3(sh>H1n`ewwYXLDmkR6sl?AATuAxx>j@PAv3Sq8sw&C zZbK15ZT#)iXXq!zX^t*5IFzya3uPH~h$hOIt#+p3n})NI!>})Cqj5$wwp0}4Y2{7( zWoe}hs&PJI=qJV6uBAQ^1azYH$pt>^a#=7->2hI{STUS?+&J4o?O~5pTZ7c&jGPUd zQ#x&q1&hTw{-nSHV%s1TP3HFef)1GsZPoNHpWf0DlPb2TVH}@o*1~F8xVVy<6E-nQ ztDog)eo`37Eau3t7VFdee#|gw%}?p4ls?ElnI!@90JskiJkg`2ns>D{4oN3ahjh_^dgpi~nN~>)fr^htC?6(9lS{s0# zv@K>kcTBIfZI8v_qPFK&6Pj(Rac)RuXmJLai^@sSA<7`e8U`JknaWD{f8*>NfO(L9 zE`cMtF8_StKDbDqgE@xgOEf} zPzZ{V7DW0eg%cYBeU3gk5>rh)fBL1sooRv|pRTM`dh|i#iy4dWZ)#WUX zQot9LaR?H!17%<(Hp1E~mJ+Z&zXoLnPyrdXAcm`#r7!CXiQ100e!hlF(Vf!c z(Kw-V)Eu;st=htJDrTF5YTgXQp6Z#??QE)rSpy+9epg(W>uR-R1GzhQ#-UiMIo3@U zxkk?yt*N3-*PJV{Vi(jr2i;9R=uqo8^Bo80NiVG(dOMPbMkj5OjjmWw@n)kmS;^t+ z&9;rqT9ZwP;_GA#t7bBjxw(n5si2C)Ce^Uq*HXoVAPSmS7RQP6f*qkEy^-h+R;JdR zaGZhN3CLk*YJozhxCwQ5`7*d@&~{pi(yIlvHPC*+z$L$;)e0(7Dsp`;HeJy)=x0=-B{y|lI(qiSK<<2qc0uoucppdG- z$xhc#ohk3mnkJGshU(~`bP6j8FNJznQTW1)C1$x$JDI&=afeYRgx2V$TD`?awKrQg zkTe@AO9Sd7y1#6!0vm>i4qJ>--p!c~@2yo55%IMoPc0#Ll5St&J)~-BT+nY;8=RU~ zX4Z`XrgGdlKI#w#JR=y(pDhz}Y660+KZ%8~1?9$iWc7L>H6GHd&~KvYDNz^0XaXe( z*e?KfM1W!s>Z38(?0}J=7DNR}5aUEh0$vr=3_%a%hcLX0kVFFVwpQGNKzv4KwKoV% z$ZU2556S0Q)L`NjI~sy&BE8&}1FfiX-jfa5OttAkC%SaVt!XV@X)*y1DLJ=5H7YjZ zEw2XSJ~rFd2DzRZ;kemT8QJl9T<5BW5;x{+N%ISpAP~3ClPq$3<*?f?SAdW-7=@zA zXqdIe%eX1``YVx%q2>a1L}tPZO-@}++Afc8g;A^9QbLN!_?+rt8NkZ7MwWUp{!$BvTFKDvrxJlm`G(D*y_@i2-2skSpw03%n9BOsb z0RjUX?XUX-kt>1;kx@ZPN!xWmY}S2pGzU+Dk}2Pf9h7oSiKT)_WoKH7gWP@C;l~v3 zu%Lvv`#KhzRqKVsb8|qltZ{xOObK5FFM*59o6)82j5}DT8F@F zP9SGnYyvEjI|dOo>txRM=h~n$Y@JyuB{6=2uDt2!x1X`|)sE4`1|(GeTRX5@>Bv`92zg~I}h7jo`liPA`#$d3jplC1?T_JJ`{m^;xx}k6$n`K zV!GRebLr5s8pSnexOBSL2TWHn1v`bn{D7?o=Q;pIR!pOvrpOO)usZ^eTc7|D1tCO) zJ_^MshD0!uAW%p}WK7~1inS6P_AwOf%)y|50yR~z%P53^rp0s$s-ed|1>hw~wsu>L zT5%9jEj&V#RzPA%2#gLi1NFeP0v{n-6cmLaN#PivOF*S8@*#Z3b6=YP6#Y8NTVoZj zZf_9W-j>L`m5?Ay=v54kO>~3lBx{l!mx7X}jb@|42~#!FI7YF9evYb@VCfjEzJm{<&`4Qty;2OQ=C0Mz z_n+eo5j+KdbhaL_)+w2zjmAUW6JD(x0={3 z)yF%J{K`EaW&c?J(KoEWaPcE2m0x`3d9VHPN4S4`?1!T81G85@?^hpqt@^B&zvC5m z|Jmo+XTSYtuWB!T?1Q)e#-F_N`=6WK^!UFE&ws^>>glKRyZYNNeysa1kH7imU;oOx zmAB4b^x&87de@^LdJK8aZQAqidyf5)H@yMdo88e4Hr)*7=Qj^2Ii(a~oC6#GvJ C{DQOq literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_3-11.committed b/tests/testdata/eol_service/ledger/ledger_3-11.committed new file mode 100644 index 0000000000000000000000000000000000000000..4c54bc29aa9cc9b4189e72466ac7e7b66cd4a808 GIT binary patch literal 20079 zcmeHP3y>pMbsgJa$2Jrdj0sgXjt~LHnayl})RKB(gGZW?)UAQVdjkByu zl>B~9hiyl6lr5w+6x#_O3$qLvk3?kxNh+cwkyg|QVknX#8I_7sltPeXoK-|siHb^u zh#^vnQ;I|~F+@m_2!q0Bc1V7}WBiH_-gECa-u{tS{kHj=pSknxA6K8^SHL3r%Rn?i zVZi4_Afgr~5Pm>JTEkS(R#({$A3GgxEAEztKvahvgYc2Cf&0mCI}n9iik(U!YiOol3RDL8mQN5v~{IQhBaWBHa9Doh=zoL2eIE zh$$or)m#J-{GgbO4`eb`EwwGWG!l&ZCf`&HhS(hOM71nr*2!W4jfiZWVtXiE>7gWP z3suHM=|+@WZ4hJPdY~X%l7(C#S#kLhP)7(AHlRY=&{njK&GK+#wLsNJiFVx(+Bt)7 z=BSF1PZ!8sYt7AVR*YP$?&gg0Ah)>|*{D{Uf>EaG)iRM=MeSUbi{y-omfI}T!fHax zuNFul$+JKK3f!NZ38>p0%qSkdiP3x&tQL74@Ee%ylj$17hn|+E%X$%OZ z*HDU^YA)@rXW}-Qm3t<~iR(uTW1Zf^054y&8j`TdOX zp=X2WS7b-)E7SHphQLqVapT?f>)vp;9>RialqgeXFhiSV^=*KeP{bUFMrGP?mvFjl#Ne3!JzO*XyXb=tAwfo$;cAb0P)_MI4 zUhDk#9sl;!pP3TT#1jk(FCr10(-S=G7+iwECjx`u2_`67mRHLKaUC)FV*5n28!hFQ-w#iErI z+Sw|zzAdMPtX;5^C5PI!%QY@lYh+4tbBRc7D@)cI)$c47`S_^BTi#-rBTA3>{I z{F1|Rcjl&XM^j5vOxB8kNet zAX5S_riXT=Y7Ip)H!M+nE=#ahsanzG1H-#AnH$N+foO<7cZGNZtHCAtW}Pl}B#Iev zEvr%W%2pbR|H&m+G|2$>qQpg^?ntN=Rng3=^;%-UX@e|0hd$uffIybcL5P@$B{jdy zE9XhcDB528jJ;FeYsWj#UsHfEvyS`U_Z;_I=?R%cU#v2O>>f}DXk4d;eZgf1vY}J; zT2aTmOz|+LaY8H7QS!jPe11Bk8b$VnOl3@alD8!anLvlRZgfVwcn|swKH+(d5`gbK zAy(NQriC-$ovA7Mreznj3qB2@FPo?-8!#>gQe~quL{;4tss%3uuUCh3y{4aoh7^g? z&0Y<(O-k!6v0B+RzLRka-syev+lVe`>nm6ni9HzN>uKgdebanBSTdd|g0@MdgmJYT>jIkXW@Nmu;KpnFJ z*w3qVuzjdIdknzdY6sdMeh=U&$N;Ai0n?g727z1v2j*hodD{IfJUB=vG#0R2?#mgz zgA7Pru55q$ z;+dZozxI;N18(fKc?5psxqSUY%@g~4-M#b441Wdt-Z+Wcac~@T8zMsqCgBOJ1%l!Fn`gNbi5}!5~X0H^5J)xwVQ5 z@^Z7#e}Q62DP4ea7I+aLCsAndl-}muL`TvRkRz(NN@bvV{hH91KI~29BO2H@z;XvZ zT_M?Y>a`UccxX>#Fp(2E60n@pWK&KurT}sktA#+O9YI^Mq*~s@e4i*KaDGo042W5w z4wsqm1o3N;l#NJ3aT6Nkdl;M>1lfE1z;~HzGkq?TC+k$BAyO3vWZO_n3DzGwUnabc zG8Gs2gSI^Aa2w|d8~FXuN2^ZNdv&v7RMPAw%5WZCzz^sM0prR*AB-#+(|TCMx`em2 zVBZ{kx`HKDQ-k$nq9g3+oyQ|r_4scKkP8BP_GB=T75EjCu?2eS2HRF(6A$nTx(u-f zpnEzV8=$M9`s1!vJRP4UCqgzW`feU75t!#8_D&e6%Ct~g?}JQW zZ15Z$#Ia2;XCOcuV43g83&wm%tQ(%b_ONWm@f5Zx z4+nO|@@2Y+b3;4zR>w9pkvqmax?&?-9@urQ3HeO$Q`6JBJ zI0>})_BfVX7Gv`B_sSHw^r>;XCAA@|G7@rv=D^=k-Y+8Gk*% z`%IvuN#K>vsjZ~xR-P>>Vks5nm$WrwZS`nQEDp2WiDo=qf?cW+q(&R*L9M$a4F%d+ zEpLjgq3mY(QK3jKl@hRlAfan3c5i4lhg{Qb*TrNx8_QIS`C_Xt3`o0JZnid4vf;u; zTSjQx@=6U^ZK9aiZdohYwPb2%N9A+;MeRLH;hnD4v20Pd{l}A!y^G;ABf^etL{bqN zws=dhB?vZwi}5&xP+X3~_c#r!lr;PUyC78>cvE1%kjjc;G#*tc4c92&%|y6w+QM&ZK-D>9oOF}gHl6r zVB4l6?kR^SiitXwW;(mO!34m=qq@~N+UaV2*s{K_saK<+z$W%x@A6vW#z3oKp`J%f z7&>ZbW@EW!PlyjHdi_9UwP$)Ffd0bJ?N;#KfZ1L3RjWG?y9m0(+r)cXSAm_~AVhs2 zx(B>M>^V(yt7E|t08QDE^K`dkQU=aU8iA#rt#r3!rR#+K0hn%lY9^r|0&n!Xiij*z zcV~7TwmOO_b~G%?GyA$ev%ljoDCh+dJnr}Zq2vc$4K#Sa9AQi!x9E|gD+idvA+>(o z^Cj_5tZJ_9%4Z4ET1IicgySe~y5dPnPKTGxdfx3T9o+kg%*xl1m> zA>nT9B860YtG`y@C#3dZ+h~A)cx(GesDBFjYxgIe91l7Pg)}v^ z*bW^(9y%G;5EPGz!3=j5r`I(@_=vRabT_R;PhISf-yI5_!6bNpoMzV=giK`+TI+VL zE|8eCdO8Z3mJ?Dn6NO%+p%;ZTGvx6xelavWY3`B`%=T6nEgE9yNJvAwL}J(tpdOwq zgD4l_p~pU^+j|0}L!7I?tCXPKg!a=Rd}tOTsgQzcI|fYVt&rW5n;|$#d@>w56FTGd z=5gXPyweE!Kj!f-_0&zPBs+p79c_I=IF%R-9#E3j3f?9rL+EcL^Uq z4xUa+X54rlv12!7CT%~Wk?>Pk=Q$wLQT`sPAhr#4#I9&eqmd__Io0Lze` zTEzGO5Ht-Q8Bi%Lou~amKOnUyHf*u z*c)s46VRmH05)S(=bCjuljn?FV3PKY$rNR~lX26N{;7#OFg5*$k!FjMu53xJ1NeoM zEm@SCI7z2M&R6l9jJv^qPhB&)V&l2zNEmOO&s713%^I6N5vRkFsOuKa+@6Np5yn|- z$WMBAW&8*ZRe9Gfz&!*Tkel4enP0Z>VejscN8bFmH~cku)koib?9s))JNwc%UgJ}R zPq6<{aJxuQ7@R>n+}X3k-m%b|oUXqJt`wO*d*;SR`D#k5Z|2*%O^(QobR(DM==!KV ztiy#J8>tVJqhectOFZhWys=SDHS?_oCA4^b zoq|=s^stXIxh$;YzzT7mfL$7!rL{FD*D6ydO02e-?(_;2-%~2hY=zd-LQQWL4J%Tq z@N3obFj=qYMYyu0S1~%glHuISDmN$-Z8mA(T|BU|XBHgT!O;`wq6H@jA~}3W{h+gL z8l07~B{^9D3d0OwE$Y!`0?CscNgeXk}e@iEh3Yt1qFLD{gPEY&NuHn`2~V4_fTna!QD4Sa7l{x;DF-ed2IuVqg}BpSRXiP*EyK{9EnC?(t?)7GNKnw% zl!5nuQ?#3KFOq4QGE7%t;*dXJr>3_$6-J`OAcAC+7+i6MB$FEx)A z$ZbxSMK;SUsoP8k^&(O^Nw3$mzOn9-(fUz6lchPCa8KBMF3T4>xrkzox&;s0{Nb2b z^o&*Hi8?r4MF|Ev_zYH&_bS4P;K8_JFlS!A_H)#iZh!MvKmYK*CvH0Wxi>uj=6lzk z_24gUJ^au=ec|5Q2iIQxrl%x6dHk!3w_ScE`Z?AmcF#B7`m$jqN!@YJ7x@qHFT45=qW9nb&G$e5v)7Go`QG#C z4}I&wcb-%)uQ>^X1wi%0+bmNPfJ{6DTgbM5M> mK<-%HOCaNG;cld5xcujtV9yNL_Vv$LSor<5g@r1=u<(CQHvNqN literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_34-35.committed b/tests/testdata/eol_service/ledger/ledger_34-35.committed new file mode 100644 index 0000000000000000000000000000000000000000..34a87d9d82539f6fa0ab3b1455f14e697ae7a33b GIT binary patch literal 1087 zcmbtTOKc5M7`|;KG!oH}5{|&NujS7_tsAVqzU*elL{H6PW$HLq9^lK6$D-!4uu&6YHcxwPmS9Y-u^nIJ9MZeS zb4b?fcKuFKu(@!Uh(sbR3(2gg5n@}2LrORCx|M`NAvOsXmZf8gs3y3CP?!+tlnaYs zQ^%4>L@dd4q(Y_x2!Mu!fJCKOVywiKg2;Kc>nI5b_ea;2#a34o`OM+U+mk0=T$o$_ zW$n6q>d=aCptyNuX(Av|Gjd>QIP}}i?JkG-`JC{=p=tQj%6GzpA%(e_jO}-NqR(-Q zuZ>5Yf-Uwb2`yeSMet6?4J-}J0smQnIgZPVN}N!g163C$Hw32)7>qz5Akw(1NewWI zV}W%9ktubRAc%oPfLa=-0&5HeN;%i{10|-L2oft8)uka2=c++X05M|Jq{PqwBp4%Z zaRdOB)G{PfEe@d0458`V43r`R^nlu3fo5~Ey}r4no~CTL#g@(WwcCJ^@`Hex?a8LL zv_aIgw=vm*dp39F3)>qS+a<$(kI~k*wyC?dw?9Yw1_ri?o#2R|qNIsB)lvr;0(3>Ito$MMlr{`H*6%X`o5Kl-{eePndvK-B{F;B>rU(TD3l zmTaCJ&#s*BogEoY9((n5-inLEdt#ptSH1ZzoyQwHvc|jPBp?oKZG`7r+8O59LC$uk?Ok5-S7k#|jRuPhjudPXc( K{8`I`wEhBtJ`F+u literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_36-37.committed b/tests/testdata/eol_service/ledger/ledger_36-37.committed new file mode 100644 index 0000000000000000000000000000000000000000..4c1dff52ba304f15c3eb3f49e48ec784318831eb GIT binary patch literal 8086 zcmdT}TdeHXSw3ka8Xc5}UKEN#A=)hnb;23n$3AV;`Zi z<1|Go(2GiGp@5(%kQVXMA_zqhFBE}911%s>g`klrJV4bFBm@G=LlngF?3t5&PR_{* zIq-mAmbKQumY3K1|Mh+Uw|?sTt{k5GUf25W`eir#=MR1EL3Z+iKY8*aPY%BRfp=Uq zU%4NYZ$4ODxpM!h^4NCA;xS&xAro0t~-u@El7%MapHQ}H7|CPH1phZ$4m3j4;?G_vQy7fdEfEny~h%= za0n(45^+%&g?!7hDI98HzD*JY4p|W8w@?SeF^Ga-4}u)jh7cI{NXJG=$o283UUxCj z57iIDcSQLSme&liEwZ%1A8M~kqR5mL%hKj5pi7aV15u*~t$mj^35jTR*TZQ`oR86Y zJ1N;K}FeDA-N;+_|g^J!dYddFn44rl1e$^A&Cnr zsWQ>gOLcHvcNJ73qlh*njW4-!!VOiG(LiVI~S+Xjaz6Kc#EtWJ#yG$St1r_7lfodk=FpGN88d30#6nO)Cqr>B9 zv1tiGc~z+YTZ1Bo3Ceatn-kXK8EDrZ=+JyTZko;BI<{M|+U}PZYotLM*s79A9NurZ z)zx7{_n08q%omX|01HT$azb;=l8$&f4S6XO_AP{g@a>wCT4u)xj78eXC8(h-r`@?D zTx|qvFO(CnD5z-L%DB;vUbUO+{=CN= zvicm$@`qix5zpSGJFe%T+aN>^YPc9cZ%W+_Gxj*NJ0 z?atyZi;Dy|QThZkY%VG|&>U7``VoeY*Q2=-t4ep&7N=}EMGIk=&|G zorg($4!r$*;q7mBtAWY%MfQ;%Ke^;hVnoYKumC|GcBm3!Xq9i%w8GLVN!Q-S20if3 z(^cBvr;Py?pg!i7xIXowd=_s)wM0Y_mlq~j)v>|_(k5v5SV_!vIOsB60r4nXD&l~) zWD()y1hvy;E2oRRlJ#zL7UOus^9Bl6AiTv7<&4B>CTJQ}mV2AYJ{RPjIgtBWC3u6a zLt>Jqa3e!NCQWzU8L@E-%t1`)CMxy4MWi|70b#Vc^(G#3>oGF#ZkRzX>mwC{d>nCg zDq$m9U3XH{9WR?IqBDa+p!;zHL8r&h)@iu1(*5~OEqCAA(~K_7 zq3&dD^Ldhnd3^}zURs2}O~7a8dA>e(lGqP}ZEDqrMQyyYcIS5D?g1`Uz?;rs2qW

WwBIt(JK#?~=SwshN*px;;bLLb+g z+h&Cl;LwG!a?H`)10&WX9#BeJVxT&6aJfJoF^sO4yc|w_xVsETXfdb=4EFN{APAX3 z`+yA9Tw0g-4o8myM$e7hSa@XAP`gxbr#42mH|<+X%-B^I>m+DA&vJBUUTRHoEXRP5 z(MgA8FOmqbiX4qNwA4`nl~_aRXec%yRWH1yf=|NcT3_J(aSO>Zh}9uA@1Vmc(5+zG_e*>|TSugKuo&BLv@nI_938n(oV-CB*8%(RjR&@84J z6tOpj>3Hp0Rncj%wlZ*YlbJY8=hC@$YnnETK@k7{A|Yiy-t9fqY%*>b!LrHEt-&mw zmQV|dhrX>qnT_OlOM~YUM=ng5%6E>U7@?Ih$)G>S8Ee>Wh$-yY{fUXDE1|h2b$%8{ z%W(j&yr4~uF`a_q)j(|U8(bMWQde8^>Zm_WeJxi8%2tNfNKoMr$7Z7-%Qhn~L?*LX zYoMazwTtz}rcxRm2VG4zhvZnz5w68n2I?q1XG4{xKey)74j=BP)^OSkxJfDO0w|ot zBG=nYsus5MW3NT%_BeIi$n9o^6UN-!+II_`FbJoygBy!Q>ILMmsWK}ag?m$7L^ste zF4t4VSm1#^Z-lLi^w#sxPx-KnN)ATZ?PjuS$Q#8Dy0W1yO_#OOINauOf1k@^79pZp zoT~r(Bm_kd90Z2Y>llO+w<94hnE;QhnazoW9PDo;AvalH0%i(F{*QpUmR(%H9=6N; zRe|$MGrW!-XLTS;x52fSnZ6v?UNQlza|zduBj3O^&CA2I#HElHZeB^uTG1WBEO=zj(PwVqXo!<+c7}AU7b+rSny&_f>ohPlnl;5lS?*UpvGd z;7lzQPO4Yeg@&@2&-GHT>vBleZj>G)7g4bb7(iRoF-i#3c(C%RG@oPL zh?t0JpsXrJTlN<#ccn_^psSHdlS!LqyG2`S>@2qpvoVoSTgku(N*hjKx-=qyY&BQ` zm}3l(`Kv3y$`0hPvBZA8l_EeP;1x*(XfO zgmdMI9(xE9haEcU_j;wia01$vV=aUEe$dp&G}XE*5AO+nf&s!}dfNg~=3SnV)*w?~ zN>rPr@vgHek;0^xl&~6<9PaYb*6jJDLoLd(r9*vmjOzJ7!NB22=@hEvaNrLCSJ}{a z$gz(MtA@30ngT8k80dlco;#@r#L@G77Xlu7hFwEzYP(LoB22c~4Xe-veE?%VFTAuS zu4mU#1caRDB%oq{BDwIRXP08`&<2#8m%WsS8oZqD|d>;Up^{Scc+;bF-V+Z zP@|=ZQB4_!!lnyUScHS7z&MI|TMZUao4bmzfMQns?t*60voAI$`~yU5UPD+yq6qj&{4x^Ni-_TwO2ttO$J$J z1`Mk57NTo_s18eUK-f0zs{=B{*FAn|uLC+uha?uyH&Z@ZDxFC+?~_Vv6ij)2#X8M| z4w_NNWO2Sdi!qz18icG$JXUh_NsKj1y5yN=b+aIwnk^F-3bDCheLXV=%N4qs#X7R~ zc+x3vRb+2xeX_l*i`OwIaZ~WcDG-6%6E*N0avV%1?WMCw8b&gO$x)Bp;Jr?JH;FT7 zz{=orl>j1tV^tOnC_f}3Bq`?WNUj7KC}u=vdssuKJ!7e`;$33!fl2KDvH!^Z^b{!`_t^PS44-o(D+=b!q^zk18&#m~R++`E3d{j<~?KmOrA zeBWQZ?P2IQe&h{*bn5rL$Cpb8cu2WpYk93!U_nL*wzvL)L+!#z|ajif8vINkd0AUypQ; zw~jUQO_Ewp!%-Y3aMWobFpxA+$U!Is19FuEql86}EyN)Rill6CUum^`6bM)K1tx-^ zxKE%qRpb6?0b1ZS48WL&guN}Qi?Sj0!cYr_L6_p3qPdpWLwZTZ&HcvlOSLDk5P>2H zy>nN6jaja$DeDeRPb^zrpo*m#DHQ2aazjc(G{v*HP-ITr2o{+LNC%cFvAvVii9MP$ z{oRNi8M`^zTnu(jG66RV(j3g~xD_`z{97-MoXgi&H%_oT^}J*Ajp9yc*V8vF4lk!m zuD4rF^4gc)^>_dAp2>fHANb^p!i(Qr|G-#72<7a3a}|qS%O(RgR@thLYHcV!=*;1hItx zu_MHBzW+bVJ?AbYtGnOK$bOA^Q&r@>XZg>+Z|C>@(6@Y!{(b&)lYfeSZvWcf{gKhf z-~LDb^&kKJf9@as<)`2ECHfHmzJUJ<`~-eiAM!U#{_p)C|BgTM`1WT%e{uikfASCg zy`TOw-}P+wl_QDtIc6+*l10r zoAqXYsvcE8KQDjzA^Ef3tZ&tw$E|pD`*ADUc2dnoJ?3oJx0;D|E0u_+wqo^G>v7a+ zKiN(_Zb$3!Xlm<8%t=L4+o@DLdLh8@xlcBxz0RoRyqvUNZ9M&Cqu*M*>^U1xH{N!F zjfWdU8mI8Au!rAy_>IH;gF&-e88j0oGbfi_)Z&%s;pw>3%%yIf%+6IUo{AsEvlsR2 z%YOaLq30gwvxWRgDY__}o~KH~*-5j1^SqvQ>b0}uEhl<#UCeA>H)GkeQh%H%-PVTZ zrw85EFd03)J&2x_Yx_sB;)!z`Jv@)6=1#6McQUbQ?JPNRa-GNd-A+_2Zr+;CUN*+g z0!wZ!4sNl!Xswb`tLQ!J?ew0VmT#Wzp2W{@)BW>dt$#c`=pM%_!$R&Pc6_n7I6kcm zk1x&_$HVgV@#)^yv$IOKHY~@_&&tu`U8jG1mft!aR(i*$Y2f_{Mk-5>ZD57`RjIkmW>|`2D!u_n=SUvYNr=Hr&5|N&Q6Nmt9UfqO9CkK zPOEs@KF%$U_P3|8!{&UH&)1J0os0?_ud$+8eb##E^f~|xmX=H;laG^$=973V9(|IC zHgQs0vFKJamUvu$y!|BJeEb-g)o3;n^(XB_-Dx$Vcr>0!wjU=NsmB|yT}=KyBCgL( ze&tX6?*B1%(*GNO`ra3Q`#*R36F(pypfCS{qCEAX{4Vsz_Rsy`@R40=8t~&zmQZfeiKmM=+`CX6)y(!(^x-oQYqvLdxODw zba30uUB{323c1RllWbKF3ia&ZaXGPbU5+1|*N5GDb9hrcer*^y!P&nF8#i}K2u)cz_rCS{=gS{%>}}*wvs;dyp3x~de)r9!oK{-AZd z{iwc^ZFdsoQsTO~yPfEd^N+6&#yhn}uj7ofvEE|$!NGWzihtTX;`5&q68bmi{=(OO zcj@?VK9BuUDffkM`-S@k^=S#MJ`AHe_4&{J;j`~QegA6r@BPvbMt|nt{*iD0uKzgl zEC0dQe*1Sm`*&UBOB}yGsqdcz3B9wZH1>0&MrXd0o;&H? zXvW!X91nJn=I6Uto$hcpiC^zd3(4o{UU!x(JZd*fdsnrZQ=L9~^!OrIxX$0~JgX(2 z>`rE>?aJ|k!TE75`s(JQ+T2Z5XQ$=e-m}`HX0ad5EXIYKtKxPhdC|{5dX|q~+#c0O z&f#b<=zQ}_=(i<+wIsB#pOgQ&GtT|3|L125iGSO)22VL!`% z|M&mI;Cugz{%?&u_?{pA```1=%zp20`R;Ff@a_Lo`=|cui+}Lvl7C9Q_YQbOE0$9ZxiyyXBMmk&`SG`_CUwwi{c$L~Pc0@?bD@ZqBo}d-=(QbMw5@ zxZ0WKs$2cm z;;*LVYCc=-?3bF|2hsGa)`Rh0{Oa;S@nHM5l>AgRcqN_x&R_rHfA_(ksQ=$P|HfbZ z*$@8wcN{-3C{Igg^`WAiu%AC1|LMQ|wg2k>`al1dCx5>FrT_VV+x<(;$b0EOkUjXp zfB#2P$N$;4ez$t>n}G6GO6PA%%KIllI`8fF^EcbiGsU>`IR4<`d28=Hdz7vnjgM-H z>e>0?U?+7ofBtxOS}oRU^Q}ZLJ9{?1Im~QrjamooMu(*H^Yg9b#d&Tp%iTUMXR5=} zgLbU(yja_Mz9=NJm3j6_r#?QqNwvq9squXNsI)goR}+WjdFM2FnygoT9Z2W5#m}{* z^M}`Dw*Q&Jmw)M(f76Hm==*-=H~+@e=of$6LrKSEKR@9o;N4aqDjE;_`7b{y-TzC| z&0qWKAN!~O`CtBt@B4?vKk$dXmXt8$ti zpPd~fo7KzF*X`fsdZujC?G<@!s zTl;$4$!89_=ZWHFBc8m?UyM6v{Mk;YC4c5KJH5sbc55mB7|-6;s>L=r&YO70*{?+F zXQ_F5yB-4|h@#lrB#;6djSBo(xH@Ir_c)W{7E_>Ttq|;&z zTb)+yW}eTaFK9i^{y_|%Up0GOxXf=ZYvh72U3bo_>FdUR|1!TnYc_a}bhMru+#Zc; z-9~QEKd%-C`Q2#88D^*TvtrlDm7i2|Tf9Ce*S%~mlHKNhdfF=Or1H7!bu)L9d?w~i zRdY%Duh|&ZVvQlzOZ(eTcblWapi(`U9`08bjZ9yzbLZ-4kt~f{t(UxN4x3l?{o<^VNiAx_<6Af*3OZ%?39Oaf&pR2LfT0&k`@jboA)GT<8fOsCyO26($)Og3}nIRaPxX3V)n@M>JmO?dG%m%UL7>^ zJxyz!d~PtzXFA;z{;of-R}1MT&Y)Fd{t&B`OrAjNDUN|}3G8hEJj~)*;w7Df{h5F< z@UKy=AkdiZ)k^6ua0_&f@y-=UFdxOaR;JB(x#PmPa#ic4 ziJxNboi5J4MGw_2R&fylYo}b6D$6V!B(HMB0;@fw{ zjjINDawC3DxTEJ-cONIJGp6TQb z+C1o;yR<3B(EEqXjMfc)TLg_X68Z5t;4ffR1TG++kT~g_V=v%$1Co@OI;^9IT0 zL~)$o6=UhM0c}Bt0BoC(Ta`q9&yZGpAYr{vUu}0JcbMXb!W%q)J}sJn!*--xw#(0T01< z`;6<2v+P#=41TT>2q$apW+(Pl)!efKPqWt3ca4 zce?`lqj)KuL%0mj;>@524-RSu?s<7rh`ev+p$|7{Z?Ws1A@`+zvwfA%7RRI?HRD~Z z@utuC1l|l;HLg`V;6q91$w6{lzF&*fYkVl2+Q7>I4-I#gt?(qU+ zBq6Kt_YKYkJZQggCG;iCw-5Nmi!s0oI^Q|)w>xM-7YBb>oSzk<%vYhyX#G5smgz9X zW0^n$y>%yE*cE!DkU3Jn87dkT=dIRvD(URa=!`$#;@J#N&i*LOLetEKJwggF!7?*=lCG&HNzQI4{M9C#6{KI1`0_ zH|~4&tbFxga0EFF{yLU=)#V-cL^ef~X!cR)1 zD#w9a@tXnck$EXmJE~RtV|Pu2Uk>CfXugy_sxa@^0Ujj*L)JOfT3GhUJpo??P7+>% z4}!nV26(;;+DZV=F6u&eCc6l7rL5%@J0-||e=T0sM}Z!vyp1iJ0xzbf!AH|RBTW$63a#Hm~lbnRhd;p)W9 zk8&1#$&b;6T}d{s(B=EYBL?vrWPnR6fWun&Go5I7OZnZ_wvM)6SK}e^&j7raa^%p7 z&rDs?XD_l&6l42P@uDfow?_Qn6>vWa-sS3?+U5_iNr#~A>M_47^a8TSz=sT-@`#^t zbphbtBA@B1^J~=MuR)x2QZ1zoyCXTNRpGB0k{)qz2{{c~Rk{J;s?h)0XHe?Ca^ll@ zJwE7zZoD$nwB_&&(c2XKe~<#rP2l@6@k{865-zkRtUcPo`aoCYYl0kg!uHpIZgAPI zMCbKHF{ybkbU{nD=PncVMRKcfwie9vn62H)MZlO!rjB{7qfYD`cF!@^D|EoXa|j*N zcP?~}7w1vfM+2zTkmfjB6ra>bF~{7up;uhjpdUYH8u2QxWL7aldMoifEt>=F$WB-A%)xC%_(RVNr#s0L z_zRr91LFJBnzpe`|4FbtseEgIfk}IoABF9F_+j%|f)DgjrZaXxGj-TQ&rg-loA@5-IascCtRJ_a^#uiZOWkbHG|t~><}upP%TeF1+x`{$w8z^4Q_JSYD)>B+92UeL1ld6G>gZD4wzmv4p+`ZJYo=SRni04a97g)yR7ZiU_(WrC&TUm4Uq4- z3?7R(Lcf^HCJh@{$w<=4cA8C$^<%P4Mpv9S`;e>EpQ*>E|b(h_JZ{Hzcii>@ebQwM*~~ zA6(#990HdQeP>?knsDTCE!z%qsy58V8vDhzn9(3g>xq<-{xfYMlA+@^O^xg3s8qm$~+ zu%T*Q_ynTXa|wxy*%V@6gIj{FfC2fA) ztvkY}z-!Sw7<3-!Q&Vvm(mS#)Jm9C0G)HkH_j8NR5>dz*-q$7jK61$`_W#H;L_A2t zp4U&lA^9CYPySH&zp9ATl3x@yE8?DzW7(7eKggRZ^u~Cxjo4(eY0P=9*KWi*Y~RTO zvTDHil+ZXL=FohGi^)r>!1#kngLy1$`P9*z#7C&=wmTmJ&E(`k`u~Wp!9hcWYCKXSy z{}K5&AmfglKKY9}mpaC}N`|}qljk`!MtgAw8Y@ckkuph|IO#0!KEBE%OH zEPK!Ql2QhW7W|TTl;2_>hP@I@b82p~fu&6iU9yj&WuZ%Q%v;!|q+?n#wq4p83%J4dWuGYJ zz9=07c_fNGj8{hHs6o1AUwPAm7%(Rvje0rH4=*~IlsI@Db$Vc;>laxY?g!Ehk z2V8p?c}c|t{MqoIIV3ko&mSOG-^XXS(38S+5relTZN3COSn;JQ;?lApT<7uldGJzw z^wIML#X;`|BLNr1dsg}W87}O0Vu;!aY3mwtcfG_Fm?c09%W;2RhADUXBL zVPEgn{|xz+)Ewq{41^9%{2<&{?8Z7OlnYb2MLraC5cbcKy$Kr{I%F(GwkZ9rabNQ$ zmUpaoCJtn*WnDWw1a9s=m&-YK{e}(B=c-`?9We@?5o^kG#yM2slY_3A z92Masb7M<`Q;I!9HzFN*RqhVc1N(b(>D}|QBr=6- zCiE`&cS$GgtR3%^x`Fb`kxst`?NQ#y>X?+7+p~tfPO|YNo}F`UCuACIZQ3wnlfTSUAX zIb{^{?QuNHT$745a?Tp%J1mbc7Y&@)G2=|quedaCg!wZ2?zEpQDe^%zpGBO}NLgLgoWP28nkD&{<;McSPQ^h!F{Sir=*ilw$0 z?T01skYL$C-Z05j#3@Oh#VD8eigm_ZDs?o}@ejnkq&|Wws9ue&w2T<10#b zXus)r1rBMRtocMNh4=`3=)~{AyPEqI*!6>pXG7MBMvcVz+~AK2Uz+&KjdAXd_cC~8{9xb*z#o=H~=39a@}? z_3kphv?9+>>7~nf6U9u{p3kcvinBv*JN&y$vm%$MP5x-;9pgs)rcX8rXcqFSLvsyb zd)?NgthM->txJnF2aFf6=gvW|CV5KWURZarfxE>K?}@R59;(r-+;Rg~wX`3roI!Y*-e9-oqbQOjGy zo)YU}y>9(|DftX}CVZM~?;Njg=ihd|ue(l-mjYQ|2Af z@5-La?@@djwLFx&m=*pY$Og)dfsdPE2EBCvy_IvUpgRj+CY}MEG&`I-2idM`ig12r zeaa!}+zGRmN62V9*M*;9Kc86(=FQPj`x3I$t)0PIj6CignIC-JvEua}ofGe@%7qg! zGxv^MII;oEz4BUeq45B1i}Q0N-#O>Z@7F^d5;+*>a_#cG^xRvQ%GV;sarwL&m-zh& z@=_6#fM2UA^;5_Zt4@ydZz-Semh@QYweYdTkjD()VgfmJl=pj#97(5@oH?GYyWVl+xN`9zlgngAR zZY0|ZXR2|^@f0wCzn@~G10Syh546ukrZg=UmGu`R7|lz}cBJB;dp_Drx&r@;d*XExI9{h;q6WO$=PWV=ECAx(}Wa z^rLB?^RRSox#R_G<8%Ft&h@5zgXp=3_b9p$zQ=7dx0!sMX~?kwctcQ*)hfAa;1T;Z z-U$Hol9bQAVEc6muEEzx^(=GLRe<&>7E69U(#beq zpY2d<9-LLEU8-_@#CdvYLHGwbcfp;9=o+>}NUT@k>0Rg>d4;eMH81nJU;cv0n=9~VD9QX%Fjce zFVlaJuW<%{b|t0g@?GmdW?h@qeO$V%WjNnk&D}?tyQms<7vGiL84?dt`{bHp=uk{!#0x}R-_?1%G^S%Z-kP3e z`9uo;;N#|g2N%;0*$E}ofuYU?y(LgHVd1J-HzW2$dOYb;N*;)OmUaD7dOwu65(d}H zW>m&5R1H;+#<$)zj?PP0gHzq4`AocmZeSMbf4Z?gR ziy9B~B=McC&Eq@Bl|$V;bUxqb=p}*JKWgrMdlcu08g8zeldysw4%>=*4~Y6S={sh> zky~2@on6!%a9v#%F)g;O2DXf%8cx)NQLX4c`NdJE%{9n-9dwi#QfwN%B~a%^HQZDu zpJ)5g^?SMVVNFxyTSWci!&j~2FL|*SowHDJ?FgKy&~FE~>fBXqi)+VWcNjiMsjK;O z#fisCd1U7Tt*(pHzU^gs(uV8^jnUdL@bho1QOkJ4=e>q}2M*&LAA50{e<6=4sNl>C+Z-sF+1qPDe3ycxJ|< z6wftfz4*-3VSo#WfqMD7%o~M%t!z&-rle#QdNZ*N1X+yS3CKLyP{?Pthm>*0mJs9BqBjwt>C$Abf_1cdgzp=R!%lL(gr`<=1c4*I8UWSKIe1 zY=c#6r+xycDPOcuBa4{b(luMWQ^baXbO(wdUy?r@w8*gzB`bWsCh#!!n-cB7zYY9` ztaIPfItcPQkn3dBopien>=?w00dJ-c$T9Y(qP}PlC!eUbRyhywuMb@hzEB4~Qsm82 zo!ZEOP7GgFe4FSCwXA(9({Bbf@Gq)|gdTJ2pw6r&`(GhP3h%HFSN2;_v?TU}?@;4E z5V^T4_=9(y;=Cv~tSstj<5)vqu4QmAAU`1ZrA;%s4hrjVf0udLY1soSEn*CS1?Um- zwAfa3DtwvQZ!)`sc>()IW$m!nc46Nt$0&xWUL7%d2fkeKx75c%{I6~5>1Fu3*Rg55 z{-2tgLmA+A((_;X`5V^-YWX1jJK8sT$KC)o9*CMlH8yNHs#P#_`4RdU@Mj`6@1Fml z@8x+y-*fjN?5(x@9KpSd_LPoHxe_AYWX=yCM!!LE5mDPCeA&4mznr1_$o{dUI+H-0v-K3+*WgzB>bR!Sg&o=0g092kPMANWwsB`A9V4f0 z&_qs}w$nwAG}VXEkbOaMIwRJ}I^~jjXzAY;wI=L0V0q_{Z>TXiM@Gd8ViXHr8aF|_ z3ObPRhf>c5(-d#(#QGVy?ETkJ{TxrnLRG{~|-xr&x0&auzME8i`g##qaIw=48QhUpR=Tj7k?@1tu*gl;cr>QMH%RQ)c` zR2>NAjX|HGzC6_gpV6nUC&=@MFOcJ_^4u?}p4@`p-FbrYKS*3RW1D6!zlCoBGJJ)N zqH)cjZP)(Mb6b1ScolAEFdPu;WEn1E8(Lr2FtT(*Sr?D7m!2==l{{O`do|v_3t19{ zp9;AYE!5^opYhyXFUt`PlW$U4V&M(=u-4biShme0rB3x3zopsHnT=@0N zYZ-Ty4xs!CCT}%=UrmRu{cGh2l5L}4_bzBs_D+&G8QYIP955qq(H4(rJk?gvKxivjA?&We;bNqM>Ou~6?ZmB;Z; z@)%Jko*bxnq^TReiLHNkAI#eDK(1c_{kK?_j?mAGM2E7U?|HXMlPoz~{?7 zRr{U8Y&v_2I1g~Z?L&Y(oV+C;efEBMZ?1Q=r)4m}o~#&Hfc+%bC+u9Wt}po>e3BLP z9YWq^?C!AUoI>wj=`1Lwj9x_!>W;}axixQ9 zIEZ)2_FJ)LAI?#JhV4gleW>uKzC{kDrUinTw0-&Pme%3On2Fh=j$vr#Mrj&@k4ops zXgrX0v1`}hC>j@MPw^SucVx}nra+$`=*Jz0`*%^!s;EVy{2|c?nQN`6#t(f}p)UmJ z6TXD2ACBEDksofa0}WSGZv*~@#9gs*#j zM!XkSU(8aTCD)W{y&8Gpj2FGGxjzHHxdxW>4dXMD@2;mq0pFw5_$7SpxN`@*LhC7S zX%or36x8`~A5O#fFZ)}1@XfBDQ9Ur(mvMDIyc{6^2f7^pEyt|jZ`!XNOBa4-;3)c` z;9daEtpX3@{-ekpzN+n09yfA9Wlg5S|KRzPGwS({F{w@jbqHC$tAO-1>IX08Fkwz| z1u;FpWR1T?z0RVhVHXsIFnxl?R}M2>K5VE3sNs9b9 zvmesbfAf6rk%Z}fO!tFDE@vNd%|C~(jZ=F{6P*m}RjxzCSt%SsE+lG3I42ysnrX{` zM_c-)aW2Sl$cDv3)jU3$HhW74j)k7iwf!{3%A<5o3*>qD*_D#$A5*+;pjU=@pM*L; zs#h_4NhAMh3LJIc$qHOMm3^(vzQtY{xN@z^cP(&d4g7TXDd@(ECsRKI3BxXGy4`1+ zG4zw~;hr4qh3=WC2I8%kzG675E_#e2uYh#ISopqSi}lg(7Ij!BfqNi)kbFy>@q+6M zOq#OK1%2o9^1iMZY_FL38~0xEnt-GDtYhO`5PeV#W!tKKvUtp{>IJHBJ#>tD9WiM| zGYor*Wl4Pse;csxEZU{yE7(wq#~FPZjCpK6!MgO(kmQByXIXO|x293wUuMh$+>$&4 z`PG$KP@N6VN6rs^k}C(;D;N*8aCxftJu4=#?v9Wj2i_>dZwh&4~uiHx$cgAn%+AO7&dCnwv1L|ufcP=sPP`dF9;cm z`$^)^t74>7#rZul9{}(WOg)6b{qQ6K!Nin zKeq01u=f~fE1oO8v9vRyI~FKsNbp{%PgSGHxnp}@*!sjDdugQ~2h)w|ixD_z#ILbu zzh1QzN2N~=y}S#ePf48bP=Xvq&R`s$;XX%gQ>Z-?j-o~hdUdyPk%G+k;wkCc$X|?e zUxW7f2z(s96JXn;FA8!tbI|>l_&M1}&Ap(ppY`Ga!WX3AUFBZtIwrX+hfLWA;2pBJ zy^FU}mr(Ncwp^_M4`+u>x~Ud-vk{C=&$8D6wzKv@Vcej8z$Ng}Zxch+aeIg$^YT#irdHIm=--pN1mA(O8g5<GWu=FnVQe+*L`ZSn%O}Mug z_o8~>LiD2R)rGB0cTREr66a``xmxZzq>bjyzrb2Fo)VtR{>Rk&mDdFxuD+*U5uUY> z|H8KIMGmFNC4r8F{#JCKL8{k6elqOsK7PNTd!%sJsoM`v@f?@7HQt}Ne1-3zuy8=} z55XhVI$avZEl#)_i|`xw9bBVc-oPK@-BtNWWV7PG+xi~-FMBRuKz}&!xw7tkqI4R+ zU(RC>(cQtYg{rW%rFj=BzwKT46%(e-vR0m|f~BTeA!F5DG3&@Nzg$uN zJS|^hLhj-on2hE>WpOu-7f;?uewupJU33eZFL~~XVw_)|*XjC5+vqh+`U?37$bRJB zH%s&!FV3x_Kbh|Y=m}a^S<0VRpGl}5pm9X&W;Xmn`MSYhZ&|ht3uHGg-XyO>%YbZ( zVBG}|Gj%yv=hb@hopU_ipyr(P1=WMAo5q?9JQsTX8|{_j4TUaj1ISO@8zOz-0 z_;1@^t?eKF&eBaxdxGI!w(y;7krg@jQQSiZ9-9S^LkvHgQoajOzv1S1>$xpDb$uDj z@(Z~(5BEeO&N;Y6t^cP`+hf(`Ts9Mh82SeU@*7e6h+Ic~znAMTu;NGJJjjQ2W%Sq- z{$0y=!oF0`bGVn;>N&!4KcMz60I$&_2tF)p?QS2ou;*1QR@D=^^Wp9T+>x&MF!aw# z3jH;2P@7Hgfv+C@AjtPF{duDneyEaKGBOZm&JV|L4I8Fe@T0d*X{L>0lt6})vhXk zY76-^wFK^Uh~MBI!x+(zWvd1H>CN0Ux~sM#eWi&$@(~=|?^T$h?sn$h%cpQv{Pg%a z8Y*^7cTzZKmG$jTVFSt9z$NRjiQy3C?^W(N z?z5fAoQ9CGUEa!YHhnJ?FKAGFO64+_+`GEfo~7Sh?k$XOlmpfa+zRRYVfVHXIEXXx z>jRaPkZbS}A@_p#G3@Fl*!TQnH$FDE@a2)Gn9N>MS z7d_$t6dOT)E5+Hy&VKQl`a0Zc&A!|@z>oMK_YXX z`YN4ZE#HkPd%fopz$>(FdU0;u`&@ZOc}a#|8_FYWU6*KA`M(QY>mKN`wkrbdBPr)Y z=Lw))-fIAU$vdgT{ZvlwhU%PsHni-M0v>-Hl4lUMC)How;76WXXTtknz7^b~DTr6o zeSG=7LK-jzE`m;_?N2##lqV?gwFA7T95jx32A*rX8hX1vS(bMBHg>q{f~8Q$;L~;PsF?dJVW&OAv@>LNfb{U z#6=Jj#NCqOK0~_8v$^Q1eNsN6ncv2`fEPcP!|EW^PoQ|QcRc@Iv(^}hU*g_T{8y#8 z)PU~GRDB}d{5^HY1@k_p)#d&I=tNE?DffLKY)a7+Kz&AQu;9UWkC{Bq#i8|V%tShd zdkrtoQJbRRuloqUx&O}6{-`gj!F1M(tV8A10fnR^2qL&-Rr~huOZRz{wB^A!+@%0LBq+B2%1NBe)wpKQTlJX_ zM*v5mAG>rV{UmGjxeM0M&Y`=qucAl&UF7d0x-*XML=^neXN$PF%Cg!|b9)+w;JsmY zg}e0T^P8FSk?jf1uaK8Q_KU6WzH7eZ?r#|_!Y+wjIYZP4eUtlq-kOg3p|N>scPSk5 z*eq=KcUI~d1V7w2E%0#U>4UEDlh@D>&<&8wNB7tbw``nn&rss1h%+FD6R(xQud|AO za=&Wm?amSWKrU~R{KBGlcg$sYfgEu3=%PD~`R)|tCw;0h2a&I&dTv;`?yy-vo3xHM z=)tl29uS#3!udrjdMofVRHtgeljVZopC{#;nLg6QpaOQkblBfJP;=r5>BJT%R&-mz6=pAa^ zq9)dRKWvbXCD6x^OY{Bkbl;wa*FNsp%%!O2knapdO*(Wk_g-}67xCB&66XDN^bNH) z?(gK$1Lt+r7>Qg`+de`}m-9gb?|NYVq0R|@aojVk_XEDMs^3&&_(r|sP-o%R@u~Zv z$+t*8Gr)}Rme9GdZ&_a_e7gQVK;}N7-zwPly*fO6R90TE^k2kLI{=zX!AF&v1=VJSZ#Q^mfR~Q?0Rdko^-a5LmwMM_MMAevzyPJaM(ePF_BYu+a z6FG1L?m1%U01B4@8}PvJG4Z}-kCAW0##`2tIo@_MM4#1R(VuHaGFHU+1k9<&0OBw3 z6(XmwNPVMVvj@a|(Ay;d4yY};nTG5&e0=J8%LB zrykowJHpq+dn^Ug}Qs(Uz|9YFn`#)f&Z{_>+Go> z&jr+uqqj>|jAQ4l34iy>{7h+&i272>PY^nVmv@p(rF<8%^;C|o7vEIgtSf7@A2Zyp zV*ITz?s;ejF!tL%7~8lH5BJ;Qe5i*ve9`loylY(a=vIAh5Dyv_(Yw2VT0?Oj!be}! zeQw}$g1l${6WL9><(T)p1>FX~#5`ZSUs1*Ik>ZRvMuqsDzXuNP(Y9(b#JTx-qPS-g zxtX`r$A`xuo3vjm2jFLb&w|FqJt2rwpvV1V#41k9hz~Z<&s^{$Z@;CqD_hYwM#&MJ zIbfZdH}?B<*Cg)7rC6W3r#nAy1?7pzegUu-hKR{ckqhOH?b8cjd+Z*^3m#vt&97oz zlKZ{Fee}G!T3i?l_@LjRn0=%Ak?`skARrj+}_gkQkLKk-< z&TvnP_NPE^ar;nJthEb$Z+H8VA2$g1p@$8#u?G5VELodvH;HwH#m_LN>^Hh(uj}F% zXj<5YOZ36~?iTJ7$|7F|xlFlKg6>O}`E2257?{iA&!gZo4beG*Vr*D%r=s@_{F7&F z$iDD3C88Yr2ChRdP~^c9H)>EK=*61J0B*P!-H^spLtpWkl38+;4qA1h*N3~R34b>UCChAS&%%!KXk zV4C;UlKK2EA#n7`SIWbe=V|dGznl!bn`lYyg4U^)`IL36f$`+mwO`riH&ZdVxAY^w znXgyH&kwb~*R)?;9bM^Ds&_+RpKZY(;Hyyf^quP`EPG$ZOatx#6nyAY@L{U6pk8vW zy=wHTgMJef%ha+P`69yp7JaTil{-9@egOMu9p6q^UfL=6GWV3@TpaQlpr#S}EBqo` zUO}KmQRdqJ3=eo9Gc>WIa3dA=Idm`{W z*e+Jz!O}C1sx3)6Sq4N}ooHBEVLL~mqgCmONh0zCjx8%=i{rse^5 z58LJFr-$cZ7v~1D2d&qh6FP~o-^Ftw_WPclJ5JKN9)4E@{PXMl}NIS@B<##QBK3&;nT`ho7(?$?1=+mu0gek|`4vF6XAz9x4a zyw@c*YXD#H@iBMZZ+>Sn^LF<>sfmMflHhv?m0dx;%@p7qrI>uwm-j4liV#y%`GG#$ z!Cg02HZoufr#2*2(+`UumyH&)}y zd*LfIWK5U*=s5E->KbrAA#7&UpZW87LvqpJ52F}C(EivL0ND=z7S&Af*nxYt=!WE5 zkz>uAW9{11=%s=9BgevIywD%-@xa)BhZCQNfTfQvteBgMb9()8D{b(C#gD9WaN{@2 z>_*yu8+)Bf-v#wyjGFi)-6KFErzFCq@ezImVg6TWZe|Iqz>{7vZ5EK9}RZ(iOe-WNR7qw~4vZqUUY zd7;ovJTez^dThwloAOK@^N~FO>ge-~`f>F6G-D2^CszA@7kM`J-9Ey9u4D)2ida2X zb?(S#IWh(w`Rrt-%g=ITh!;?7n(ugd1AbuBr?Aa@cEW0V!=`OLpV7d`vXi`RF*%H4ah#F{zEZLMg(xAOS~-oK?|^nD@wvoel|UK(!xHr*@bkdLE{ zIy%0i9rnc-HI@_d<*0MNs~>7r{vY6kzUk;8kpNEbp{Bsw7eMCgP(JSw_aNjoB8Q0Y zGTW`=4v;$TKQPbD#o6`t+eH78=050??~71+{+;7JtNLZ}{SWA8eT5#iQ@X3rYfB5C zB-_pIz4LXdVq9sPY1_o=$;GYzMwaD3clzYsc z3(CpAd*a>+SBDh2NA~-+EC}kmU_YOfc@fYh(XU4C1Lt)xLjM!P7D-dyKVTlxa{+og z)$7CGjq`v_qx4YLyGiHa``Rz~&N{gd^a$;O{}bJ)y;CoA4R3ruL?0I9#^L^Lk=ud! z(XaP{eNF?=2x!0fq$ciEx~Ti|EbR3w`VZ_)>Ro@(dg1G847br+p>lzK3hG>qe$y^K z)A_^yO*k<*lr_?VrgeQ-r?rrIRJ-F2couN>0Qm(s$Vqe1uR-L19#cbByzDuT!j08px3)IERS70y*B7t z+z$>Jj9f7EN>#i}+mh-&dj5Ww?%Hy=e}{M;J@hY;(=Y>52qb9uxzOp2?cvhL6Q$OcLlIur6cgUMYkJx?% z^?sf4;r{t0#(UuMD;>|{=V>l6M|f_lX%`E7*qe_k<57L4*Eq|51wUzj5`CdYvw&L7 z9?o3CZ#vPuU;5G7oI@*M#5G-|^z8iX@`~#vGdTu&y+H0DUk~daH22Y8pk&CS zj2J_#H}LEpxw-i zPv~0uSOwppy|Dk&#d(~|bLg9h&(XbfNs;S>`y)XwxM$w%l>yy-KRs=g=pHC3TR{sh z&fcIFPVqW@mg-s5oTLxDS@{RZr^Q6w0W22ONmf<+M z_zAI^+IYAz?R7SuZXD;znd6h)lf%1u& zF2`!?PrU@yec6ve^Fnw4wCcK(a~9dwzc!Y2OFG$r}e>9J*R$tUjFh!@@Fj3N;J0VO3lW0qLoahqFaeZ^hvT& zZ?>bcRx%#nZYSdLdaIS%jx|!vR6CiBZ`GYdGP>2S*W4QE$$A^)F&N!PrZS6tmp7W7aL!@ckjcm zJ$>}m@Biq*{ipZ8dhgLA{7rxU;C=k{;n(=LdoS>h{*oVm|GiE6`o#u~a-a9W(1~;= zaV zq%L3>DHo46KNceaRuSxHHlK_X+^5ZMYuI`!=l?MN(L=Gn`h38p8b;y(ODecYbZJHys!re8>jZ(d{X_R(P>%|7jhaAXbync??`k#?^&aOA!P zsK%FslJx-!2B0v1Y1f@BS7@y2X zPOIG;wbXQkcmi4y3mJYuM$D=vpMsiO_o5FY@5c-n(@f$i4EtiEKAF@PSegPlpXd&K zxY?WXPqD81`U%2#b;9~|85FN0!})X;X|y8sQG^GUhZ~9T@?J+;gJ~<`SwU|!Yjs)^ z?-~m8VWTy{BJ$$KD?RskC7N3dyX0tbH!jsXOdf~CgQb(-_R`lK{a;W92N{@kaa+ZgY z!T9=Nq}%Is{VT&)5ds_|?U%kJLHT8R=H5V+!KHg$1IfCq~TpZ_bg<~N#X)qpj9!92Pl5rRl+LtqL;`7H_G+3nBtxupE zfj4Z@6GGgG!32VW>p$M&6%ni%TNUyVb()W^iwBrwRtHwMZZPv;0R%Khd2Ou8g|b?j zPc_;2+K2V`+dJ=Pqwl9a`oZ^1Y52uSYZD-ttb9kv$tBs2i?vu+D=D{@!?v0O&RBWzhu1bh~p4Z@#}8~>wo?zez_ zv&eO~*X+uZqmPB)#xs*vb3Acc4t*!J8`9TUu%_qjc58BPn}oA(k))(vwTgKC3d+l| z5DOV#_e?DqTZrIXLY@LTL>U`Mo$p_L_+rDO-YaQMIXy?hwy-;aFo6{RE4%#-@HSX6%09%Hgq)AaBd4GQ?*>kD;6ur1F(Kvh3C2v9i` zWFlf8z)e~~W8OX;mu3@8@jBA26MjT!<_XkRC}t8VSRJ7gQOGLBFm}S@98AZf%ycMqi z3+@#xZQQtMHD?}{=23Sy2_YW3y5hsg$M1b26h8h+LKRSs+!K81!`{Ql#dZH9;(US= z(Gbvv!9bz8xVoe}6CU5}_Z*tWWwZ7?AHMfV@AXIYls-q#`?JM5@6Q$?@82YwOSUPe z$3yn(Qa_*#@Bp!gtO|Ld6!5wUv5SY1#^WugHDRF1!2%!#l+}Vsys*ZWu`W!1@%kgZ za)tSqtzqN?K?+0;FE-w%f9btkz7Wam6;Jb7BxRiL@ux3f&Td2=xKE4u%}-!Fz8LYt zdxc&902>m=DJQLieZ_OFrCwR#BIdrpMLi!`yZUc1+XRvWq3;2quvFN!?hJBF4*4C!Mrk|1?# z#NJD9#Dn4WXOJk9XR!X24m7=|d^36HbX;takd{(R(4YExdJH)h&YSgPFC};8{Sj_*R3yo5}4(J`V|MIhURkLh0jABsSS?>5106Y=Nm>tzeCbP}53*sm zdSCQ@sM;?Ek3>FJSQ!y;gi_^S+Uv+Y+9`OMG{Xc)pGG8b$k2>PI15k6T7Xx>dYK}v zO`PAP1fGKEY7EV%V*%sA+6XzQczz>P%pjj(ztg}p-2Z1vqi|(PHJ@0g^15KA{kyH8`#`h090Sujg@IK4K) z6M2W{;$Hzn{)5R#Bb=Y8UMTrZOhLWL08wlP0i+S^)|ZnOyxDNF20Mkl%=u$lGRRo0 zOPfRlX{DRUfdO_NSZav8cTIVp$s9V|Z;Z^Lqx($Yd?N%7f;VG??3|bS2twq{o_x?R zXqfK61eTGq_?k-N$ypse7PKKVoIB70nJ(BKE``62yRa8eoo_PEe{c{k2| zgO%#9^iUdBA#9XQH08WG?M_CBd^#? zusiX-Awg#(HQiyt9wtUhkD5}ELT+Lvd^#IzeaQ&4S$t6n4Yr;zsSnEX`(;vIViX*7 z=3sY{?)Dz^qP?Qvh1SzavP*~9aslr$J-Il!+U4S1iFSgpf`H{0JFE9*5M|l%MBzaR z&g~X3FMRP}zx40pOnpQq1rU^7tAqvc!%8fmD3MwzwA1%Kv8R7+>y~aX(tb>O?>uJY zN^dK4j%7Yu#!z9gE=A7BSC%*lHGKRIQ9-V2zz9eWeL$000DZiE8U)*G72dESb>!?u z_!2Qo^7SdglulVXf^grZIu2!gDht}HcxjGdSeK`Wwced)7z4bdXzdEF5%@S^HS-9L z6^~(>fm7J#nm#3ZR-MZi^=}b5Ksf7x4`b9iozzEDg5H?D8Oj8t6PA4I#m4FN*m_J& z#=O@s_|`!eL{3JX79XLpaS`0+4a4UPF8rEiM|6AqdQ+0qs`Hy)Qc_JxxUtbjjbsW5eY)WcR!p>}BZL@>cwsqr z6>Cw%3Q#W)H$7F%G>E+LIL*KzPYRV` zUU9lSF`>JAu*_JDB{0Q>5QgY60BoglP5Z{M+K79L-!h_yY+(wK5vd@r^fbPbzZ*xTy zc!&59LGfv8SVv|>bIO?y$j=y41S2wrV_xqX0mUiBX4&ka7zrL3l5DA8jS)h@*zNfg zvM++DA_DupW^atZ>eU!e4v|&CffW*se1^0>cc`WD660jVyTY6inSU0xh_LS4;E5-$ zBp?mf3t@jdeI|IO##{x)^njnJu5jV`w zISWvauUlZ>WC^qobp?_3W=QntO=rx@X-R~-3cfb-YI>!|7!9Pxrz1g#<(-a^Sflss z!d<=L#?w{!NL!OKqVCxE7iP?gr$LMWsmFY!^&#;>=PC%-ocL6bnx4~LsoJv_|A~?D zQ!XqiWrFf0U^8&g*8)s9EvpsV0|Ci(?uCOauUSiTB(Y*+AMP4m^@|>oDVCp@->-sO z_$s_SCLEMOz_iDutr@JvvuqdBL<~@3I6I*~h<)#(OWC~5Pc*)(iI?DdfFDVFmBn&a zJ3Z7AEy7Uy7lTN45W?uiI!%&LZwsujv3klmvpa37E%?AaBU%N3B#0>J4P@EAABkBK zfb~B{yr37v6knFUoB+3~=fDGCX^3Frqb{q=qA>!XEv5lBiR`ocO@#ZJOlh~mXk*>!oY#g12vVyl@elxSlP z7a2@kS~Z(CGTv^{N?UldQILR+f<=^VeY6=ofV;OhVn$$cJK%A%-lXLyUbj#jSQa`^ z7^Yjl0w0AAO?WkCZg@HAT|uRC7r^eqLCb+RC(;>{?*KOEb$#NH{Xc1u@kH!hJOM*= zI-S!)Pj7NGZSPp0lmN1eI)O0Bw@luL3v2-NSQ;fORFM)a$d69%5 zV#ECFsaatsw(rSUhwC6%>K|CspeHA)92}k)p>(gInz^q{RYY_ZF-Znsa9MW{HHKJ; z7?P_8^wLbSB{WxGtkT%Lj{A>~S<#|k75VhkFwSeY)`%S=GBw7+68eMJ+*=~F)QBEU zZrD?Et!uwZD3DpM#)R*Npd6IbH0=44R70_P4z=QK(D|)=T{L;Hr%QixXCs47h2CDz zB}Gp|Hk8d5G|r1*2;E#p6o%UL>~dF>n8UIJX)rw-BO!?y!vwVFW<`Whl_q1jaezV< zVDbR=S2~r{ql9Q6G0k`lWEz7l3U@H+WYHfOq$^l$P^l3;jk0#k5#%;u1G!97+1@!Z zUJzo_c(g79K(eAI?QRJS)Vi7JS$Yn~2ZG0zC=%=0HoFw_daNt_i_nLV!vENNog?nyMR@7{KldiSxVf)9T&7A_@uJ2nHop zOCAe%52}yNG)N>;GR1}Coj1kOoNF03@uXC{uX%9S2U5#}`!S^uvzJQw(1c@)i`4MZ zt2z8@(?|o^G1OF|h1?iH%!qI}{6QeUvRtvQB=!1U8OPL!FdhV-w0+#N!Ip_UQtd9LFVU7wGu1n$lJN47nxnMyqo89X{jLe-2>|p{M z7!dxr^NM%veSQ4iCt}XmqHBWXdGo)#NxcDS1mGjH32q}`jlhqr8pd79l7YlS)=(Bt z41hZd^o5nWx%(Cdu)Ht3bR!@Cc;Z4~I#&7)J3*!vAyEPB_7_JQyoSqr!E5CH6s zDU}VLp`ewk3B5~FVoZQK4vJZ|2wsqw4jraG`q(&Ddyljfdw+5gla~Ciz?=JWDK5re zyN`GvVuF~arw5fL6G9?lH1$yMc-*7TJyu#XrCefsEVN}QmGliDgnn+$HM%q+`GDr7 zKdGaV_)7$^&>Uj``1#(}@W;w%cuV5hnj1~V6cN5a8Eyh&fQ`j6*Xw8G^-jk)(cX~w zBdxbSxJH$`_}^d0)}(}w_ib}Q_8==3WO6wG znWtVbpI{)_V`+BhXBH-bi9y22${sXOmT4aWL7V=OPW(m1S%t*jB;34rZ^%8khAhfh zL*<2QUJ}N!!&Iu(MpX!ED7jJDD&`G zzj1m~s_A)2E$`4A4~5NWFlMulP;`8;5&X7k*13FjJ$DYPU;+XwdgaK6zy%C$FAO>P z)ycfq>>o_ia%d3>TuCO>mZp6=>J*P^TUDYj*(#cY%&W0Q@?SQEoihhSEUA5kwW zswIF<%p_%;t5yl{aC+Xx^K^LH+MzLAOe0b-5h36OIg9vVzbQV`U599B41gA+yfFM~ zK5mhj_PYd=g=hD29zT*AM?gCf|Lmt-JPUX@EY>E=t#&Ri6-8u-)-6=9_G6srho80@ zYR>hxdLw19u+At3+MJEc=q@$M)Th|&>xfWYeL5m@9=sZWE%=lU-6aS*6PsNd7;JxG zA+f?sdc>V4D56SFn21I*T69V&2V%8C!S*1gR~XRy`m<$mQt-RGTF{?y_JC%dLQ&8% z^uMEm(!p0o7i-x{lXo zJ)DD#Sk3j8Etq^t&ARALCS=*2R_SsKR|R9W%kLjSECfE2fUuw^yuq(iY~-a_59zwb zpXLDT!HD@D%?%c&*Jg07AmMLJ=zEfea(SB;(e$y&(9B`o$Aeg**r)X_&NV1=WQMyM z?9B%kX?>mJPAfGbl)eLT8oVCFbT&hf=#a*wl{eljBica@(p8S$V-BUyBL{*67WpG znCv*=r9r;GZ1)Dplz&JL98~uU_s6tz1NEMUcb^Ct$OR5P=9G!nfp^$Xp*C0amyvuKGcq0GJG4h6L>J z?!>nYE7(erf3YvmvQXiP%nrc+8u#$%o0h^?Yc6l;0jRa}?LrZ>4x^2zJB8lW>S@tq zAP!IRNR9s1tJj#cmc*x>GUAvh>)D}Du$uJ@bj36)Jq0ikrQ`*WhK0DnZ@Y?`*6Tva z)f$c#Y@g#7!@O@HN*q-#+9N;&dp3vV*^gLpZV!A`GZ6IZjAV8=5lO0zqHVH3Tw=-O zw)HOJ^$#rdd+UR?*tGz+Bp0cNi@yvoiie2Hs#B!jjR%A-CIqnOEAcKIA{_XX?;Eeu z0nsZSAhDONH9NSTJ?FdVsOTM}dx*;eP*F+R8ysUgPRauqgBp7^HyD6LVl0y{*$Z}G z39=B>3as}-)Yyyb8M@5V%C&7E{=^jSUhs!InA}F7MXOXs#b3faU;jN>&Qat}yz2JyTLM&z{EbFo7sREVlX7!WE=JZL1XRTq;y7-BRQ z70Q-*^f(y-#lXV}T-~x0sXMBXE#hwPhFNd~Pn4wqYG6ddUX&l3>>}(u!nq0*=;+zm-mavTApVWEFQR8Xs0jHo<~qSNLKrz^eN(?L#4LM`=H+6644KLoOdZ(9W5(c)vGZfGwN{xgs3`HE?jl&1_0<~qg+*twWL^ev~IELjQXOAQUe|o z?lf4=(-}2!dUFlPuD8>xg`1rA*0Xxt_?&Uv!|o+@N3}gHFE-q7Oso}29?Ze{0q@Nv zR^F|J%I4HAqaoQ5I+`c92FjeeZ`Jor&|;Q3!B_>9bwqErnwVz|RKn`{NCMFVqwr;E z5xN#{5DF6V(=%`1c4cYP6_bXD)Qu_cphCJi`&jr|m+koV3YM%gMcdkSkY1^mX|7)Y zjVMF%(;$tSzSf+PnVyLvZ0V`K@{GHZ&eoKkVeyD~=f_<>W4#(oM8X-=dreGLH+ zy^?269IA8EBSCAV*X$C8=Lli4dgLKyHHm)O1d%R>5QoHVS&TCm{d9pl;ccMB)+ky3 z#l{iRaBwS*O*izwQyM@qP=kTX%S>o5vO;azYC>T}bS@ua)R6EBRxUu0=Tsh&%Wior?E@MYxA|(5iwoKm~UL*@Ual+!U#A)Js#{Cd5@4 zRCGzLOGL>Fu6PwUC@_YZHDux*0h%Y0DHY9O36ssg4LDPrQ`Vqq(_UW7VwX^P%3!>h zLiE`hP0<64B*}}7boSIfcMlqJi~sD|;Hq8HQ-NE$d#-}DMFk0;aXeP)#1!7;g;#WCWTWV(A1Ozo5#2O=;RB0TGE>&hIfZp6KZb*u)A1CcL0=DE|;j0Qql(A zWhv<4GqoE<6i*3_sO2sZ$k$ozUh2hULC#c*JP>)#L52oB;^MRrsp>VY9YyWxR}HUkqYHis{=|zh2x8P}CK7o8h9Vyq9n6zfj9L`8QWa#_|YJ_SpNdDS%V=FYca zR<7>q@)xiA>V9Ktu%CIImHqUM=R8ZzH6@8rq_l^UseAXX9!C0H`}v8Bix5OwwbCI{ zR;)_1$-Tads%##*kAftYXkl0Od&=udgk&pWVXWYe+Lz z1l}@p)P=z+GMC25XGBhLhx+qbXvv^Z$gz@e6W>9UxvV=xaE0RZ8(^(zfV`nh;zG)+ z`b1nRg^L-eyPqAdCMJReHR+2EvU|9%6OSPD40;9?Y{vb*kYo0st9_X2s#tli<$IrC z4X-0tJzV?1K@9F$C2ziyw-#)A9FQ(Z3B4sV@cimA2ZFZeif5Vr?sJ?vFC~)a=ej{W~FP*c!ugjv$pu>Zr>H zMffU16B~i6Wiml^tem2e2ioN0f;UOU>*Qp<^?FNk)@6(Rh;aG^+Ff0aCy3e1N4Qu5 zm+UdyLCo%6Z*)1INm>(Kq{f%yDK!{~9F(3FHo1^MKE**0dZ;y-;G%Rq@}X#sz%PzP zk_JAF@`FanHFfQ$oBi4d9*N6r%W~}U_lCZ?pv&bQP|)#o-tgO;|{Bt#0qZX8OmZi20Jk~rzhDYrlPQ% zH>zk20NVIAVl+5i{z?D;(YL++oc#AYR-zGol5EtQ?P#o(jK{aziFmx;YNfVgjZ`z$PA219btjRGZnf+6_(+cP z$wq@hytC=X(@$PMMDJK4U+zr5@pPk+ifu>RsW>V=+IZJ_(rh~IXli>a(R|WOCELlz zt>jj$*=i=@7#<_1oMf`mh{c-Q$#yE?Y&?Yb6e0*i5&CRs5Pbe~|KAsA0pFMX@t^(5 z&;5;uKlV2(|Lkx04W0k?eYq<7@_*nbKKHpVs1N0Lp+7SJ@W21!?e?GilegL5`7fq^ z&-h30SAXT_f9k=v{_0=<%fGk!$KwBLR=xOL@)r;Mu&g_LJ?@<94(jkEXVs#GDl1mP)mw^KTI&f5Mfw zJqNIP+aUqw;s4*>w+Ba>Rrk$ef){X=v58^h4N0$dZ8XfR)ar+%9vGJHR;$&LS{g~+ zt;f!+rr+ICKQ#SNOBz&VuO9(Bfut}{n3zAnAvkd*m>8EaMIbRy5VP4hR4Rr*#l)Bp zFovXnqztz6JNJFR{<_s#EsO11)zozN_uYH$x#ynOJ@?*o3C_yuN(|qzy#zm&78Yyy ztxPTNU+Wa&^T#{BjCU#3Ea&5)lR`9nxZ?}?mi>|AoIi4y_szHS$FoOCU;1QwuN+Kn zCXeuWvXe?rt){}>q%YB1ImuMFH&^P(x%t{kDsj9T-#$t1Emn3m)4|ob`ReLsGPtvs zo=WbmS9a!-Q^~pY`u1k*c-gmD$;FQj3k%7mop`OWv$f(~twvHCu~;X$m-gM*2vt+@ zgYKGdv0KbkmNNcYd}pgxS+6$(napBrGkp}@&eYa7(>vWv{a~@UkT}lFC3wljz)U5_StMdhbdh4;z_;`6QV;9tgYMjy9^k^h6K zz+LiA7`Rq9kE_kbrB{B{!wjl2(j91C<+76_NJ|5-8qA(jPynx44ZLHYhM~QQE5*fp zN1cbpnOy-J+x_OTgIk>t?jR{5Jiub7G|_C8Co7$LZL(F$`$MzS4+1i_yHt-Jjwrmr z7-*8_dF#b;?C3yD3vP}*Ha`C7W7j4hdGOYi%hxVHGCnzp-}Lhpm+|Y-$Jn>=oA{$& z;_-vy6ZD$z+T)WO?w*ue&F%rYLUh=;PA9gjp8Jr;THN zfXaH)7N?;HN4O`%+yG9Aa^~2`AgYUqL>>DBIs8X`x5UI2CxR=?0adR8@)e~rQ%{5P zDLcrl9?!_f@?^7m6XI{w96}A94vk>!yU2RcLC>ywO4VY`_8*&UYQmIm*<+j`WuF``?U46}!7#dv6sP(aJVoUI!=Ctw8KeUP z6~MJ|;1^K>i3C48@D$My(1#*|c?_pCg#w8-3X{H7#=f4SE~Dl{cj@DlWn944ME+9( zmixXj0QhI<1jbk_8;W2oF$=VLi;an%ALeUa%!8fu7y;D=w$GM;ngBH%D-pI6c;OV; zRZpc_M%H+f>j^WwVsTb^6&NSwt2}sq4my^Rte5UfWmknhbQhHk^!O~2ns;gG)eNt5 zGzHkYAp&Z&2(>P$vgPSb`CFoK+ZT~qi&Y;lx(*cGN(VBHgG9Rfmv*@UN0Q29$ zrKg_a5n9H!`-kqDP)-e;G?nEVai424_SmD@hfCpy=e-YyZoT3m(;7a|Li`&Oq0v>E zz>KzlHy#FgvV~&4T8G_LWYQSf^w^u2>1a%X`mWvLU{@4wgTcYn0I2~qmD+P8KE#5fzDq|Ia+;!{)pq8I{{}6Qxn~+)!x_)i*(@ zJ}A7-EE1KyE!`mYN>J4Zfj)|4=Scd%QjKvfQd}7~uuHSBl=Q5{KY8)N) zaVHnh)Egq*>vyzUYS%vs=7)4sG><}d^TI!%4ln??#Eu;)bx_Hrx=FD;8?10_Wv(B<7_(1I?x+)vyr00u$y%Cd4*a1nnH1Z4laa;& zRVZy!4oK~=bQ;xmhUW5aHS~PP>59nyE9TO(9@6{Of`{N)2lK{7f=}oo4_*9*frmm8 zEB|V4X7{DqV8GxMufn3VC_XKHptf4bNEA{{Y`lh%7Dj>3*x@`NN}jEqkx5+@V@U#J zI(7AwlISWM;M@}n0L-v9ZsU<)ZBTq#Cbl^^WL11acN>~&(J*-x@xb~&eOz&de z3+QqoNclJf)hvS&bHMw+u~l7SrLv;$55DA@!1Z5q%~`VFn2)k2hYdA_g06BFQX5a_ za3P2_v==?U;^O}hLvtn+_BblerMueVK5-ZmJC4LDO|h>9zsMb?BYJR&nOiPhK1L|c zsA4mlY;fR;rs_4}N%->WSOd#;YaO*Ce-}B>tIYxuMI2MSN}loLg)w>A6{iJxwqr(g z$KEEX3@yYE^y23^#Z%Y4cf8R@9^vX?7-pgiL~LGIBEt6?(~vVkQGSp*dh1f~9LF=2 zNc0UMbA?j*QwsbNP!cJ+cm&9ea!n&=QZV`k-@r%pMZa*$ z(h2ky+l}HP%$6Wqpw21Hq~y*e#0J7{YTNFaB$?4KcK8)N9x~3%eGoGZqb4J>d1&8- zE+TeJmEBI~8_?K^A;QnL$0HpPF!mK&dx@DCDTeD8s+-b&lTwg1hn)rh3~k z^a%jw(kg)624PZG*ut5^NB8lXnm6itRQQgYkyI9D5VDQZvbkG8zt60MRV$i$Qc z%SvN!P3DGyojB=`)zn}H6(&VXMNO+n36*(Ec-mD5Gk|`B8 zprQwD7ChUqE-=;>o&h_Tb+`MVkan5^oyLjK5dpE`244GKqNjrYhnsh4+KIxVo*ft4 z%wlJ4qdebiNj~6L*&90&`;eC&YBbv*rxV7Qg;wp&=d{;QV#+*EyLq|0mD?R-G14yP zK8-vx+6p5#EN08NhtF7tGNj}erH+E#jcz@M1vRX4v_!aCZ42dFlilOkayXn;de zlVF&akPE#vI0=_H^sW#Yg?-N7Jwpl4&FqQTue%QwxNUV&VN5u3b4U zQn}L3p=ygvNf%32HCp&WK();@Hhm+_Y7FBbdEpWuE08uFYLZc$T3}0_v{Z(6`RKBS ziQVOq$b+MC>60ZTcbzpW-d^< zgvco0y`?48fw=8n*z&Dp@$5QA$83m1Mz`+UrAG2(aGLeZ9@ExAlTP*=+LE`6&7*mT zmG4L0qSwh&RIE1R3hEl8(r=o&wsPmfF^8$-fQgBrSCVvG#0JMhi}z9@VVK5?uo zaY`e_1aS(5U}Qj?(C=d{-aN~BOy#W`t&Ddz3}|MSKa#O2NJ?kH*sT`0Su@`{X1f#l zvDvn#SA_benLv>)urGP4I8=93ZFiV1CxnP~C4Ft?wMBx-A4f^81Ys>N)l3u^tFt(k zraOAdY9OkVWZN24B{iFrz-;~-G@0e4A&elYYl=Cer9P2RZK}~3$p56sSSd}*T~L>9 z!oZw?E&5u787#{ZV;c~dT->U0Yr-{Ul+h14E;LRiX~nGOuKE5fx(&txN>eFCY6&|M z^q^U!+Q*^xEJQs7zi;?sH4PIA-+(T3o;X4#ILAU95zrtjS2k?C6lutgGk7p}HYNiQ#w;u{)kzQ;}BJppx01E90`x zs}5nrsG&j82ib2Es?;?=0JSYPLQ;?XJ_}6X`hY4-R|h3jTNABjvt#hj*|Kc*QInMD zWAxZXX52Ufundmq6BSN$I6w5o)qaW0m7M+;F=1+lq$yi`n~r2(D^L9(cgIqUXX<~u zF>^qjRoNn$N$#^)Q3@{!hA}W^8-n$d+&HlohVZ@!4$`D3Q7G0A;Gr}+8Y3|=;X}r; z=Y&fzz{5CNBVc03VPJ1Q+aSoxfO9oa#?z$VxPqf?glvYaagrdUcddX(;wfEAp0ncs z4sVDyb4iDSjG(q%x|v%ObfcL`Wc`j3gwv$Nd~!`q%=A;DDt6ezspRd_A^;YyEFCTtkH-h< z6FxyR8Xm&yE^(xD2R$T;mSD*K^F9q%^qfGZNXXhW2Xv1$hD&s|;JO|o!U9C%LSzq& z+GYE&XUehwG_Geg@hTC^aS5wB2Hb^~hS(8)xA_qmHi=?AlWcda;;ZAs0<5glrm$T$gCyd9{Dn2DQk@J^AyO9$XV=se2I0?w zGDs1*I?+ZOni))9I;%A`$+#HO(pz|HP_O|V2NsXXrE&+L`}UNS5tLjQ%!YKYNEo7c zMQC7g(1Dg=Dp@FDI)_1g)nwZ6pjAD@o<)rSiyQESI-W98b(Cq&*~`*}2Bb%6@%EGDN6U*V1BcGdo0KU`u$l}&+usKsxS z2?Z06o|e-U%1D7ex2+#!oirVXiMY&csXEH%19Bn%B2?7J)0F`msYAqo=44`h$oe!{ z*ipL2XFli^W+dN(K1mfZYz5EJEB7hV@V=-EQX#j8$UCx&J5p)E_aZ%Fye{!4ax%{b ziA3o_K;BBVfao30@K~OMaRk(?jf+)LRqqfKRXyW$B##^}aZB4}^Wr+qszPoYj;`6j z$mqFt!P%;YtS5f@nIHyk!aJmu@zDrEw#fv6*rKR^0jem=BVqO7VsD?Zy!5k)dajPL zc=aHXm;`qCi;^fK&XBn?rFf7#&jSOs+|5yn23=N*dKcIPO!A;&w;K}Bkl?Lz+I!v& z!Ms6e0ZA;`5kZIHQCoo_iIc8RtfPOa>x%RW1}$L-9gA=e%h_#~=gaYf`Odk!+pIfG zn>e;q>uL_(?IWob!M#mqc3G542B>&oiycbCqdQ&1*V>*OcE=#26k=eqBkVRzdf=+B ztWd0}o?M1TLd4{O>q$EX&J>m^qxe~twKnT%A`l}y!s2SSUpf$wHBY14qSSP>PbipX z)X2KD;;K7MG8i^m8YRJesV0pG&Laz*DchlA2e2vfrv^66ozW0I^+`i?cQ@ z9Fixu)(+{Frb!oyS%R5uiMC5&CoyP2>`^Q zG>&qAWtJD0Ab=m4nqX-JEC{T~Szt6OLjdtY7AQ;bTJG^Ox?O|46DuCgo@oi(TFKa{Z%l>-5tt{zEfSqK@n3rsCgFd2LxTC|)>dIt(2pIf1aG~sk>Dnyfh(%sGc7hk}t=Cd`>udlYEKQJTkmc_F@ zH$}X`2#VzQ2 z+1Nd3(#GCm+XW|Se4MpSglr&ZAY^Je5ScZ7%=82!k_}6#yMAS13K$16SgdTIuC`3Y z2#DJBM?UcbQnPZ4Jwd!VK3-=R0o0i(V;(9)TbGs+W5)oh1gXig(xFd4z6 z0>@DJ)^<&URG3kW6KDXCE;YWI#Vs*YaTj;7aP(fZ)6b;F5OA@k|LxM6&s+)*T5A)> zt-=QfQX--x>lQ0m{V_)L(eJezYPNMzzmayZFrSeFn&>ps$S&22X4~lOsfW9+eLf=7 zJgDlzEqF?Ut_6Zd#8%e^2iq?!CYF3jMRYrE3n)~=WHf5gl2hVMjxBev9gJyNF^nqJ zSI>s6lS1E}cLx2eW`}p}+?2~51!Y6NKbqav8dB_SHP1QYTpGq=L~SrQB*AP0c#NI+ z6k!@=Sa+Gz0oR`M=_pZ~(RdCv;%u!q3}K2X<+D6H8MkHU!cxke>7w#AiEJI|>dp8F z)iWm(X%yuqr|+Q=@)hoBfgO#q|Oi0M6Q8!AlUW@s%p<8L(BSCT+!-KI=5JvJ4Z z304<(Q-!>rR$Yv%+s=^=?rN|{OC53%SRtn;3z1mbau@@M-6J;CXO0J+&LJFiV9rNe zvDHKX*P}Fys4P<3LwpxoOE0!7M=!o}SWcOeEoi*@L=jWMrd+|M>c-p~zs$F$Z1sV$ zIb@gUp^*_=P+ZEmz+{0FCW)}`yZm~)s}#V&>3$ymXqS)R-V=CT!Ud2f4i(c=W9#Iq zeq}>Y&a5uraA4hMm0}e~*O~yfySV42T6abj%&>~qlw?JC$2i7DeRE>zj5@jCYt(e0 zZpSJ4SRvh0x9Aq8v{4(@Kf9#c^k37AHK#d*2KKfq(O7JgP-Kx!laYy1Ld@u^^}~3s z+hk0IOvXk$T5u*k95DX$%0WH`aglObTM+w!7+9$BF(t|Ym&!cAK)OhZqwLwIHDDJl z%gtnxjw{un6=>lv*QK0AJOD{i9uqSv$BsjdIR@3RcNW3h2-dU9P6=u|0Labd<)~8AQS+Bc@4>yvMl6XKMdZE<~xr=!d*h=;^|q zkZyu;SR<1X`h~Gq9T~~uk0MOm!OCZ&@D(G>=~;mikxrhAXjqKvURyhA%C8$Jt`;~# z*d9lg?)w~09GzUWA;1JXItTRZBGx+hxj6v2Iz7=a+lVCGMxHjAGcKuQ3fnr7IDPrh zxVIW;OM7r~3vm&CIR9mkkryH@D@T#O8wI#8#=UsQD_$1?5w`Ft>Kj#QfT)UtNG!@$ z>@4;=H`tLC5qF z5YMsU>Nbanh#>!Rs+U}>g_b@(mJNb5Bq6zODb-wTHyhDr9r5BS7F2NdiHn*8T}}cL z6iHU6paLG4>M2RwFh&&BMVl6*5tVg7$x{MP+71QJm;M)@Pnz9>PTrJ24}%y zvP+N#9D)#y^vipAd3kj+HYa4oO@AZp3vQG~DtSl|_RBcb+5kmPq@#RatZPb_a|Ew? zM5Ba_rWT!|EwBr~vg|l3Ce+j_^b&G%C+7YJrp#je_4FRdVIVn=iLd5(jU?x^8&1>} zT%n8xlOplXWF=YU%#*BCR~>CLjFki+PqDIc`V6Z}lcZPpW30tEHlW&5A?HNMO!5x7 zJI}Gz#;{|n3LBF$jVe0;!A~8^5H3jt4ZA@AqrdzZ4K+7XtUWBQE^fCiiK9xLFygT&f* z>o6JGU(&3XHUxpK1DRn#FougbM`Xhq6`LsscI@IT8`OExQr^=@ zx`8`LIDi_xc-obrNmoo6qR}^+JckRZ_v{04WPdsPvK>C^iYU@67JI#|Uc32~3QZG# zPBfAXp-7m?j?Rb5jvF9I2Xpgy9C3FfmVq4zM-3Y($mTohP)cL1JlHj4|i=biq4=(?IjC z(PsUdW6Riv)1cK4y9wHj4**hu8VVelQ3VNbjcB2a1IX=S9u6x$?%-C%>oy!3Pkk*y zXBjC#+8ut5Y=c$Xpjy`E53u4;UrVwMnUDU)+)cPcolpaA9-!=*9k8*SHzv{Y+OC}1 zi)&fzBUCB_u^eHWQ%F9GjW%+CktMk~7MV}kX4OzU zus66&8vA+aoBW$*)rb-_OS?J;)9^^tSMkL)@+ksJ95{yPBF?^E zsTT8jr`s+bn?CzFyh$DuaGh!jz~=A`A39Nijh2Weaqt?%h*Y}>!Ok-~ItQVYb2-F~ z6v~`3tq;g{_^jKFLloDDjVR?V35W_vJTq?h&?qL~??tv3`#{7vyG=9*Z_U%(q%vVg zHRT>NZ)T7)JzS8aHz|b3)_Q|=o>|UB&u!OpAG&7c5xx0Q-T8G&&c~({b|c)MXKx)@ ztIk>UNrDL0I#)cU1|U7zSpm^$BZ-jm z4Be^@$C=-cY|Wm{h>qoz)3}ivZ#{HMtZsCOi>?c4m35Y$!cq^@Rc+T9=S7s2_Fb9k zdsg%OwbVQ#1-@`dDTX6c+C&2sG;Cb2hEdyM`mYtiRq%OR!@bJWPBJQ)Vc2lZ2Y*2lDh&;97GD=_%SIc98Ng z+v1l>;YvWt?q@G@kP^WLH4%%Z0J?J(gXMK%5V)T~WpIM+*vy#d0D5)|Q(l8A?zMdA zE^s*Y99D7d16#!4`Qt2PCL94sO;TcSp$r^f9n%4~?&;`~*#7P)#O~NMV@-XNjDCU; zOd)LsP$C*_*XB1Kx_}%#>7#7Y9o$Gko~I}KqgZ^(SGtY;b1~^490%_?f>kD~BQGE1 z%U4lq*jTt4W)oz_$~F>tAWc3lcoXjDDBG+qjPHhMbr`a@h^KEN-PJ)8QF58{g4r@QJx%uRc!K28hGku z1&t%ul(nC3_A9B|0O`09dq{y!CFbd~OG#JB}NxJ*72;Sp~gKoQGO-F9?`#72nhSjNu?%$JyKu#<1^%NJ~ zRKeGh>>viMPUFbU%?3Uw+?C(g*CU(0ir_wH_{qe_fx|U8SbX}(JE$gJ0+dAR$`#gW z_LN8TJa;*Q*Tev09k@}Z>ehP**N?bn{T>4tOUC~gLf2$Wk~Q+d?HnpD;fW`-GVM6J8h z-uIy^Mn+hmQ!c@LeHvF|?lqBQ30G{E)TJrp>Ryi81qWl@!#sSJwJBB(jJEt0%Af+O?oOP5T z3hLnB*d_7z67lIc+b=GAk)`D4J>tu2#AhxvHR~;fd^mAY3ShJEOg>*Ic|)@Se|{z( z3YLP?#b976U(EY`rC>1V4TK88U@kW`m7fikLjJ;?=WsxG*(uLzVeH!2ML~{T9jg zZ|9F^kCMLh$@X43nA}Vr;qzoCm7ZEng}q5%qPKFAscvts)RS}bwUt!jcs0I#lH6OY z>};lkt8??!)y-sZXD>aK+*_~g%q6FibL;i(&Dim>Z?Te#9~~AJl1n@BT4QHx#k*RK zq&8x)PI52pyRi|frs4ln5isf{I&SbR;{vLZw4}%#n@*0D7u}gt#77xx|#aH zVsRmHoS94X(%$`0w7znfYpmCDjkQiLp6TxR*PE-=a3C4Jew`Ddi|sMHxn|)Q6e;$a zH0eL|7s8$GJN5Pb_DO1XF|j$By0NxjPi^q2^;Le^{W!4|*bc$E9 zd)f8<&Fw3J^|4dIznAV4UmWZ&1-&iHIX|6(kj;Aw{@iTH?<-_|g<#HK2nF-A#ZUlz zKRp}D26A|l%gzL+12cu$Kw)~;8}t`SzFaOBn3*m4Uv>r_Y!~lYNKE;Qgu_&RDmUvd z27@7Qz@PKZ1asMZ$vagH`g~}~=gSt0q1mZiC?A4M`XIJ`2xcjp_5B?sjXByj-f53r zy9*a7iz3rSpLce;=q(jO`CN9YFq;kJ{iP!M8wyMT;Iy|;nwbqvm%Le@Hx!teDukd0 zLZOnEc+Khf!DO<{h&NnpF;t_}>4@4EJ*VKF3+sdn<8XR)3=UzowoB&zX`%~t@9F3p z!2y24R?wXtp!tPK2ARc`wMBrE_tlP_KHXe@&pnU)ATjm*|M;K&ACWucsKCF`+uK!{-@vZE4{z`@o!y!^2z&tap?=c_&YCs!-rDe zdHmJ)eEiow_=mFM-xFU9(Wl9+<33wiT7F6r+$$f5&jR6!zq>3`38?lk{Y7H&#EK^x zTTdnC6VY%g#-846Bom29?<5)xr|ac#FA)is6YJp{GslbJ^mKB1X7M1iGn1%n`786y zUbMWOSZeMhPWHU9a4*@L-;1RIA(DuPr_y-RTVC_7#1pX~bZ_A>UvKvoD*2VF&a0b@|FlXzOOfpPI|fMi&An?UPP6qED4 zUUUsTSofwP;Y2Sy7cNA~E18A0Ky1FemTo38?Y*VF1K(=!PI9k!BX?(GYHM%*&dfr& zceAlTeP8$PMIza8c#lo#>EZJT-p7BMz!R|pQIg^g3%BlTU zyp!}^S+DIZR`Te@mGVKcy3#Bj9)|Y1Gwr#}qi}tGvQVGPciQ2-aJ#*8Cvg}GbsF`_ z<^AJsZGJV?S_%e>?OZ%E6^O=eHrmsX{lmGQe{XrKK2t31Pw(ZYQw#obue@2iaV2s6 zW`nU_Y-P@c9id|L&3e5m_SueI3rwpght*;a53Y{kbaDmGWuw{1lUDFv@DFWL+=C!x zgWjoW->k3TpYnTQh;pHzcP3EE%}!4Ty;*OlG~>?)f&p&`M$GHY!>0MBg2mZ<&OZyY z69go>I2n*~+1yO1;49`(HWbLi+nfmnaxhkfyx%`HT?kD3{N7NZ==bFU0hq3!*FQa5 zDCWGgp_#ldR0Jfp(HQ-mv3!>BLNDG)bn8dcE3+G={q4$~{8A`>v^nEV^sYqxM=NWE z&c^a$V7;-{oU4YnGQB`>d3LrMELB^RbM;g?(OXWY8Yj{8UTb|l6*-z&>>X?u)4r`v zEf6j=X6^)%spvs`E)|b2uFhwUlgY(mH<@p5M<;Wc!@ZMfKX4O6G0b^U8m!YQ7P&^! z{}0k#@0Blj(HDpsKk=t;`|9uh`t{YvGw*+QX5r00z4GC!KYGs>?>_N^@A|i2`rNl4 zzyDYN&3oRJ{eyq~*57#Hl^1Q^_oY|<_TT@Z|MuG6+wW`rQFP-~?|4i2D|2tEz4ZB) ze`V9Z`PL^Ny#Jev|NNs*+=+eTw|?l$&wtItH=g|It1pW`_mMZmza9^7|K<}v{=v_0 zzUn{zd~PzCpZbNlw|;FSHGTcdTc7&eYqqZS{?UsLKJvu+gP;0WvG2a`AAb71E)*7eu@(!;;~vGAvFJ^$Wv`Y&EooWJ+wfBKnE zyz;Y8{N&Ge|JRRwC-Hj~|0j3Z+&?8(*1kIp9w$k^S}R|iI4ut?Bd6M;=a#3 zul0)m(N4eX#eer(-&}nC*Z%g`AA9VNAHOHG`uT7E$@YsqU;nc&O+TKv^~Mjq^UFW= zYk%~XD_{QQkALGUpZQAbeJ^?K^uKuD{Lg;zjZ<$9f9iGr;EDhI?aaMz`t#rV+;>0F zz5d7tUi6W-ednuR-P>sW>c-UWC;#%FeBgK9@t@vz_{;B4eDVi2-~0YQf7xGr=<4S~ z*M5EKk4tZgyyWLz^2zMmKePM??|kXo<;apPdi)12U3xc6s3U&yPJ7rlFV9=nPg@%NoJCHCYfHj%qy>Xwsq@_M^#kPt|(F(Rm5vXVZ5sQ`;*aBh&#RV-CL7svLx{CJX-elQs*~+I&`|%I6 zpS@@1cYfzL=ltgPI^Q$VO*6~C*_n&hj4zJref?ao`FGcE;IqGd_o=7Wpq1{Kap!26 zV`gT}lsM@-XminN#QmRsNPOYJv)2CPuOItu8b15&?hCJTlykT2eD!5VJbChT;Ks>C zt7o`%bBk$KHdmYd&5bbFyh2*))~kNquP!uMk!=de~E{C?y zUU2d1RkN#Bz5axUg#1V5FI;~QnWH~L=2!no^2;BC4qyA4cYiSb!RHU3on3#}47340 z0($M*)ykUHuRYA_dEO8@;f;qQC=`Pc2>f77Te(H1w=Xeb`|SMwVrLJVS-pMs$Qdw` zb*r||&dz-Iy2sD`%7(cem)`yOJ%4q_W#$)?Pki8Hdh->hUUT?cMmwLl=xX=ZpF3y! zkJ(r2dq4f2+pnKL`LTZ*ttKxz!G@}@fAAk4$nUx=c>2VJuf5}pn}2=xVNdP;motvJ zdEH$tdfffwriYKb=9Jyeo)&xM7uM~(Eb+j%a{u^+RjX%bX7|?+ zP(0Ak?97oz{`(i#uD6zq>&{<`pE0W6w`cE1nKSSFH|W}{uB+_YeeYMl)WL5YTz3Dl z`I}+pb)P(ydF*cf>%;T5zxCB{odr;&Pv;dU$A#{?AvxO9%$ZI$> zdi0V9zt#)Z-u8{lyUcOHr|GDpS5DXkn zKouO(Exi)eEh}!CRt3TnQPhl^I99=8HWr0V+eDEHhJpBmTNq{-Fl;8U3XWPIe$9%Y zt@Fz<(3;;ef2bnoH_m%wIeea{z;(xNgPUwRmuWmVK&qrtE+Z*CsZz>vUrrtLWW~)0z8DiAKu>ADP_in{1G$2n$p}g&orRQ<;HrY@V6a9tk=%$+sws;y zs03_FLD^Q|Mj0m4O3`f_sdC3|xK7!FeT#1Ol{7z6i&m0@X}Ts>?!DiviB1v(q+LE>mJDX1z6^hgVJJSG&yqy}L{iKZY?(R@W! zhpKF;yi!9X3GWDqCY!J-7z{KrfF6_)n3tkN%c#X#uuG{f8|qFbsi78QK(uPQKx?wf zT8`7#A>LC^UR9Vuz#*{A`J^k#wT1=peo=9A2HeWnUNaQO`W?Slx;37cE+{T4x~>Sk zypgACeo=K9gW)^IFi;G*ImpviPCzMGf-J_S-JUrj1p$hV=M9yhOQ(?Fg_?Y>`4q6?n*EwR8=vjqj+gnzVhotkO-?cEq;C)P@$U4OA%+nrYyV z9xO{Gz5#4i&NDD81N}-3ZB+IYC=2tXrixt|;xh&Cys=N^B$$>R;h;Xn*266D{jvW~ zVt^X8=vUfTGyC}}?Eu>kbd?8wrj|L?Et+sGDW%Pk>1ASq5+pUm8t6>WmSJ-2E2Qnv zC0Wgg7Ch*^3?CvjMMJr?%i9h$lvPo*Ae;g|n^zgJCi-Rw{2~T?h*xRg%aF+1K!>s` z>SYB^7E3h-*rzObR$g^LTv6Z@M`4uWxk(vrm6Zlx=b+Y5R%)6Bsbo>9?Xw*LqQ#=3 z<#iGI`5uAvR+r1u$RJt&Vu7iWO-x&>HR#WB^F=nh3Dvp$6fB z?PMf_8gz`J7$^#?DpX6#Gl{<7vwc-5qe77u4Ug|yQlMsGm*9h~8ci2<;E zMR#a^MGrMeR@9_Q(UhRlYz~5x9A%N5syZTLi7J#yDxQ_)X_{3jb(B{IGFX9^SB0oq zgdGda*K##dEoqcc)FwX8^B}GPd6qA@8DDCMqC|_dIdVA#(j2fRC7%ZQC=EQ%p;gNf zNeRmM8cH{Hq*YL8S{&;a%rg&sof5nO31Tk^at&%gq9ePmY1QO#14>W=R`U|j*v{0V3R{g#L8n@@;J_j9)ucf|nCS2% zue2Iw$sAg)N*XDbHB=H6Q$QER7EpBH+#ra)S zpm3MN`D`Pl0;6jn;X=7p8Z^VzS?AB&wn?j0Sj`?-j!&M<)l_x>4;#CDVuO zSR|fNP=SY0A#2!`bQw{^dTt@n={84Ij!Wq!H8McV4sSH_Vgl(Pet60$^V_Dy-m#O5 z14_eIr`c+Db@!nAywfJ*Oe|sR2og0hIA+7Bp`%ef4)&9kiWLVtQ8cDwQN)U26-Y-> z6w!4%iYJVC3`Z<84p)rF4qFklb=}e`)t-L#j2=2Ej|`|}&8yn3fPmM|R)Mz=b4zuxS1_l>Drwjb03P=d|PZyNtd ziedS=Bqho$4PK;7EmIeLj%CSUlmwJml?Yi9yke2y%JN7fNj$1KXxgb4B5BghD`B!) zX0uI=9W@||2)U4MPzsPhvJ3$$fD-0FdBm_3=2~g1Z+hJ@T{C&EL519i8gMBv2K+BB z)3}gt09u2C7yZ1fV8sSe}j1od7)m4Qt?u8o`EiS*Y4( zqEs!HY1ed`3vyF~6UA(@jb#~nQ1-h}A<2WsjnYa#)z(Kkfi3zFizE_!n(dEL7||ez zULnq;t%1~)991Dl?UJm;N2x}-)(O?N5&`KbXm}FaH5E!qcxiso&&0Zh9bJ$JTXyqF z_LN0`+0-aL^%CZ{>PU=M;rF<@kBz=A~tUR}xwSP3qx-4dA&(T*`9vShW|t~rf- z0T>uDS~0jq|KKJ^*mMXDxem8C9Yml4JDRNxoKDD z!nD|@smd^hNREgmqm@*)V`r=s(xn$do8di+-ns#?pAH;8pNWP>W_7u`=7K z=DiUqx;e+QJb_aR88Oj}ke!G^#|d5VO}Dq8ER;Fa$ajH+JPkKtB9t2hw`{B;N0Bm- zB7}5=BcZXATdAs0BuTNFi3(|%>f$+xwko)sXnR(F(eJASFeRe0JX+!?CPfsh2jQOZ z{+D+$HekLdn-R|r#>MKyX#wvsS$Ck*2fltW9X z6cqAN5KM3hN5ufHR#fmJ+r;lM;7L&4w%m^YzHi;W)N}mrNIgq+ztWuW>}w7LCk{3L z_!G%n=MCL;n*m52TeoclDPx@f?9NjC6@n|efgXSqHlH8os(mV$o!PsLOl!aR#WS%F z?SAyql4Vm1OLVm4HDg*6tD-KWmE4->HKVGUp^C~N zSytWgKBlL7U{9Q8hiZxq70AJp9^W^7(KUP_FaY|bp<**%C!!nyO1))x>?KgAdbv30 z8mLeQoOP{K3qc%)SVn`v?qeBUV$)jRVw5Vt)oK>}B5syTk^nuz0Y5;W9M{Q8WSj-q zo0V3FMxBIAAME$jP(A5GbZI}63gmILC+5CtSAU<$m^L@p(cVEWG#^%_8M4cIgIt7jlZh&T`KY=J&SyaF_M-3oUWA{gCqBG?_yJ|`p}E!=Ssb*QonWrg z^OsBNxqdUS=ib%P!@{Jn-#9l7ajbRn+}62{9rQZ>T+r#+bKBbogm=FC zl|2z z!KjTIiScoQkw9$>!=WgEd*Ya(n-vJQF$76ePy_)b0iJ*j+{D3A5dsPb6r2}TbRGG< zh{kW)=aDZKgTDvSc;lg;m%S1hp;GsCcWInS#zpcgpqFv7c_r31mh*p^u?W+&TY+uO zZ+oS!{w<`1J7?kW8)2u*#v~)lG*I} z!1iIV>j3-h9*9%(MR0r%zQ3ZI9dW`X)$|SsVJ~jNXCGWU_MMLemu@`rx}BfB_L0@! z-gf&XH%2F(H7-GKT;>2J=#)6=J7|;MefYyiZ8~FVmz90+#;d=Ts($IccV4vTb?+xm z-*(cRxnak2;0cq9{USi!59|&IwVTJEj)7Hx$xgYa?KBP@7zAn)tl;GbfpCm&E8Y|w z6p2E7m?4xH7mH_FsupK!CA3C2|JUgDGLj8(sVdkSE9idcHaw;Qe0-!J0Ns{3kqT2| zWOo_eu1(Nw+|lHNBimzKd(kJ^bP!sDeyAC*%^BAJekx}%_)Z*odDtd2~h-;CD#{T9@S$7+6v z`ba}`@iwCf8m}ip01S+>ZPm>{=|x|U@is??VN#eP+iVD+!T*A4H(9mY$TnI?9&2+A zyI`~>xYTgk@pKgca|fc@<)lrK&>&ZkpjxREQ6kxvZ^U7N&V?&_5+M;BM}%b4q$&#* z(4^Wp0K(l5EbP=NuA3gl5fZ|BEu4!hj9itS4ij+Dg6L|Qnh931P;J?DzG?USc%v8Z zriuf?qa&7=GJ`HaySrVj&GvCT@V!XR8TMQ{FL$ySW_JySgriAn(eK7cr=JSZMy}+= z?TQm?m@zqxR>Nx1EiC*_w0qucdR~1T6WV@rehUWtctYvd?QniebmM$Y@78Q<$#44R z7)~31>~| z@@f)?s|)P~Rm*cxSWeOvqRD03MT8zi(t^n1D~QtMb?ghFFL8>vFNy^n+n&a)X?w8v zRhV|HFiy?}XCga-uAlxoYDVUUmI5pPb!w?PLG&3HU?tSpB-+-a4Y5IOghS9@6+Ozjfo;Z@lZW^+*2s zzn!`CmGh2!`)i-d9oxCRd)mfNzGclh@44^lYkyk5{e>H>cgW9t{^G{&AM!uF^TIvx zzgYb0KWw<=i{3>?KUw?UIp6BoYu|eO`s4RJbji)D;y-!#+RqQwKY!x&Pn~}HL)&NZ z{C&?pUOsy6sl7jrZD-&2(HlPd$k~7Y;9uT!(fw=wWBpa-^|@tdivgP zU%utB4?UMTi@osi2hJe=DK*-8)$Q8h-m~rN|Mi|XvA3))UU|(6$N%PrjrZYOE{7kg pe4ISyoMUd$FZ_P)d!K#dg;Twc^Tib)M|=|?A2);IlLa6B{wKKC*?0f| literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_53-57.committed b/tests/testdata/eol_service/ledger/ledger_53-57.committed new file mode 100644 index 0000000000000000000000000000000000000000..7b627163dde289a28f0e2abdef9f4af723e1aec8 GIT binary patch literal 5406 zcmeHLdzci}mG7Q`!Q~}J#W5m*iQuD|>3VimksYO~ySn=Qntsf*qeH#>UHz`<4jLI0 z3C4&lyDUKv4WJ7wVsJo+A|j7P*nqh57|}!`#4Jc41`&kG_6*(1w_nKae!JQ4OTOHH z&bjBFbL!r6?mhQ+YxvEY!KT$5(GDlly;(DCsWRyTmx%IH zd0uLXrt)Go#U@(B6rU}n^PSut0$uF>l3G}EdAmS+)GFo+O<8$>`7 z2%cj>0cQ~&mP7)E2n1wB(Etb%h7$$}U?G4&Fi;=>0w)Lw*i!Y=q^EUy#T4-sZ6!mH zPNzB|1kAeHgs2BZ+nwEHCYSK>IF$4=vVtdao`i`(jnNKO?VuEs-j$L9XHlJXEan51_ql3H7W_N39Dn*B*jP5pRTTF51u%_?4HNJfqyjljW53) zSmtO9&%Eh*{`h@!YKBv)%vA}Nm10Xw&9__;b>*URDw>kJ;$o>Qn#&c$Y|XYtYw5kC zD!>SghhPL~<_rkZjDUopnUw^t*&y-~$fE!v8rWN^x{-W7otvuHClsDdM3h{9Dgpy2 zSaUtO%KW#ccg=nGKbhFNx+|Xqdg|5yJ=z7!>g%+1bvKPa?sfmLY4M0dl*9ZS<*t9& zxZ!DFRKwrgGutxz{3xw<#83?|1{@8HY^V(SWryk!pX2SfH2TvVZxNjBzlJV0=?>4ur zy|;JNxbl)um;NMiY58M4C#=`87oYjg&wko;_sLhw_0-bwA|T)N%k{_HyH@4TPSS0; zWBSfZ2Z#PYsxna^`@u}gjw6vSBeO+sG|FCyRmi}PflH2w?_k%d_ zE?C;#H?(oa{-OU0ePC zzV?NCc748-ynKLv;ZEwtGj|<*@Wa^fcd1z;`i|VX_Gb8{*(-N{aY5U7+b>&&?|Xa9 z=g_Vhy0@6V`48^ATj=eK9@&Z|7B&Cs`rg*P@uO+$y4Ay%tiq4|i(~!jx_Yffdv$~= zfv-eJtGWLAk6vmRA#`z@=QR-1%h3;xUVMs~vH#zIO)qbjcJ1wZ?UgL?M)#@@8{IoW z<{OLdWlkP+Z7t31`PSKK>~HUSeD}6(o4a%&djHRVH}U3C+it&TeR0@leH|K2dG!T!i-SYmuEA{f24o}NJSza4^Y|@GL z8~mT#arE-?FVA_=W-gu2m z!ayOpZ&0IBO=`mG*fq(EFKAi<<_CvP`0)E3W0&9Y*7;Jytr2x_uVF5%;uY>HZ>14ie{{?2}auIx00S@AT2CN$oHNy(B_Dg5@OVicx>H_oMk+1ZpCg_%60@*LnS?%rxlWji;SD56Is|7F9iw~ebk|I z5$&BR*l*%+qXjAF%6ZwXI0JT7F7sSbL88TC8i-+rNJ@oMP|TMgGEBf5cCok$I{jsP zCYZ1Rmd+GwaEVT{sv5nO75D%@MOszTL<*Gb3R=7fZI-Riz*k9_ zN+~=o>4ZuogE7)h$#NzVkGVaSD3Yp6Ax%_*jL~E^N;`9w5(ZHKT1XR4EWr3>e>}_N zoq*1l2-_ljWfeL(Ek<36SS%8;0+us{y2+$jFC z`?Gcw6>}UzfrydrOyvwzyl7J4n4>+3iBjAU;|+cbEUR)T;n7)JJ5z&Ok9L{_% z0m2~x24O&AS&l#f43Rk8U_b#DAS4Xt5flLk02Bd$hdBTOQ4!}k7_Y!CwBjkNCAubsI6Vrw>`7`GRdpt62e)U~z$HSw;8sbf4 z*b1VzLGSgMe{`YkVb?#SPhJYV{{ZsDgJ*x$&_`g(??A_C!_i~MH#KjJ58bhDMe{4K z_W$gdW7D{E_qQ$AgvYhM@beE=S~^cz8t;4a*ZWKE(xZpJHT@&_^d6sM{j0X7#}EGF zLbSKvHhS6Y1AU90Jux4(?ye85>%Dw?|LapeAf~PXk4t|?jeTtFZg%kz$KmI1SzKFI z?Pap{&;wfI**5z3vxdb-tbV2+uPO|b@2_4aHDS&8TJ$&k+~xL>o9?}HXXCclH(pG} zp32c@9=bTJe|$X%9uDoU+5Kr=CL96cfJ8v7P}#?4?`CYcrvp#Q zj)cc6`NTHJmLln_j*q3|A!E+xLS253$r?hWk}jHUFBZCKHd)M>+U-7%$fyZpAr%9F zOt%HKQ3cjrV1oi#P#gwl#wl`$NOxcLGSPl-5GkxX{$lvl%j^#9Ul(sdvWLbH)g%lg z`2HG}s0nKcx)x2Ej%+`9SK!cK^3nLij6| z_$!w9%fS+cKT9l8dDycRtbrpqfD0G_SN;)*ga|_wfC7VjOeY-HckjgXmy<8- zey6W=eD3xUGoMX|55Llr-9mry*2j;ObJs7Vm2w={Jt;?J+Vk zWhfl99kTGFiw?t=&(*G2T%EIIPjSl)x98+-=e#!ctyx#F4?X$p=|jZ@gG!?<)Kqb? zD3%u~MrV;`w-)UaO<1a;tGY5#m3tGaWwm8l22bIBiMsN}rAs7J`3K)VcL_iW- z$d@HyAtsAuP(%Sl#`xm~I+a$dVJ-H0Ylg_zDu&fsjKBa24o!b8(21J6|ijd%Ihaj8l5!hGjm@|#xEPYdd4<7>c4@GPyScmT~`1zr~T9Emjo|) zbLQyijHx5QQQ#54;nOB(r%XP4s?@I6d%)b|WY5qm4&%Sf!@y<1` zTzh-%of|J+wkvcP|Ms`fc=Z0U(_X&EnM_|hR|b^NJiqq2_=a0rub=FB;H2}`zq57f z?>7DXykpl--}0+h-gxBbldqO9pMJ-p0)ua!SmpWmO;b$vnKjqUe&mL)RDmbJ-p1`y zXDxl~3T@$%MVGx7e(EQtwQ1FwfPeG$X(#@6#)?1PzW~9jCto>d#MgXt!$X<>Uj5zE zH@tBz_3jq&yQk3q{MxBI&fTd^e}ewvjO|ZM3EyQ#G8C`qpq{*WrqX*BB zJMr!ejgB07|+41(}fu)Z<3Eca``^p%qXMuR1&M^48cxy)Q32;q@i_?@zsSMvaBSXd)Hfbne^y~j``OQrEO|5X*uN~lD>03{c=ZFnI{Rj?a#S&~==;GR z9sAtLy9!5VetXi6cW?ajn-hCB@3}~N^*>H~VeZ6^HQsxlc>L(e%Qj#A@YdY3y(8wG z_s;FVZCrEn@~^(V?&RI-tJiP+p?AxVmtL@~b)M3tl#Zyd*u4>tY}$=Uz9|`w%oy^_Fbvd~K1ei)OFUlBJ27EGe=%VacK?w+^{!4;vaCl!JdBIsbxT z!=gduAaMAQuUT!OuD1H$wJN+RAM$%k2G(Q{E(7H5_HmQx^9e^$`CVYL#k_{?@atx3gZ zIzn^>KP`!L4iW${+$=VOVY6v?WWVDGbjTFxPKyX6K}F1UN`+t|lP1fKVWwFkoG4mM zJ*3waG+k=s$xtDe06ZlRP=PwK%9pelZh%o$kRio`G!-w@s(I*Swh)sOw8!d&8dkC4 zX;#c6(&&`(Xpf|PIe)F|&l#E)3Oha`79AT5Vq|pW-=_8J$6Pa~xcR=~D-WGH>2zk@ z{hQ7lbm+U&IeUA!E0i6gDJ7Jm5_m66WibxJ!;M@C4^;|q#c$X?r5K7BrI4cmEM;@H zUt_bb1v(UvSlX z2h!0>uO_Q;BoXcg6f=-W$BjtDa3Tn5=X+kh#Tpb@l>>2>t(kDD+RL_sUN!28k%e&` z&M*Su3nGr?w3N8PWh1uYh(gCe)J~@XXgF5UZCHo2RE=x~vdL16Cv1?*IFV+q76yXj zI*-L<&Trekq&tEXkfEuNO)->2E3sTKiLibpnKeQ=ON(d?D2_HcO-=~SG+5B8O+2XB z<9e_^_7tGev|{@WDqHl)EDdy{i8N3t6uen)q@fEqnDj^Oee;w8QtpD;v>~!(kL1o| zPNO2UqBJX2EuL^^k+Ne3#n3fMV<}T*IV;%1AsRs24U)sNfsB%=nt>Jvcv7`exFWiv z@F)#gjTv&MLuzf@V)9+8?)OS{Mr>J>Mp;&=8S0Q^ORsyQ)n2>ik7vvXiprJ{ph3jP zj_VdiS38UiYtcd-m&;X56R}JXR%|6-OL#(Ojq7_|kBu=O;*Mhr#zdp;0+|b&P2Ct< zjFS5$9aXl+79$JBD!f&3aZWdMQFi^if9Uu)kNZvCkoH>QQ2@js0+L`52Fg4ykSKs7 zWr4sj6yO1}jKd;=A^-`1G60CM06-ur6QTeUfK*0ZOQvD;_Z%jp5D8-h1`{9;@FXg@ zr$LzjQ9yuE2`%H8z(c5r;Q~sK0*2$DAj@S!L`3)WD8QI&X)pm8;8s<|waEpNK8~g3 zZi`bA9a71OB2^B@JOPr@Et!I1m5|in_1dt*zH;*5l?3P;zh2F}^U&HJ@Z1Fb*t{p5 z6RA6PYM;Jq?Zrnv9l!Wn8#ZpccI_^+x^(jGQ)Zn1g(p9C*1~Y(SO4`!=9=qoS#a&) zOOHGK=%#VOyvuRr#&wV)zm1*0`?k~m;)gYqx#5}f_jrE#!iA2qbp2NX`mS@>&9N26 zwuiTrb2m*qZEg4&G<~@~H9WuY{qt7O*)`8|*|)yB^P~;y9!Xqw)z_VAhu!fNc~)ro zMZY}Zyx+jDK9Y*Ay(c_&>DF(41HJRT@DbNvvSs^K*X>$~hBi*luetNxIq&^=!87FI zRp5)|uhGX|eC$U4>ZhZ>{LXP#4~IQEa`fT-`s2&U-RB(s^4jYg>#DQU!CTK7bm&9- zD>L@JOc@FXZHFv`x2g-5C+2tOP20F~_QFe!;J)Sbftbyyy19giKX++WW@tv&~3pdkk(C8||d)6pba0E;=RKnHEu z6r2yBn?vQfRMJIGRvEHJYrq;gK!V5~?@+Se26~)BL0l$j#~IFvWhDN9EW1}WuE#PZ zo}-IhKSQPah)q)*=L2F)N~PnaLJAh(u*n6BHcJ(Q0}k|2$)Z1kuq@>ZG2?nsq1dn= zEObI?PA%kwEEZ&H!5Yum!Bof3mU|eKuBKogQf4Bi9F`c!@_XFP80%=Au0qq}x>fqZQ+!evYi&=~Q{-SFZn+lX ze$W6ZJEKwDUfxN$fRS-oC>iu}G|=a1iBXbynocQUBpJ-G78y(SHdjKH7Jd)airfzp<%IUL!B&DBZKvX&7cyiwX+cy zGs&uL<7prY7vOX&5l7TVM~)wrR?GpLF*6`hHYJF3yP_V-dHS4VC2COWcdsfO;d73ea6EE=w$Hz68r-Q@yTWVPHQoaI`I zH)MTmKWhGM4)(*nI0VZj0|77c2rpqW1Q3`ElqFIWcu*pEM1;#S2}7iNxR+(j~b zu`<9z0EytBL;@s1l4W3MM<780D3cH`@nzSbgo~n7cKaZ(=mrmH8Fjf10YzDap)!iP zX%A9DQ9%GfkwD8NEDh~^+<*fcjO!qH+8+j0RI6o}s+(JxI7Iw4P~Kl03c1z|dYnI| zyF`0nWnpZ=*nS6iUt~A7`24X0AC>qP?=G~{jq#?U`NMWQVhVJ)M_V8550_!0BA-&a36X8a&c3sd*mJb2*yEc7Si<2s%@eL~uW)^NU6 zP)rsky=>FAdzKoFk%3|&%|cg0v6{)4XgE7mG{rwbZrM7Bd6UzO>F^_rw$oYctr;{>b; zS;5IQXpzplL5)Kdk+>5N1m&EN2t;xv%G>S5d|9^8iK%VUn@bsvmo>}&bOB|vFb=XZ z6SgHxppk@{=fO^zcWqz<9#+D2H~7dmR0~uXj%_%NMk229XhN2Vf+W{-0b7#jTG&ib zfvl|hWSsY+H;q+**Uhc_U#3%Z}uEUC)P<9ET zic^M%s&s|gxtuNaK~GO zP0sKMpj@qJfLXJ(P=D`SuwcwK`9`Ck+H+3_R_py4K6mQPxMWH~AtVbn%?F}{85C+( zl6wEcChc4S(S>LxDkcr7ruUq#SufC%RP0fV)iIJ5D~e_ScHjd?h2iB%HTZ?ml^2}Y zc<}N`r*3|%v;FJ=xc38t{g)?0;h^o1#k$|c&VK2-lOGP*w@9zA!`aOHpmsiS} z*`FWm_&%CU471%s;h^o1#mT#Wf8z%6l1)3VJM0hCk!LP=VUBkq%AWhBo7#6|ezMD% zK2-eMX1go>e|4X~KGX0?w)>N8_y5;yH|EB0gZ%eDQnniga8bZSSmFgYH^3uh5bEax zK%U0|l7w)PAOI1>1p-E3K_qb*kO2vFuM~Lzmmm~5DBBGm$aZIa921md1tyy|)KnFT ziF`qG)DG*7^qN{a#`58I8Aeei1%@guGhAl_nXZ8l6>l@B5GsnPF)GxhVgX03dP#-x zF-kOCF-V^Pv|=Hvk{~Kf(QMVC6{OmT!)S{OxyiOJALH5cJ6mMI0+Sia@TvSVK`Fmx&`>BPTi{Ur9 TbhBXQ$O!w{k&&;>85#K>6gxNy literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/ledger/ledger_64 b/tests/testdata/eol_service/ledger/ledger_64 new file mode 100644 index 0000000000000000000000000000000000000000..c3d2dd81407e0eddb9102666420c6c0eb06e61ce GIT binary patch literal 3219 zcmbtVX>=4-7Vb_0Mld0QB+Ot~0TACg@rRn* znF5lwDrp-ETx8f)P+Z`UBQW5^NDxIJ#0jDhNIPK)L}8SYIi*e~2s$WK2BR45Ani0D zco6JxfHIVHLI?$u1+xpHbSUiR*n(mddNQ?~ShR8YpfB%k$(gk-hgtDF+^|jw9m10V zgUT(s6Co&<0+ZkFX_m*n+|*m9vSm!X-g0}Ywe+*i?Mwih-E)2IDgXET*f&y%6Oztd zm2)#J8L1D^;Z$_W-b*sq;9iB%P(ab4h#*=fiCXWg*Dw%{aVS_6Fje&!hz>u&>D?NC* z(W>H=e6_N|Vo@q_m(l`MGv$7{x2Bws8VOuO#DkhLBP_=mm4+y*CkPyO<5C@7%MviEKlfz z6}}QcTg#HDi8iWvUSSbVU}sEG3>4p$OB+@r!nZj~{tjbl;*KBWBZ-Gv(fp zCSn8%t-hKP7bTpckO|Ok6Gum*a!)-LR#wJvzgpt(E2&5r_u%1hO;8t=%Ok8`V(`Z6 zeQI-gNRJ?N*sdW!SfQ$A!zkg6DtX9bsPbd9(~Ej2v|I}@JX7f_E!54cW$%=p%A~wY zISS8{;{1Mhq%KScSx$JP@XtupP4flC!hEn5(2hEm6B>z7-@yMlgf&^tafgXS01%bR zqz(v#04GV>vI9~Op=FdEk^v4UA~1zGZt%^a z5J{6Tg-E3^0!m?w2IZjCPErmjfH+AMqc94z%V`*d?I>k;kSHmT*d21%VW(tvNG>p? z1V9AW%?O-EjZI-BKr&z?0wJTzY_1Bj^>$1FVtzF%2kP~Fkc~<_9&J!YRFfeCrmTt> z{82}BM9oP9lvi5?D`FO13^$pXdVOU?XDm}FB<>o3g)OL6X^vx9xDr;`K^aXmtgptO zU|iDrvMQTU{xV&_RzfqVgwxv#`FZn%{UxvC=N39{sO^aE*2N+r`d7F7oK#~Q;!8`? z&foCj>DFs_`{^r7*jV=5mx?EFX>sMP4N#a`x@=o`a^L>kX#U@?KA73ebezoqCSE%g zj1Lt4xM-UvHtu$I=Erl7#uoHKADy347G$gK(aVdUS~szWIy*f)^3z*0V;yb1qjKLe zoW0STzb7ki^qCR&=kATsJK9X&tiAl&q2Jul_Po4hhUt37u4fM}o4)1nZksc7uDj>s zk=5H8XTAU*TO|nB?U{4wyi{V4wPiCZ#r^()w=Voi0*^*%I!OES0)F9qfyjL( z_XBdxF+<;myfsN3L}Jm}I~{&6wc_TV&mEn8U+2ZjC%$y8Ad&%t9}~%cKt>8oCEv@Q zxU{z0_ETMtBoAvoDtSvXW!hs8M#ndm5=^qOULWn;cz`2O68_(>VBst0Vhg1K~gF!qG$>&f@v5jk|CI^2qIBAX{YQ?M2_Vp zV_lI*AY3ew_&AF6xwvqo7={1@{7*_uj!{ww28!$`3>U$m94#W94to(w3rkT50MjTr zElK&Sj&jex?|*ZM?a5jO8q>A_jp9G8Pfru4r9C7cwL8Nyzv<fLSz__XVIm+0%myA9o^H!Xe_)uS-tUhC6Y{d85P*Wx!R`k^HA7QrYI>K9 zMsdL}X~n}t>5bx1B4H;Z(i+8L(S=u^m~-Lb$<}uK3f=sCYe(Oc-5>TuQJwX8qW=sK zeDqL*RJ8iOt9#C6ZXei``$ccdjn@>n_Ou?j)JcN#x|$X>#hhEm?s~$N`~I5K?oYTg z?VE$2EgzXJQ%-8^bH7)U_imNDadbZRepb!fjf=OYrSG}?Q=efL^Fg=rtDSlJWt+F| zwa&f%sdvOvuY#e!nG%slEmvP^u(q5iWoC|kcIlNdt&U>#y26#U@Xa>Ontcb3EcnA4 z@vqx$ul6llI?4UR+vnzym#1C~joqZ}>WaUYGs~J&{oI8*Ps_=QjhziY_L{oB174TS zV6WB9y7cgL$;DHnl~?}RIbZ78Ieg2>11_8#C^|nb*{L)vdj4Lm38|AD>o9mu^mf4 zJ*`Dv52&)*>sHQq_s#zF!{@I9tJ1|wO%s;1UY)&q;Kss%!*5D!Z9C#Gjp-P5vTsTL I|MI&21-pv#5C8xG literal 0 HcmV?d00001 diff --git a/tests/testdata/eol_service/snapshots/snapshot_63_64.committed b/tests/testdata/eol_service/snapshots/snapshot_63_64.committed new file mode 100644 index 0000000000000000000000000000000000000000..5785534cda39156ad6bd574712b1ec13ea104565 GIT binary patch literal 78794 zcmeFa3AijvS`c_&H{IfCn?(dsq^);&-F+{0->Iz3%33bG?#|lxeShcmd0ClRRaupl zSxeS>`;OYmGJc~hA8x}m0z$(e11O?^jxvk`gY-YRJed)=V#yrrr#BmVf~kAI6lBK~;p-gjR-ckkZ4=3n*wTKrqS{g?hs^XKpV9yZ4;qA%5pEN_^vA`suI!74E}7{mWmbZa(dc|MH`EXm8raf9D&S zh45ehh3{NE|9_h0P($d?{Ct7@yk~xXp7DFS>?yMFND%rL18sF7XzD_j^`$Or>V;9{ zd%uo+`BdY#`SI6{=VrNcLE#q*4=13?p2wG}$AiJmmx8X%@2_}87I|HI=;5bRW&41X z@Obp0ytpu5dS3FFU%v%Hl$n^!|l5ANYp)MDfl00uu=bzW^Xhswi(>j9Pem<`OiiN^4J5 z8YDK;`vjrO^Y`Ds|ML4E`QG~luXrCkdhp)+J|Fx>Kfmw+{CfHQ_}l$U_(Q*p$6s*& z0=>RGM^GN%K49v226JsWT|i5K_{D_Rh2j0VL<-1#&;94gdk4AZ+8Se3-%h2(d=8|2 z^@Yb!`H4qfcvMaI@Pvk*Mm2x&DuD#F@<4y|<=hwyyOwv^^};gI>bKj#M%N!6_i5%T5<^xZvlfQT?gy+J&(1P0OJ z`$JD(mK4$GHzAcaMXbduKrB?`0!D1nlJ(*J%X1*CO*6BcMKnZ64BPY4OE9EMMe%&# z1BgmX7-D#`R7I&TtCHCqVmv}Eg9|bJfEl4xv+)JUjdY)S=y}oapfPHRlmf7q=lp!m zZvi!v+3|>S=;4LDz@G%K2UZEh@#Y9yql-!L#4}kgbWcz6@Tvy`HU`(@!RS5lNXkO; z>|r3Qx-^jH`#2Prlb$pOBr@dvGooQYKZ%IeHz@r)kOa}j>g&xtg}RuUFFZycPdvS) zj<}rsGyy-cnfw4_CDi4_WUPsVdcAP-79JDh&4sKA%2JdT`}DX4s;cAxj9z$bfI5v; zR0EzfT6ySEwDm*JP#z4gV+OB0h;W!`-}NqolwXXhJOv#Ga|u-M9C!Vh_B2 z3LH%VLIVah<;ik_22NIy41rcu^@Yt-ggk)M@%GeT%Ma_k5w<&@H#jMp2bjGDk4&T`cU!= zrOk_yDrh29Kx4>zYH5yUgC0>HNDKXf)PZUY?dffG=#rwTgNL4lhHMh!LkaAbC)WgFnR+x8x@{?+ot-`D@s)Mwk7*G`}B4#`zkO$VpvE zRj9rK=cPc*LJYWDHxY&|cwk(Plmd33FgGB_dAxf0@_f&cHth*;>3oabmMc=?3i`CT zJb&e)ugw!YID7Z69(u+PJTJY+^bt_YbAC;b%BXz}8oQ|(8HypGK>zz=;*JotwFd}T z^}04V;BuIh@sNH1Zz2qO%YI#}>2qk}iD$?o{_vod=is)2V`eY~c1O^Nn9NFGIC_HN zWEYwm(4!_28V!XUwJX)rwMBp17b9|Jq zaSrxWv+J>CxNlHtT5lu?`kqSLb9XNgLp-#7#fP3(UwBL$KK#m{Dp1*TpXjBR<%gcp zdi)B~d_)r=5a11i0tL^-_9g9>p!mgD7Ev>{nsvAH@(YjUC$FGV);R9f*SEK+uWygk z->|}oZW_%)A$N7dKR_M80Q?WJE3{9g1J#`r8$I;&f&o#QW1<zQOG#?=$}=_O2O&plqdG-==Y z1SXG{DlVKT#;*f7H+bCWM4P1|F=?(xLE=Hc0GH!L+ruC9B^DL8-o&KYgKGF~PC3B- zG*TDxqKW4Ghice7y6=jZ{kNFYp7kJRmqibP=MkV|a$gp}tM%OP;T}E!UF0bz3gD$6 zh4dZsU5|TCu0BH!yPdoGA#%hC%`>DyfSq! zu^*^3*1Ke)e0h#2Gf=#yB(PLqSNOrM6M(QE9X(N{rsB2)f?;~&z zY%DM@Iu*BjAOToSny+9kj(FW*3CwrkdzPiStYQCC7EvhZAz>lwO`PAFE;JchutG6S z?aPA`=FM$6H_L0f!H9L|m2AzulaTh6hmgkq&Ehpt{xq+y`&7n6X8p$LrjEUD*6?c6rkDF6qEBx1gZgjm>`SUkQ9aq z&=s$s$)9+Ig!4!(LM9ht5cI+`pey4ku5JvZ4Fxy)-uIIFr!X_RfQU^Bo9Gi}KtqnF zNtPevdDpv+7Q_>~oBM_8A9ec)aF6FzlPf)hj^I>XN9~E{KI$pxGGiJ>NczZQ5C=^( zJqDdgB;;0vH^J^JMT8p}zqtgSGmPDVK`>-NmGreXAA^snO2CCuk^FXuos-~L+>X-q zz3)XGxg}v;Oljbk0VPrA!EprSt~uXE83^Veuh_w)Abm9=y{Nt|xTSFqM5<9UECM(h|_{X;6< z0qp^Tf$+Pg#*3PGLWiSjDwN2GxGnq@Xo!DcF|ru$x=>x3_8Tb$vnD2psb&`PK!thh z)m(yTHUwD@2Zi^Q^Q#DD5MzRuH6=onHhmKVFpwPsHatY!cT4i#usLXOKRGrB4c!|C z=aXS@VDUzSB|GuT8iFHsCSer3ZfLOFff6_+%7$igs|e2lKhX2EHVIgk=*nyM2jPY0 z<3UAfwE|IM2+Gpni;&Q59boZO%)Nv4ZwGV znX|OMBHvvTnm$@3g^3IWyh{7fXTye1$FY92+lG@zlgBk?4CE&c{$bq+MHyyrxa(4x=v zfG82yThIuY9{d4I%nr!MlShYWyEd0MTq|`b*bU2L{4ZFa~i8_$*5IQ5a)~*B>Q@Ns@L-HI*HebS$ z73>zlQnJ7r8^wo_W_1kXF?k^pU{*ldv~DWXL*fO+Q44s<6D^fNyJU26!-(DOfyIi& zfdZDeBZ5M3JzV3OG2e%Y`(W5*L`+ozm(A2fwu=Cd`p8ZImHRwwL&4Q-3Ee~7?OwRi zw!$$t*Ku^r4H1#i;-sUc0N0ak;6A{t^TrA$$<&@Oqq+dP_d#q#}7Hwjyu z921)ofEV&0FJPf)0Goo`4nmcT;C&b1UVp@5=H%hVjc+M-hUYt!v90-8sIUY zZd!W>VaW@+?!-C=ht~4rC;+>^T%c{5 zu&M|P`?4Twu&}z)pyUL0Rp5mcWE$}pBK&MnC)Z0HBSY$v=?vNV=i(N!ta}$e*;`j4 zl!nj?;uJLHG6Uj-ynC!gnuk!f&ZF{Vp%yj~q92ZTZWu5`oC0*kuz;knHGwSZd+cR? zyk`O2y6EP=W0P4@8iWx@x;K4A_`${tYp((c&5@@>)T}mbOwB&; z^FLB#xRgyxv}FSAmw-6~Ui5Vl=C~}I7`p)hlUsW)MA+q}OO{3=7(4o5<7m5IsF1P6 z^0D*%O>{esg&hYlN#;PoQf%a%D^hTTOb*quhqldWWDU^Za2$kw$@3U($u&`Myt**H zBOw9Jk6U#-I#S7;jDjF>cn{g$Qs7PWLP()SwSDRwqYZ+0DCX?wh=VYDJdceVK(0aj z7xtFmZB>U5MEhLlu=Dr=#dC2%EDB&1TugMpO4yWj$|z!Xce4kbV0-LO?J?qou_c%e zDmmLb$hc%uV0H*b81?*E^ua9Vglg&EtNjYM zaDw-Z;2@e5WkhK*1rJ5lZ3ZwLZUWn7DlyeKupDA>00DDKjpqbyO~!mUgoCkTd(n$)17JogVHB_J z?l4*Sq?&i@6N}%?hEMSNfjE*CRVL`%9P}_zbP0ype*(l%2S*e=!P6-cW^KY1ZmwS7 zJ+oXNoh^9D9wUSW5QzxUp&GEu_C=51WdX4NhZ*ls1=7S98?l@NVb#3>KR`=|*dhF0 zB}ZWJBr%%zK{@We9&n-^ZP?7<@XqQ)@Y@B)Q-022dnzp}QH?l~bA zk+96{PmmD|JfpCI89^57fkSkOsy!42K&9bWJG5~qY=S4q4b+r7I!E9gVr&Et;pHHZ z+a);|OaL0+jGS#w2}VvGW4SrTmxHoo%)AkbcM&J|ycr;yZ_;#>LV$?pU z)Qxw}9^aD54Wd0Y0Fxt*)jvgNWQlQuNOyjsD1FTaTNv0D*a*>EZ|*Qsmg;ir1_zyO zee|E3PC%XfJ){r_I!@WRl{P?JYREqT>hKilbYEbz671^JRO@(#u3e!8-@Eh( z^SX&YA}3~6RoduQE3ydDJ2?5{a;F?eNS(DW&rwx#laa@)>bZ}O1~H~{X@yqCZ$}Vho17pZwrJG9163@`W5T+I zi+lUbo(p;tQJ<=#I9`2>Bu)am2iDf*KpvAorv!#9Opz^Z=0x>qpq98fmZAauYc1*# z2!=cMben()dm`e)yrpIWIuX3xpY}fOhG5blkjf;G*%7eR=yc zGCmPV_6i2tRBh;32=~D0W9K$V*hq;b?l|A=Z?SZ?bxND;ZK<}a?eXq4lDZpLjzLK!B-k56gc(*iMEu~OzH)_PZA-czd&N9vLWIc! z(Ua~N_zmPkTU@P~qfG?F2=B#F2e)5pHD(N>)u?Np=J7nTfnE~|tQl=&543VncRI=7 zu+gSbGQ^{8|1uRI8p#c4^)wbvl zm?7{ki|6*-P-9pkjw5I+H-S9E91Gjr{Wv4WccH6LZop5z8cCRcmda_4fX-jj!L|`KRZAG z4m5ATxpn6bat+Gisa{Prw6RxO9KgvmKKr$u2)TjWfRIzm0g-XjTdXHAk=(E>b$4A^ zI0cLVIWAUipa*T46(c~@Mt{gBe%{oq#A06{-n@T*f^%?9uqneHDn{JW6^BsWcTis? zYQ^(zNTp;b2lC{}yK$-xl0FZz)DQ%&ahf$zLYO*!xGbP;p`3$<{ToqUpff#JDCHd_ z$Ag2-kYLQMK8~{E<@w>-&S{;kS8o^2!7dn~zzw-_j6$FV9NL~tT2x zoO;JefMdaQqxu!7CSeStY0~EArkb#<0wI?s$~XHgErC`;^$#^<8p|7~Vij}4uqk>W zfM%Jy7TT>kDASJp+U@|MD&fGRjCTj#N$`Fj5Gat%jh0YRyPmD#% zF|L+N0EMH=K9om;XU%pH44c!CC^#A6Kn0@}^5L#Yo>^Ik5NH4ZDMm43@YPw|C1zIK z<*-=n_1$MF2M4=fxR}`F zOH{;e=g^8OD&b@_)S^pH(Ut>(Epf2-7<1nkp!(LcQ|qLl@1AxB{f1^gkj#@f3YHD| zy`5cntlvF!a$$4dY@&+h7n#phD1>BRqyrd>UaI%dirs16+Gj^)^wP z+wmOOh?}+EDTGl>N#IF#GGfb~3QJpK*e)1$yROGaU@Zh5wCw1+Lu(x}{*DH(l|icE>!F<^ zI^5OZ9xdyTi-8s7^u!?&m$p2`0LJbSH`GrY54@H`c-w(_I^r5zMGWBfD4ilIj?}Ip zz6)DR@3t$)UVJC8+%rE|d^cLK@p8@@()n}2Tu2s@W*1X0rwt(w@VxVWD^G!v!}+cldJYC zHw4R>wF~Syu(nFG)g03N7V4zM3&8dY?s<_X`%wjFSj7#3bnIGv=frY9>U02KN6UMt zyW^B(tl-@9QaTi-bfb2vf9{fprvEn0aC0_?pn-kWCba`~9NR=F;>f0xkrA;CF+*Qn zKOE2PHu1N!W=W4OrYc$fAXa^i=E z05H5DSN*^~0Z=mdG9=)R{hnMy!?oCoR{lx9_RxY8p0Kk6=)Z=0c&wY2j$`XY@8kfO zxZ~S}JdipJb;P_==(WNgA*})9;6{8V^gCm3anh|UKI$oE94E`}^^iDN0v`ikF^Zj@ zA}A5l$vYq#F2+4v+ji6}zwW5Gy1*fX-Q)QDvFO_oCk~xlbVC3Wyw^EE&t1f|&b>#U zn-vJ;>h#1oyb;N88%f$^!njB!qpMWJGVW~+v`bwRavS1eEIpH74l{>75Ti?*3cM#niTpj=?Dv|dF zfiY{H6a#Pob@bKQ!2?`GU^(@Y8)3W3p&5dSfvf%zH+HnvAo8yT}zo0Il9#F1E(I9tx4hmj#vJUBQ9u{$*)yl5t} zg>|>rMp@t$JaQ}rs0NCN%omNvPIYl?JdS6Bu1nqmk>(&gJu^Rw8c5$|3y? zXy!qY}JZR(Tey2FdiRgg$Xsf zd;k`UASZVw4*M4Vb4!_pi1ZpCjKv@cj~QQ`<8_R$TylnX!x?oAu5in^@y^LgW|bRH zvZA`y(Kd&%G69&IgK;_h1D|0%&?M3;}^1GPlc}8l_Fv&lH?tj^J$K)s;7>v zTG%)#qssUI1UYqRxQ@6W+TrFP0D9&)u1a8?IIODP_OPms@}iQX1{f%W(_rSj&`}bn zXZL{I=j|+PAxw@~T6X2QxfYDWIqa@bc2swS+tCMdVH*zIisP0WRNxT;U@B0D0yi!*L47e!s4XM`99B5pGlN?dUvi_0#?!ujN6$kF zkcCdpk=1au8mi^m{Be`7h4v)Cy2G_Jd!*iB6TYWA5LSmgK$GqcW+S0KXQz<+hH`0| zw+Z1>8R*a@N?k$@s=yVmDI>91NcK&wI8Le{~XV9#vy>Tsz z`v}$EF_?W$LGmf73&;V6EXn10G+B3#`yLzO6fH$aT=obdi~ET&ci>}PM`7eQ0(%3O zNuwK5VMjAGrjriS7aMoqh^jbs8y2V#^PVn~Utxr?lDgFeO#7U;PYeKpZZEbk@h~ZD zSv%LoT{aGGA1~W zp7jc_y-4PM^c{yV-DD%yFI7bK~IsQJ8pz6`ZxsIn!@g*ia^~;x$fy1v=~Lkg zBcMcpLko9t_U)BwV?Mv{wi}O~KKoO6lO!l`aH?qlb`Ia^p))Gr(K4b*1iTI6mQ=e7 z!JcMz^b~}$oXZn#q+w|f-epPT;SIMN4pBTpY&6hTcyj&;cDqkTF|i@XQVaV)jB!3R z(LhCPo+c*MtY*2#*qb@#OrI7c)0;Ge$gTB`>pW*UCwhl=JzqoDS$W?rL5}1BZ~nIK zd=(|Ns)o_~6rI#1zlU%~LZ41$ZAW+zVRGPL%EN$jv< z*6pjGDD4JRv`*yi8-4J!oni1dVn2H^n>}v%JrestEOo;Epx2*ZeR?3i`^gStka?sP zCfJ)&96%Tcq+Q__bCO^LYxh??O$~tbq+SG~$Lz^)6Wyu~%tY=-c4;SP6E6Nj)Yd3nrMYjvtD(faa1v4S6s}=`u?!1e#vVB)uzwEoO_8X@M`$p8+oIic?HP6ZE znrVsQkWvDUOx?e~dgvMBv!9P`UIasAxmMN^DR!)C`uzUbW>q{rw2OjBobbX{?s{g_ z6$z5t2|K0=o>F`fa#&KYAIUx_z^MEwA$Gstvp-Jaotlo?G`LC3jmgOyLXPMT?#~-b zOAZMIJ60oM;-@fWw(34&xQ_DlGjOeQ0`iPD2_K|f@pE#i6kN=JvirH0tC13c4QfU# z`p~=w=XJsmh@XMVzzH_T{k~ww+(2)RVVYOP8uwbh@ECA-;#tXX?E_xKz&Wdoo9~QU z3oiBokTyvXdmGAN@9W-l;86E$^K5K?-z$Xg*mPU!15n)rK`;$za{xtUAaRs&9qs{; zw@>BIWhsPUEK}qUU)` z5PW0|FsuWs1C%`h!f@0>56p{&xjaP!W0e@ee!Ba>^`M;|c+jP70PZ@sfqv4Il>~{{ z$mp9Ai8KDyG=!js2BhNMAz+HZT}PB)w&hrD&b2!T2j*R(up#G0tA+mGfjZZ7k74P9eo z3=8yxNHAWrMPJdh z`Tc8cdwd?+K3Qh)#kG6$@F+tbcL^@2Q(^2;nmwFm?%?plfq!LtZSZ*|V_t+x2urvF z?u|uA!2||~>`*}_?>um+4$Q(1O|u8XRyj6UOG(_IZT3L9)Q&@)I6G%Ga)FeJ$>q4C zrmO*h&hLHhJ@U7A(-y3#k$i4%H{O5cTgjJCH-4uQb&pTc20l{p-ZvP}&GL`Wt>i%G zkIqp^bayN4h&X?U-k91S-oY9Qy2=t365tik58clKMXiKcP=qAJg4b{e$dS6~{L#zj zu(>#S z&FoN5{;+{>Jdsb-6Rsh_Z@&4N`$JYMUbmq+(xcBb@O6v}xz)#b{!P!^A2L1!7*O&x z=CmW-xo7g3EiVLvZ+zx{V0n4Hx`DRU9AdM3Utr+7-w&UiP7fqjkiaP8lHYx&{yBi6 zQ-~(yqr--?+L zE}9;eu-AB~KF8?Sp8X<-Y#HP2<^}TQea3HI-~+rElxQj(l&HQK5qi8|4D$hj=}QrY zjs*O?Bn2t49|}i;eTt{4NFd}FBUB_DiS()Mn+PDkf9Rs+ zmr`?>Ycfiz+g66vi56%!vx!z?6YDgUYOB#*HYd}pl+J9M@yxnGjU%y1an)0+N>8om zy;O7AWvW_9<^l!orI!dHmawn$s;7xtpop~A2ANn^?zA?G9#fH_q}*)}*F7dnWl~*# zZ&IZAR%Dq;cUNMnF#rk6X}QU7 zrklL4pv1z1>6lNz?V++zH^nKFYNNTURDQq z!)_``brAe(?-5*(nsjuJ(`SBM$IanR&~{zY}nH@jjVyA zheB$Md8}VKjAOs%;7KPwu`JSy)JeVoK0>_RYemLlYlC?xGXcJ$MHmNR$*YA^w83?| zYI8~CDS-7iwq|dPyTMP_#+?Cqr9d~F;+2&E8V&T#6ptN4?dcGMPt!;FP41>ZhHugd zufy1z?Sc$$`F3{*Hv1(1bSIImNP`|tDa)=uvhB8$e!iVmx=2^Wz)MMsL#ch52RrR1 z#i!Ul@ZExM6{lS&4?4MbTo#dzoid;gPmBSSQJbLqv;7XpK9oH?1|V0;a;22NH5zytrN=Rp?;v@f@M2lD&EVPG z6zYzV{A4DqB2Jpy)3W{h#7CJ7rvg)5fvp;KTI-W?1G7~H{N*)|{aHP7Snoy;K0o5S zSiY^;%7RWm;A7i1cqX|3`^SQxf5H~QoVQ)o zdJ4>QX=smD6g)3T?$0d@zktvNd^3?c$%{Z2u?;@05*dM=qcVfT{$(Z>6%XqI+@6el zM`Nx8vY#BoylNr6&i)OFn0$igGb%)7m}*~0H)S-xI7!?8^5!%DrTDdP$UNwceVG@* zkG#p(zmuJ~=Iid;lLP(=^1V?k!d!x^{%(QX+&VA$qFA{2yXKTm;25jmYuRsJj%H z=z?!ow00~rf&Zo@QXhPj%pjLSQCOJ!r*t}#icB(SKG^a@RJOW9G17J_faj+udIr(B zL5p52AjT?Y;?#g=I!@RT8#d_slrqW0hI1$v$t2e@->M23*~FdLALtV$wzJDjsyOP( z(3fMabwPiHcT<`(F+%>n!h9r5>46?h&& z@nDEaR0dgE8E3cADa3anM%3I&&9#i-ssKybEFJNP0rCwH-Jze`ZfvM`+S~&AfaAot zeTz>1T^G?gSlq5$0$xX zKU|d;%!3w>JlMf*ppK4OwCicz2r>khaR&ovD z?`hZOJKV=i3-^EAg;)^CGp2!)U#zhOd^(fJSc26H=qvCt%r(FVZpX5v-Zr`uTSX0c zPeEe?cn$i-xc8fw#}CoTjMY2(&9@J4=t4#{<^sE^ z!7J4r$bx9d*iB=FxaB8EmnyiNiHkk1w%!1{;O`o`@084#@#GU@d^g${{(K3=<&m987ln*MEg|Ml z<&sEWA-i|YmaQQUFzmGpOOSh5fkL`Ci(=+lt~OrWg0{Lq3sX{Vyw7J2y@z#2fW3}0`QKr2IoGrNq8{Y4N{_Y|B0P$C(OWO?k%(6}xhK;gylM!Qf*jLOG zJk#v~?>~zi7Frt7cIo3Ht%qBAD47xREvy|U{-oCEB$ zEJ17-`ZtE}5O*nRcwU|ALL9uPSTNGYdI0{6ir3e+Q=?*}Ch@goFzb~QlSJHC;H#Tl zraTO%YK2K7AohcU)Ve+Q_BLHck2mW4Xd`UX*SMP)m{4+)y8mCPYRQ5qnl>CR3Mk@Y zHmNJ?@*-F8=0dIZsyw7dyd$ZnS;|2oQrE;VSFf1HHGHzLcVsK6v9v#Sz z5ARS0HFO&kxk3nECjn)hDfrJH!C`9Y$-Rfidqz5IWfMbVH2O{Me~&%gF1*Zpnfx4-F2KKO4lpUVH%fA>ZI``3l;`j39*JHJ-? zinG6S(T;~cdiCwzzxOcP--|~e&{Oe!G z{mzdG-}3J0XZ`a1pL+kl8lC-k^v}NSi7T91Mzz<<$wMIA?2^V`2T$R_q_Cx@lQ=N-}N`oKJ;zjpZ#C+ z-~Bh=^ycU8J?Eql5FRM>+`Ui#^#AKS-}*N3s`ve0`PRs*JNaM!)Ng%7>H{D9KdA5f zTi@UR;g5X$AN-wp4*HOKm6C;llpf*QT&1Jpa0;W{15Nrf93v{{m2jg z;P+p7TZxbT^#A+(XTSXiKmWHf-}(oBmHX;n{f=MxqTl^b^vi$x-1E!aU-?Hr^iuoB zmOt~}d++;=+{ga-m;c@JTO(ic9slrOyywHd!Dn>J7yo_wAAHWwzVqL2S0`Vr@!&dfBhf- z=I?yfpMUpPPyXKSCx7(ozws-lUqAo85C0kY5WROB@E)W6ekvTHz`HLUPQzUrdUmIQPLB8^ zo8YP=Wm)%+nM$*mDkc=ALn%8UwWbS+VydYOSgD;Y@<}DwV7PUI&UX08kQXMKYUk)C zxB&%-8rdJM4VT+2EFQxKfM&$piibBR6zc^)Y{l~zXxJh;N^A>pVOK~MwxxQ8D%3fu zvrFuXlLTFg3){}FLzn6UwlgWp1vB!#x2uWrq3;FG@WMiqZ_*4>BMt$UiXo_^x{`O+Gf|)5g%tZ z;c~1>i$U5uDocq@G9T^cwS1Rpw>sM_7g;UKLA~CtcDu^}BPaEew#mf;0aeQOdxLdm z!<9P$cGO8JdTJMJ#M+aZx9{(jtKC3(TV#^WWimA2wfrX1*QOC|x%AZ%O0>=78p}bQ zt+RX+JEgBHAK!IuW%BWZoAmKr8+wC}pOKHbZL^n7slCCHi!Q}zoQjF@UO|cHm!0@( zFr4UfdL3UB*>Y4K>TJ>17ix)Bw=1?6K3{N@Dk45Eb=gpSu1CVnf>-Gjy3}klY76m5 zTdz0ba;fVRs$(j))ry-{H5_9{tr>`%z4it_sL@t-jG?uaXb4jL|=Gs=AL zYwU~sCBqjn;lGIfqKQ)4=^$!XopZbT)xbzyy$7$qtNE|B8<4S29l_gYQ813Qrl zaZXCEsV$rjp$qlQ8h*onT^>Cv#?i9`es+c)7NVI=BF>efgJLtv6$()rmJ3@Dh97Zj zal}=k17S8Ck4oiACK??y=7UOw%M6M#$JLi-)qHy*N76aEUfyy8+83`XbF~~xO(&IN zZIOv*hcW*&Fz#=K=2{Fby|HR6M{{s?B%cWS)2Z!QSlW!(+*)6B`)t#@N-xAfFzBah zt;TNE&c`%myiwuI)>gV&aZb#vIpa#n!qVsOg+{5_8Ro4-F@Bxj5X-;M;QPC}5O?KsAwc^p zt?Z7a{AFsOMut)>TH93$vy%1{d_FqF=i(tg7f^%1b&^>Q6ZE{BJ{z*!xHBm8+{7DS zg*ZJMi>%^ea9Rts>DZ!Jufn?7DjcR7;ZZBGicj^OvP$~`naZNyuh+eY_`J@hmB_Nq z28#@}%IOuoUJ=6Cxw>0VXTmhadgIAlc{^y_lF#qzV%(L_GioQi5(eF2 zXRDRtxp;Kw)4Q`+pu|?5g3s%(37;o!;PaaIOzsu}iHS^gxOF<)pTu{~Q6@c(MRN1pW*w(kPRg&(CZ%|{)!CGo+(wSCqqDBKt`_KF zyxvMorm;jfIuyO2)B0j^6q8GOLu*dL8#)=T=>oIK#}4s%CDkqaHkm}K70ZK*9ql%I z{biz^&qu=7_`H%jJ5l&4`J4;+SDgesSJZlI_AA;1)rP@%oq3CY3WQvhqTraFd zIzNju?eKh8Y|NHrZ)(Tr8CKmC)|)BChWxtH5106}2#9cX!p4)dwrY!$Q7q0R!=+eL zO$GzuCN&uPS?|K1p82I=I#+KkL+$ET?q(|*Jzh+FO#f^$iUjKQSZPw+4g1;6qF+kV zfuOJ)MHcDqWVG7`reU!iuL#<-S7~}29Z)RjM{$Z)2s^rp4E4XBlOf4~sCPuNAXQE6&yg z)}LiHDpKfrxkM=64kdzoC#&YWy%tmKtNo%kTM@I(p^|O2XEptd6(YeM*QpQ7fhk>~ z=X?bcNLR+mVmmHG*M6##YnL-jZPKsCsd7o|GU2V7NEVYcr(`0zTIfs=I>SL>QJAp( z@n#zufmdBpnrUCJwo*3JlG@#^>dU?|*8B2sQ}5MgRX-bERl3`Z7-0O(3D~yOZjqz^LgY-3mrp;cR}B z^EXndt={tuBfGvozf1XPZXHf~m&w@Le-`tHsJ^=9^eWAdnaKq1vgyhrW~Q>e$hGg70lcZUVYo}YhIM{86mC`_MXViFYGzRx&qA67DOt0KZf84M(fO1=zG0(DL zJ4}^xSCeEm?N}Rt0Pu1FplN`0HaPPWT7bqE&)+xJ zAs|=QnVEQ4QY4rg=_oCi@%93jy;w{YS-*e&uJZ?91ce>RxW+4#f6J^#V42Wf9zXP~ zUIAF+l?mV_0)S*K8$;GP&p9ObFsoVF#$2}(K(;k#6Eda2|IvT@naR%~|C=J>%ZebW zpy+wGEWX<#s~(IwYVXO}nTs4K4dz_DKjEhjJu=Eq20)FZbc+F?3omdkLU@N-#tGKY z3&3`%;Jb^@esEQMFBfZTb!FaU9rq}Q5zO8n4f1Ot=({F07*Nr!kz7J-yE zb04CMfg~zybk9iQ2Ww?$s1}fY42={f$RRZB^f+!z9Dt`JEs*v87O<|IDUz&;4t9Z@ zhf75_x_P|;)G<`x0u42~>JKGxg3_pSHpVry>}9@nP=nMxiY8GPhp>802ybYkThQN` z{>zOHP8rJ2`H54XU|OL2`yit!k~@+$-5zLsH6XkoudGX}AV(>{xsb&RT>}|`Ja+dV zJa~At91F`oa4Q2ow(y(jXnCMaIui zeV*?{SSl3g_rk#-OYu~sA7X?68=xYTU!o|1LCe#u6o&QcFyy~tztw;m)F&bc?Fxhg(IGOM!BBlN!uZ>vU&C~1X8Zavruy_FV_ zSF0&C3I&Jino;RdRf)_}jY_x3hu4sxYnPifl`NG$HWgU=DQa9is&JzO zHuF}N9<-FQH}jGmZn(ySf9#axpoxOX$RGPOQqmL@k?*l0?2(f~kQh$v2~<$v8A@UT zwCJb%G$lk}r(l>3Fo95?_45G`kA8saMQE`ng=v8@Go+gEAD`odq>s)|NX(^*!c=Rt zgx8Qwk*U-sZHg{wb6+u$m$DQr(bd}YZf{I>T&qy6tS{BXz+`jXU~@P{2d5GEoe}e` zb31drt`LvnwGFba2b*!gjP zLkOvLlDR-4!NoH1&VWl?svD-6i>G_dY^zvrjVm!G5GeDhP)r!ptDqLCt8%On&P23r zXPsLY+1R2p=`QotRyP7Ew5qxKrAh}QIv1&EBdWa!kA>WDxLJyW4ZSK=R)q|crc0?P zL|*Dkq0xw~#bjrY&sVnTJX`ZEeLE(2scQbfx*g}T#*$kC7>#qU>(y%OL{aYc{aHnB zF{|<}81VP?aSvt^b*_J@x>xpUdR?!NrSQmG$wqRUinrC1x_Mp+)Hgdqr*t)uh{d_o za9#Cf;EWxN=HAktwXzqTQu9_b(z;QS;No<*Cv5oDgZR41Wf z1VT|6Uw)9vC;T$Cfi3;vs4hS#SlDK}EorsztD-J1GqWyavK}=X)Dq0u2`ln z)rdC|+&0qP_Eh9|tCTm^E0smP5$y;*;ZnuScxZ7Ck^#eRU`U;7Z*4+*M(^?m0NvqZ z6g!4_vqK>EqJ=~rZX4EJ>+Q`}Vj8}vD3AQbMq7qFOmWWTSjD>&?vJ@Mpf$VCIvwZyK%vH;QmuNKj36af(;0+v zIHoGeiUHa z)7G%0R!Z4cGSiK%yUZ%x)VNGG)$OOkHC0>oQ}rpBo^p37=wWs>sYH_)^2FA7rE1G%ZDXg_Szs_+o~R7d56gS3)|s(s#vL)_1A0tXGoCrTB_Y@QG$|5oWpQ zz*pZbbZOSv>gnR9Q7-1IyR=kZLQd8~wG?mF({#rtPGNmo4FL|BIp<>vd1*YV3hi>R zo{UdtYCAq{(?i(mP}YmJR)s=|UPVa87K0sD)6|?1+lo)tGFxhRsg8aA$ZSL}eTj`) zt0>-)I1I{iV!GsIrsiD*`eH_Ax#U0!%wy~6C|lZu2bxyi1{;ld$-e_SXq{w_jpk$Z z-eOl;C~6|!@cCx(EIp1Zy=re>iI%ovHk>H8r%YEZC#WzdM253vPLi{$lpX`AGFE;Z z+;eB+K{I}AJ;K390oZJP$avUAfFqG%F^vZm(>x+frYMtqi zO1l!x1qYkTJQgo*mfn$^^Lo`c_zb6;o|mgN3NEAC{mRn{r_`Z8g-2IKP4H z4nr9XMP4UE38BUb4_9g3#z@*D@rW@nau{o5Sy66xeBTJz7UOa=WS`ODFZWeisPKf} z;DZZbmv-nf2-uJxWX3G`+t!8ZK4^)ASS1Fko4b*8pd!7HHIAT)uhperP<7U0LA z>p-rCa7V@YBR}P*4!vDyeW>dP*BSGhA|J5y*ZOQPwJ@+%l={eKyU|7D#Gcbn^Fi^Y)=XN2tIv<9Blo%Gm@IA~xvS2ud zC&0PgJ_ARLB}ic1kMMAMBG{)T6YekV!Q}-?WO`x9)-Lk2$o3d9!U}MfF91h@gW(7t z=)oK40UHd2#Be|ihAEa2`;Z;G7YK#>j0yM0_TY+;M4OR9aMnPW1{s?93l-{Z+lx1gmS7PkCdQy7ZH1tf!@{1i$!A)Whx1tu2ms|(_ClD!CsH+ zSIXfzJIy7O-gLk%=FMS|KjZ%#mn@C=&fnIRu z4HhefL|`4~%-CgP&8-tr-_9MsoHs%ZHoeOD#<6~F(I_huW^$?4=&6(l3+w#==AqJN zS?88^paa^a4lQ;eW}56&29gkN4?3MBr1;mob*<|Uw{o!=n@c4(9d$vKV?}6gmu{@$ zGk(W&5DTL+bU0j0XI8s7iy}zNawwG+H?>7=+-yX5vv$23+Qmod;e6ekHGI@`tB)p- zvrvFZO?Z+nf+HSW^rV2d#z}Rh6w4fOpAGi2m>P+Iw9)Uvz_iK06_L{?axGtB+x|{- z(T=9q%)IyP0YrQV0FK+m3%g>SY>p-Y#0Gwb0*L4oKxdPTb2Su1!vRE!1P~$V5(5b^ z=BI*)mnsS)W|RK*DpM=S?PjkLOvjZpgeBMM>MEJ&Z-aDgTxDW`empmqvSOTGB)w}X zT3n4*n?V#@s(G6UrX%h0q#v2}2MH(gb$*EBWz6oKy;pa<&@Qe9^hDkqmLL zTSicp-iEhRV0xtmKqw{snx9+4CjWvNM6~woyAKEqvVlsvo>;&Y4@t2fsfTA15uCDB zlLJ=T-#T4ASlTG-3bZ2zwnOc*w$|ceu~SnenpWi$r{#0Gb!{vRxL#4MF8qo7z*h~= z=hf^)f`lu5Edb%*hO}JKrC~M^Elgn+sT3l~7{_LX*_vK8I3<#vl-F@qOpKO|Tt6&C z#_M&cM&+3{Q`5_(0EC_*wFpzy{eylJ26D#Tz`%wAM>`(KzK2)N$n6xLZ|HsERzEVA zuq;0W2W3oW89oUz|D_02j*N85td#^>`5FEgd z=nj~<;hUE3OfusPERc@FS5~5EsiA+4C@;e$y?{*#eR*K+QZE~oOwZr=dC$BZ$}9(= zW0*Wj>H_Z`j3w)6UU%)j>GMa7frYs>Z%>1oNT8thOGaB^Bk9OK3hywUo8@l5C`p{+ zH++gWe2P!JPXV#y<$Mk^ILjRt;5JWK$I&-f@&gwP3gsM~^)j{`F+C1p8OR-Ko^0>M;dt95u27-8vdQ=Q>1>E? zvW%L!aa~vI5#Dwhipa|l_QsGYus4&#DeT^*9Cr(T#YJ`&Z-m) z4m!##DQ|t-WN*6LL;@NG>uM0TZAa8B=Z%MweQq$`Qm{2gXobd7T5gU)(fXpk9LHEb z8WwBCNjfUSf=D8l13+}E95c2o6gbMc3JdXp*t)@S0a#_O#45E=E;^IxEi;3aG_}?V zKpbbC?n=$cINm94y=6L(DMbapw>M75JIPcjw#r_rsZw+lh|*a|qA*&vy<{Wbjc1t{L<(ZfR(~r->Q%uP0>Wftv1(-_2DG%l0}xbiZ_Bo-J1$cjv2tU* z@#V5JIl%jC>qV+jsP1B2YMiTVr@LlBXeEk+_$C|Wq+Tquu5X6E%B3pKc9B(I69?5g zFXrV&vMhDjg4kR8TO#=|1P|oU=Vid1SwIcV%@=yVlJ5|Mmah_x&9Oahg8e zYK)dA9UNV#m0;j4EXGV?bZYj(JWV>4L7xy8i#_R_)YCcn;!VKoB2tKH1#j46Qe};6 zvz_G6neDtad#@sp8ABMAEGQ7uM!rYkat?|$RoQ@TM?elD!krY^nnj5PWCsGm9kG*zQuk4;gZ*NH z0O30p!O9j(P-@T&cT9QRY}B&7BtTn16WJ&j!kl!d%Yts%|o{l>%Vh&sENDM)D!BF4=X zS!9*srti)LI~NZ+3u#z6>6AnW)`7al(G}p8Vlmknfgnal7?qRRB*#VTCEpM^Ccz=M zlcy)ELQjHfzY5y$30)Yj5U*_MDjlt6e4(-A2SCQ$aVs2>!d?k(<_t6oTTSX>IbUQE zXF35mp3QUVsLa^@|f8?%bdCGB8#}0)qh-M4S$ohZ~EPF zkRdoc+u-CmN1*d=k2C{*_M*BHDiGsXFPo4!vr4rjs{1Nii%T41)sS|Sw>k>eZnx0J zs7DvtCD;P?qQ$zX)|SiSiW(EL<@G(vHfr+7<(JWP(r#%?bp8P>y(@1ELKh%sXHQ=^ zcGK?R*F{fd_u!_K{Dtnp!*Iy?H{Ly2E^)H}+00yKW$kCD=8GV?AooQK=xHG$>&e=}HS6w_cFzK|NlzS{_!( z(qcA1hoOy5vFX}!DpG)yM2p0VJ}b_8j2$QXcqQ8tCtjj%p>8DAn49sdnq%u@Qq;YE zsYT>DkJZ>AFe8B4O#nosTrmb}yIdcPiuvWL$@K*(ZcZa6Kj^yayx^Oqo>UO} zIE4xZUrTgSWr_B5sKAb`z!;!WSR23W8+}SSDD1uMLkYJwkAvisCB&_kt*}m+lbY13>sLhy5+~g_LXBFoo)qGR zN1EzjK;-A^RWU-_`KCk<`cSB|5&DB(V?9VDQ}0jmB+s>)eYpcRuUSOxJ;I2ZxU(`! zJ-%60;tGv)fx{5hUAwfBtf53#Slz>VoFqvqx)z;AWp1tvvCe5Fbh#EZQ8gjyGNX!a zt~iTjo*0u@h1B}C>sUFI=n)o5DtgXs$x*`^^@X-q^Axt#k^4Zb7h=VD;!F)gBi$gH zjOTOkmBwBdwM9C}hrMw>!a&oZ(CV+7er`fxi6K@HN47irW!JaeieYtHb4l&S&B|Jc z=&{YBbUluT#7dKbVNK)piA7OWBab2Zy{b5KXH#QP?AY8ajz~1ZCIK^uo61NQ`YmcX ztB65W=ehun*WiFqtpTVyA!mEtv4ukVcHlC_F;S+)MJ0zJ{m{didEXA|Rw*vesyU3q z!Ss7{X6vW2yEHfBkOPU#kC${97!7*%0qS;AjTc%kG^4yMUcGB5Epf>>Ma}(f~ z!ZkbTY)dvf>HI&o+3^S36!<&`cNtvsg##=D5}cl_C7v3|?g;`wipo9&&~crV7_qwz zsV4m_xLzr=H63Y7c4wM_a@~~wfFqQqk{S*)IH<9q5GIPHbu~~$p%8|_C=ImbG?W7D z8o<^xhB6QX(NRT16@=2ULJFYVOaKo5PRB5&2-qNzuNa^=)9<;{?u=+^ie#;)v5>E? z7FMq*D7DB01!caARwtofn+xT=rAyG0{Gu|Zrv-A_EP_;Mv%H?p^30g$_;I5)aT!iU z!e%v?wCM>i`=Q;K5HqiZ7ecWLn2)7Wjon6r%MJ7x@tjrVz|+W<nzCv-VBDJN6eFemwAJDFyIyOo3Jj6*F`btWS!ZQ*}6W(o}&q z)1oGfsX+-7Tq__rf?$MU;0gc)O`3)YJ@lP%{M1$1buQ#JhrVHKHqlRJXSAc=^PTMT zn||5bEI)YJEqA}^oH_9WOUWQ@*4TEPd|Rs{Pq9bf13aKlW+X%50BTo!kKHY z`}lqR|GsnO&!S&G<-PLKr+#?X-=DZ;m(M+V#b+Gr)#a^c@ZY`lx!%H&)%fi@zxK#A z&uMR7A6)UuyEiVMdGO5X8=C+A@Kg7^c<=S6UwXQ4`hR`)x!+$Q|M=@SK0v&9r`>(| zBemOFV9cFpa7!QXzN+j#ux z+V0)2KKPx&6YteN@x8lVe*K|G-uhOd`uvB6KY8NatKWO(rav(!PNRP_A7HQi#+A=2 z5C5_8+V?;G@CD*dhbP}yu_NP9(e;LGYBp)x?CoVAJJNm#Y`k=$=N8=P56sp9Kl%f+ zb-1Vgk^8?zs-6S`VmgwlbXuXBnUXsU{GUQczI1FPJEaqlO8}_9TeR(I#e^ zD5`5DO^}oUX2U?>KfrJ6C`~G`scVD+dst=)54Y13@U)yDZYGHnI*DqCqUjVt04P~e zO@%Iy8VRs)qEJwD#z1x5M9ox-fMEb|4#Kw#39xipC3Ord=rDNVPa!KDK;@V)iQ|Wyk^_v% z`(R%IwBnk(C%$~gu5F~efL$`JRYehCP*o^PqX2Lk%miErY6qksZjvHsMuj>!P*MPF zE}<6SX9FBw1N8=)LNR6c)K~Tm?17iPF|#bHcp~fh;-uA*7EYv6pf5HrISdljlZ6xK zEvvGiSy}NL6fcF1d92G}$;CWvR*?v9*-6c8YaG>wVO8vLTz)!29HK*Z`PPb}f*~QO kC~X*H2g;YO#}h2-NqvwrpsQZLW{kH7TZ1bKf(FP^Z)<= literal 0 HcmV?d00001 From 8b5e85e8ecf6fa7b185b5b8770200a71edd048f0 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 15 Jun 2023 14:02:36 +0100 Subject: [PATCH 017/135] [release/4.x] Cherry pick: Update OpenSSL SHA digest API (#5336) (#5343) --- include/ccf/crypto/hash_provider.h | 2 +- src/crypto/hash.cpp | 2 -- src/crypto/openssl/hash.cpp | 40 ++++++++++++++++++++------- src/crypto/openssl/hash.h | 3 +- src/crypto/test/crypto.cpp | 44 ++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/include/ccf/crypto/hash_provider.h b/include/ccf/crypto/hash_provider.h index f93a2c064bce..55c5f425e0c3 100644 --- a/include/ccf/crypto/hash_provider.h +++ b/include/ccf/crypto/hash_provider.h @@ -47,7 +47,7 @@ namespace crypto } template <> - void update>(const std::vector& d) + void update(const std::vector& d) { update_hash(d); } diff --git a/src/crypto/hash.cpp b/src/crypto/hash.cpp index 879c07175770..44ffe11b1e2f 100644 --- a/src/crypto/hash.cpp +++ b/src/crypto/hash.cpp @@ -6,8 +6,6 @@ #include "ccf/crypto/hkdf.h" #include "ccf/crypto/sha256.h" -#include - namespace crypto { void default_sha256(const std::span& data, uint8_t* h) diff --git a/src/crypto/openssl/hash.cpp b/src/crypto/openssl/hash.cpp index 36e01feba430..ae93518385f6 100644 --- a/src/crypto/openssl/hash.cpp +++ b/src/crypto/openssl/hash.cpp @@ -3,6 +3,7 @@ #include "crypto/openssl/hash.h" +#include #include #include @@ -38,21 +39,31 @@ namespace crypto void openssl_sha256(const std::span& data, uint8_t* h) { - SHA256_CTX ctx; - SHA256_Init(&ctx); - SHA256_Update(&ctx, data.data(), data.size()); - SHA256_Final(h, &ctx); + const EVP_MD* md = EVP_sha256(); + int rc = EVP_Digest(data.data(), data.size(), h, nullptr, md, nullptr); + if (rc != 1) + { + throw std::logic_error(fmt::format("EVP_Digest failed: {}", rc)); + } } ISha256OpenSSL::ISha256OpenSSL() { - ctx = new SHA256_CTX; - SHA256_Init((SHA256_CTX*)ctx); + const EVP_MD* md = EVP_sha256(); + ctx = EVP_MD_CTX_new(); + int rc = EVP_DigestInit(ctx, md); + if (rc != 1) + { + throw std::logic_error(fmt::format("EVP_DigestInit failed: {}", rc)); + } } ISha256OpenSSL::~ISha256OpenSSL() { - delete (SHA256_CTX*)ctx; + if (ctx) + { + EVP_MD_CTX_free(ctx); + } } void ISha256OpenSSL::update_hash(std::span data) @@ -62,7 +73,11 @@ namespace crypto throw std::logic_error("Attempting to use hash after it was finalised"); } - SHA256_Update((SHA256_CTX*)ctx, data.data(), data.size()); + int rc = EVP_DigestUpdate(ctx, data.data(), data.size()); + if (rc != 1) + { + throw std::logic_error(fmt::format("EVP_DigestUpdate failed: {}", rc)); + } } Sha256Hash ISha256OpenSSL::finalise() @@ -73,8 +88,13 @@ namespace crypto } Sha256Hash r; - SHA256_Final(r.h.data(), (SHA256_CTX*)ctx); - delete (SHA256_CTX*)ctx; + int rc = EVP_DigestFinal(ctx, r.h.data(), nullptr); + if (rc != 1) + { + EVP_MD_CTX_free(ctx); + throw std::logic_error(fmt::format("EVP_DigestFinal failed: {}", rc)); + } + EVP_MD_CTX_free(ctx); ctx = nullptr; return r; } diff --git a/src/crypto/openssl/hash.h b/src/crypto/openssl/hash.h index 643561df7c2f..c02ea7112462 100644 --- a/src/crypto/openssl/hash.h +++ b/src/crypto/openssl/hash.h @@ -7,7 +7,6 @@ #include #include -#include #define FMT_HEADER_ONLY #include @@ -75,7 +74,7 @@ namespace crypto virtual Sha256Hash finalise(); protected: - void* ctx; + EVP_MD_CTX* ctx = nullptr; }; void openssl_sha256(const std::span& data, uint8_t* h); diff --git a/src/crypto/test/crypto.cpp b/src/crypto/test/crypto.cpp index 0f762417ba2f..2bbf7e0d8568 100644 --- a/src/crypto/test/crypto.cpp +++ b/src/crypto/test/crypto.cpp @@ -877,4 +877,48 @@ TEST_CASE("PEM to JWK and back") REQUIRE(jwk == jwk2); } } +} + +TEST_CASE("Incremental hash") +{ + auto simple_hash = crypto::Sha256Hash(contents); + + INFO("Incremental hash"); + { + INFO("Finalise before any update"); + { + auto ihash = make_incremental_sha256(); + auto final_hash = ihash->finalise(); + REQUIRE(final_hash != simple_hash); + } + + INFO("Update one by one"); + { + auto ihash = make_incremental_sha256(); + for (auto const& c : contents) + { + ihash->update(c); + } + auto final_hash = ihash->finalise(); + REQUIRE(final_hash == simple_hash); + + REQUIRE_THROWS_AS(ihash->finalise(), std::logic_error); + } + + INFO("Update in large chunks"); + { + constexpr size_t chunk_size = 10; + auto ihash = make_incremental_sha256(); + for (auto it = contents.begin(); it < contents.end(); it += chunk_size) + { + auto end = + it + chunk_size > contents.end() ? contents.end() : it + chunk_size; + ihash->update(std::vector{it, end}); + } + auto final_hash = ihash->finalise(); + REQUIRE(final_hash == simple_hash); + + REQUIRE_THROWS_AS(ihash->finalise(), std::logic_error); + } + } } \ No newline at end of file From f791250c71bf222ae3f369d3de558b788151a1a0 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:16:49 +0100 Subject: [PATCH 018/135] [release/4.x] Cherry pick: Remove `/opt/ccf` install from runtime images (#5346) (#5358) --- .../setup_vm/roles/ccf_install/tasks/deb_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getting_started/setup_vm/roles/ccf_install/tasks/deb_install.yml b/getting_started/setup_vm/roles/ccf_install/tasks/deb_install.yml index 30551653937e..3a26f6df6331 100644 --- a/getting_started/setup_vm/roles/ccf_install/tasks/deb_install.yml +++ b/getting_started/setup_vm/roles/ccf_install/tasks/deb_install.yml @@ -59,7 +59,7 @@ - name: Remove release apt: - name: ccf + name: "ccf_{{ platform }}" state: absent become: true when: run_only|bool From 199b87d1d69e2b25df47045d04667fbeaa4528fe Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 16 Jun 2023 10:48:56 +0100 Subject: [PATCH 019/135] [release/4.x] Cherry pick: Documentation: Point from C++ crypto page to JS crypto API (#5364) (#5368) --- doc/build_apps/crypto.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/build_apps/crypto.rst b/doc/build_apps/crypto.rst index c5980506ff70..8d6b4512ab8c 100644 --- a/doc/build_apps/crypto.rst +++ b/doc/build_apps/crypto.rst @@ -3,6 +3,7 @@ Cryptography API For convenience, CCF provides access to commonly used cryptographic primitives to applications. +.. note:: This page describes the C++ API. For the API for TypeScript/JavaScript applications, see :typedoc:module:`ccf-app/crypto`. Hashing ------- From 02ec191df71f45258d230cf6dd596a609f4d7819 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Fri, 16 Jun 2023 11:45:37 +0100 Subject: [PATCH 020/135] [release/4.x] Cherry pick: Ensure e2e HTTP limit args are passed all the way through to the nodes (#5348) (#5366) --- tests/infra/e2e_args.py | 13 +++------- tests/infra/interfaces.py | 52 +++++++++++++++++++-------------------- tests/infra/network.py | 8 ++++++ tests/infra/node.py | 2 +- tests/infra/remote.py | 5 +++- tests/start_network.py | 14 ++++++++--- 6 files changed, 52 insertions(+), 42 deletions(-) diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index 3c79b56762ec..eb004f92cc3e 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -22,16 +22,8 @@ def nodes(args, n): return [ infra.interfaces.HostSpec( rpc_interfaces={ - infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface( - max_open_sessions_soft=args.max_open_sessions, - max_open_sessions_hard=args.max_open_sessions_hard, - max_http_body_size=args.max_http_body_size, - max_http_header_size=args.max_http_header_size, - max_http_headers_count=args.max_http_headers_count, - forwarding_timeout_ms=args.forwarding_timeout_ms, - app_protocol=infra.interfaces.AppProtocol.HTTP2 - if args.http2 - else infra.interfaces.AppProtocol.HTTP1, + infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface.from_args( + args ) } ) @@ -371,6 +363,7 @@ def cli_args(add=lambda x: None, parser=None, accept_unknown=False): "--max-http-headers-count", help="Maximum number of headers in single HTTP request", default=256, + type=int, ) parser.add_argument( "--http2", diff --git a/tests/infra/interfaces.py b/tests/infra/interfaces.py index ab3f54928f08..98a29831988e 100644 --- a/tests/infra/interfaces.py +++ b/tests/infra/interfaces.py @@ -104,6 +104,32 @@ class RPCInterface(Interface): forwarding_timeout_ms: Optional[int] = None app_protocol: AppProtocol = AppProtocol.HTTP1 + @staticmethod + def from_args(args): + return RPCInterface( + max_open_sessions_soft=args.max_open_sessions, + max_open_sessions_hard=args.max_open_sessions_hard, + max_http_body_size=args.max_http_body_size, + max_http_header_size=args.max_http_header_size, + max_http_headers_count=args.max_http_headers_count, + forwarding_timeout_ms=args.forwarding_timeout_ms, + app_protocol=AppProtocol.HTTP2 if args.http2 else AppProtocol.HTTP1, + ) + + def parse_from_str(self, s): + # Format: local|ssh(,tcp|udp)://hostname:port + + self.protocol, address = s.split("://") + self.transport = DEFAULT_TRANSPORT_PROTOCOL + if "," in self.protocol: + self.protocol, self.transport = self.protocol.split(",") + + if "," in address: + address, published_address = address.split(",") + self.public_host, self.public_port = split_netloc(published_address) + + self.host, self.port = split_netloc(address) + @staticmethod def to_json(interface): http_config = { @@ -194,29 +220,3 @@ def from_json(rpc_interfaces_json): for name, rpc_interface in rpc_interfaces_json.items() } ) - - @staticmethod - def from_str(s, http2=False): - # Format: local|ssh(,tcp|udp)://hostname:port - protocol, address = s.split("://") - transport = DEFAULT_TRANSPORT_PROTOCOL - if "," in protocol: - protocol, transport = protocol.split(",") - pub_host, pub_port = None, None - if "," in address: - address, published_address = address.split(",") - pub_host, pub_port = split_netloc(published_address) - host, port = split_netloc(address) - return HostSpec( - rpc_interfaces={ - PRIMARY_RPC_INTERFACE: RPCInterface( - protocol=protocol, - transport=transport, - host=host, - port=port, - public_host=pub_host, - public_port=pub_port, - app_protocol=AppProtocol.HTTP2 if http2 else AppProtocol.HTTP1, - ) - } - ) diff --git a/tests/infra/network.py b/tests/infra/network.py index 9eb847fbb727..fc983d89d2e6 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -291,6 +291,14 @@ def create_node(self, host, binary_dir=None, library_dir=None, **kwargs): perf = ( (str(node_id) in self.perf_nodes) if self.perf_nodes is not None else False ) + + if isinstance(host, str): + interface = infra.interfaces.RPCInterface() + interface.parse_from_str(host) + host = infra.interfaces.HostSpec( + rpc_interfaces={infra.interfaces.PRIMARY_RPC_INTERFACE: interface} + ) + node = infra.node.Node( node_id, host, diff --git a/tests/infra/node.py b/tests/infra/node.py index b41129dd7cc3..8aba9640fdd2 100644 --- a/tests/infra/node.py +++ b/tests/infra/node.py @@ -152,7 +152,7 @@ def __init__( requires_docker_remote = nodes_in_container or os.getenv("CONTAINER_NODES") if isinstance(self.host, str): - self.host = infra.interfaces.HostSpec.from_str(self.host) + raise ValueError("Translate host to HostSpec before you get here") for interface_name, rpc_interface in self.host.rpc_interfaces.items(): # Main RPC interface determines remote implementation diff --git a/tests/infra/remote.py b/tests/infra/remote.py index 5349ecd4388f..e0bc6772d42b 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -791,7 +791,10 @@ def __init__( exe_files += [config_file] with open(config_file, "w", encoding="utf-8") as f: - f.write(output) + # Parse and re-emit output to produce consistently formatted (indented) JSON. + # This will also ensure the render produced valid JSON + j = json.loads(output) + json.dump(j, f, indent=2) exe_files += [self.BIN, enclave_file] + self.DEPS data_files += [self.ledger_dir] if self.ledger_dir else [] diff --git a/tests/start_network.py b/tests/start_network.py index bd986e5a1443..80287d8a6334 100644 --- a/tests/start_network.py +++ b/tests/start_network.py @@ -25,10 +25,16 @@ def run(args): ) ] else: - hosts = args.node or DEFAULT_NODES - hosts = [ - infra.interfaces.HostSpec.from_str(node, http2=args.http2) for node in hosts - ] + host_descs = args.node or DEFAULT_NODES + hosts = [] + for host_desc in host_descs: + interface = infra.interfaces.RPCInterface.from_args(args) + interface.parse_from_str(host_desc) + hosts.append( + infra.interfaces.HostSpec( + rpc_interfaces={infra.interfaces.PRIMARY_RPC_INTERFACE: interface} + ) + ) if not args.verbose: LOG.remove() From 58f1b17852360bd04dd6992df41543541de2578e Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 16 Jun 2023 14:41:00 +0100 Subject: [PATCH 021/135] [release/4.x] Cherry pick: Update Dockerfiles to use Ubuntu MCR mirror (#5330) (#5360) --- docker/app_dev | 6 +++--- docker/app_run | 6 +++--- docker/ccf_ci | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/app_dev b/docker/app_dev index 7bd6524df9db..2036852c356e 100644 --- a/docker/app_dev +++ b/docker/app_dev @@ -4,17 +4,17 @@ ARG platform=sgx # SGX -FROM ubuntu:20.04 AS base-sgx +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-sgx WORKDIR / COPY ./docker/sgx_deps_pin.sh / RUN ./sgx_deps_pin.sh && rm /sgx_deps_pin.sh # SNP -FROM ubuntu:20.04 AS base-snp +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-snp # Virtual -FROM ubuntu:20.04 AS base-virtual +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-virtual # Final dev image FROM base-${platform} AS final diff --git a/docker/app_run b/docker/app_run index b24ab3310e22..b87c6dfdb1c2 100644 --- a/docker/app_run +++ b/docker/app_run @@ -4,17 +4,17 @@ ARG platform=sgx # SGX -FROM ubuntu:20.04 AS base-sgx +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-sgx WORKDIR / COPY ./docker/sgx_deps_pin.sh / RUN ./sgx_deps_pin.sh && rm ./sgx_deps_pin.sh # SNP -FROM ubuntu:20.04 AS base-snp +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-snp # Virtual -FROM ubuntu:20.04 AS base-virtual +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-virtual # Final runtime image FROM base-${platform} AS final diff --git a/docker/ccf_ci b/docker/ccf_ci index 21becc94866e..466ea4a2a1ae 100644 --- a/docker/ccf_ci +++ b/docker/ccf_ci @@ -4,17 +4,17 @@ ARG platform=sgx # SGX -FROM ubuntu:20.04 AS base-sgx +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-sgx WORKDIR / COPY ./docker/sgx_deps_pin.sh / RUN ./sgx_deps_pin.sh && rm ./sgx_deps_pin.sh # SNP -FROM ubuntu:20.04 AS base-snp +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-snp # Virtual -FROM ubuntu:20.04 AS base-virtual +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 AS base-virtual # Final CCF CI image FROM base-${platform} AS final From 06d3a4d36df621cc06b98fbed9690196177fd7bc Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:59:34 +0100 Subject: [PATCH 022/135] [release/4.x] Cherry pick: Minor OpenSSL 3.x compatibility fixes (#5373) (#5377) --- src/crypto/test/crypto.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crypto/test/crypto.cpp b/src/crypto/test/crypto.cpp index 2bbf7e0d8568..edf346aeca34 100644 --- a/src/crypto/test/crypto.cpp +++ b/src/crypto/test/crypto.cpp @@ -264,9 +264,9 @@ TEST_CASE("Manually hash, sign, verify, with certificate") auto cert = generate_self_signed_cert(kp, "CN=name"); auto verifier = make_verifier(cert); - CHECK(verifier->verify_hash(hash, signature)); + CHECK(verifier->verify_hash(hash, signature, MDType::SHA256)); corrupt(hash); - CHECK_FALSE(verifier->verify(hash, signature)); + CHECK_FALSE(verifier->verify(hash, signature, MDType::SHA256)); } } @@ -588,7 +588,7 @@ TEST_CASE("ExtendedIv0") std::iota(plain.begin(), plain.end(), 0); // test large IV - using LargeIVGcmHeader = FixedSizeGcmHeader<1234>; + using LargeIVGcmHeader = FixedSizeGcmHeader<128>; LargeIVGcmHeader h; SUBCASE("Null IV") {} From 769b68a2b5e3617aa064be778017ba26e28a6eec Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 16 Jun 2023 19:21:31 +0100 Subject: [PATCH 023/135] [release/4.x] Cherry pick: Update error message for conflicting n2n published address (#5370) (#5381) --- doc/host_config_schema/cchost_config.json | 2 +- src/node/rpc/node_frontend.h | 8 ++++---- src/node/rpc/test/node_frontend_test.cpp | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json index 3e58f9141452..26bdd6254808 100644 --- a/doc/host_config_schema/cchost_config.json +++ b/doc/host_config_schema/cchost_config.json @@ -41,7 +41,7 @@ "published_address": { "type": "string", "default": "Value of 'bind_address'", - "description": "The published node address advertised to other nodes" + "description": "The published node address advertised to other nodes. This must be different on each node" } }, "description": "Address (host:port) to listen on for incoming node-to-node connections (e.g. internal consensus messages)", diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 7560d5a416e2..8d4097a93277 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -230,8 +230,8 @@ namespace ccf nodes->foreach([&node_info_network, &duplicate_node_id]( const NodeId& nid, const NodeInfo& ni) { if ( - node_info_network.node_to_node_interface == - ni.node_to_node_interface && + node_info_network.node_to_node_interface.published_address == + ni.node_to_node_interface.published_address && ni.status != NodeStatus::RETIRED) { duplicate_node_id = nid; @@ -268,9 +268,9 @@ namespace ccf HTTP_STATUS_BAD_REQUEST, ccf::errors::NodeAlreadyExists, fmt::format( - "A node with the same node address {} already exists " + "A node with the same published node address {} already exists " "(node id: {}).", - in.node_info_network.node_to_node_interface.bind_address, + in.node_info_network.node_to_node_interface.published_address, conflicting_node_id.value())); } diff --git a/src/node/rpc/test/node_frontend_test.cpp b/src/node/rpc/test/node_frontend_test.cpp index b21819e37a7e..8776c800e2a4 100644 --- a/src/node/rpc/test/node_frontend_test.cpp +++ b/src/node/rpc/test/node_frontend_test.cpp @@ -175,7 +175,8 @@ TEST_CASE("Add a node to an opening service") frontend_process(frontend, join_input, "join", new_caller); check_error(http_response, HTTP_STATUS_BAD_REQUEST); - check_error_message(http_response, "A node with the same node address"); + check_error_message( + http_response, "A node with the same published node address"); } } @@ -258,7 +259,8 @@ TEST_CASE("Add a node to an open service") frontend_process(frontend, join_input, "join", new_caller); check_error(http_response, HTTP_STATUS_BAD_REQUEST); - check_error_message(http_response, "A node with the same node address"); + check_error_message( + http_response, "A node with the same published node address"); } INFO("Try to join again without being trusted"); From aca2a56130b474532edd18fc8c23ddb7a6a32ca4 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Mon, 19 Jun 2023 10:35:57 +0100 Subject: [PATCH 024/135] [release/4.x] Cherry pick: Do not decrement `match_idx` when processing NACKs (#5326) (#5362) --- src/consensus/aft/raft.h | 32 ++--- src/consensus/aft/raft_types.h | 6 +- src/consensus/aft/test/driver.cpp | 8 +- src/consensus/aft/test/driver.h | 115 +++++++++++++++--- tests/raft_scenarios/suffix_collision.2 | 9 ++ tests/raft_scenarios/suffix_collision.3 | 152 ++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 37 deletions(-) create mode 100644 tests/raft_scenarios/suffix_collision.3 diff --git a/src/consensus/aft/raft.h b/src/consensus/aft/raft.h index 47c2e8820fb6..231585ce9653 100644 --- a/src/consensus/aft/raft.h +++ b/src/consensus/aft/raft.h @@ -1415,31 +1415,35 @@ namespace aft } } - // Update next and match for the responding node. - auto& match_idx = node->second.match_idx; + // Update next or match for the responding node. if (r.success == AppendEntriesResponseType::FAIL) { - const auto this_match = - find_highest_possible_match({r.term, r.last_log_idx}); - if (match_idx == 0) - { - match_idx = this_match; - } - else - { - match_idx = std::min(match_idx, this_match); - } // Failed due to log inconsistency. Reset sent_idx, and try again soon. RAFT_DEBUG_FMT( "Recv append entries response to {} from {}: failed", state->node_id, from); - node->second.sent_idx = node->second.match_idx; + const auto this_match = + find_highest_possible_match({r.term, r.last_log_idx}); + node->second.sent_idx = std::min(this_match, node->second.sent_idx); return; } else { - match_idx = std::min(r.last_log_idx, state->last_idx); + // Potentially unnecessary safety check - use min with last_idx, to + // prevent matches past this node's local knowledge + const auto proposed_match = std::min(r.last_log_idx, state->last_idx); + if (proposed_match < node->second.match_idx) + { + RAFT_FAIL_FMT( + "Append entries response to {} from {} attempting to move " + "match_idx backwards ({} -> {})", + state->node_id, + from, + node->second.match_idx, + proposed_match); + } + node->second.match_idx = proposed_match; } RAFT_DEBUG_FMT( diff --git a/src/consensus/aft/raft_types.h b/src/consensus/aft/raft_types.h index e97f0c4e5c3a..8284981611fb 100644 --- a/src/consensus/aft/raft_types.h +++ b/src/consensus/aft/raft_types.h @@ -158,8 +158,10 @@ namespace aft // exception is in a rejection because of a mismatching suffix, in which // case this describes the latest point that the local node believes may // still match the leader's (which may be from an old term!). In either case - // this can be treated as the latest possible matching index for this - // follower. + // this can be treated as the _latest possible_ matching index for this + // follower. Note this may still be higher than the true match index, so + // should not affect the leader's opinion of how matched the follower is. + // Only a positive response should modify match_idx. Term term; Index last_log_idx; AppendEntriesResponseType success; diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index d5b6289f840c..2ee47f24a102 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -165,6 +165,10 @@ int main(int argc, char** argv) assert(items.size() == 1); driver->assert_state_sync(lineno); break; + case shash("assert_commit_safety"): + assert(items.size() == 2); + driver->assert_commit_safety(items[1], lineno); + break; case shash("assert_is_backup"): assert(items.size() == 2); driver->assert_is_backup(items[1], lineno); @@ -215,8 +219,8 @@ int main(int argc, char** argv) // Ignore empty lines break; default: - cerr << "Unknown action '" << items[0] << "' at line " << lineno - << endl; + throw std::runtime_error( + fmt::format("Unknown action '{}' at line {}", items[0], lineno)); } ++lineno; } diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 1a26da773de8..55afec81629f 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -822,9 +822,6 @@ class RaftDriver const auto target_last_idx = target_raft->get_last_idx(); const auto target_commit_idx = target_raft->get_committed_seqno(); - const auto target_final_entry = - target_raft->ledger->get_entry_by_idx(target_last_idx); - bool all_match = true; for (auto it = std::next(_nodes.begin()); it != _nodes.end(); ++it) { @@ -858,23 +855,46 @@ class RaftDriver } else { - // Check that the final entries are the same, assume prior entries also - // match - const auto final_entry = - raft->ledger->get_entry_by_idx(target_last_idx); - - if (final_entry != target_final_entry) + // Check that the every ledger entry matches + for (auto idx = 1; idx <= target_last_idx; ++idx) { - RAFT_DRIVER_OUT << fmt::format( - " Note over {}: Final entry at index {} " - "doesn't match entry on {}: {} != {}", - node_id, - target_last_idx, - target_id, - stringify(final_entry), - stringify(target_final_entry)) - << std::endl; - all_match = false; + const auto target_entry = target_raft->ledger->get_entry_by_idx(idx); + if (!target_entry.has_value()) + { + RAFT_DRIVER_OUT + << fmt::format( + " Note over {}: Missing ledger entry at {}", target_id, idx) + << std::endl; + all_match = false; + break; + } + else + { + const auto entry = raft->ledger->get_entry_by_idx(idx); + if (!entry.has_value()) + { + RAFT_DRIVER_OUT + << fmt::format( + " Note over {}: Missing ledger entry at {}", node_id, idx) + << std::endl; + all_match = false; + break; + } + else if (entry != target_entry) + { + RAFT_DRIVER_OUT << fmt::format( + " Note over {}: Entry at index {} " + "doesn't match entry on {}: {} != {}", + node_id, + idx, + target_id, + stringify(entry.value()), + stringify(target_entry.value())) + << std::endl; + all_match = false; + break; + } + } } } @@ -899,6 +919,63 @@ class RaftDriver } } + void assert_commit_safety(ccf::NodeId node_id, const size_t lineno) + { + // Confirm that the index this node considers committed, is present on a + // majority of nodes (ledger matches exactly up to and including this + // seqno). + // Similar to the QuorumLogInv invariant from the TLA spec. + // NB: This currently assumes a single configuration, as it checks a quorum + // of all nodes rather than within each configuration + const auto& raft = _nodes.at(node_id).raft; + const auto committed_seqno = raft->get_committed_seqno(); + + auto get_ledger_prefix = [this](ccf::NodeId id, ccf::SeqNo seqno) { + std::vector> prefix; + auto& ledger = _nodes.at(id).raft->ledger; + for (auto i = 1; i <= seqno; ++i) + { + auto entry = ledger->get_entry_by_idx(i); + if (!entry.has_value()) + { + // Early-out! + return prefix; + } + prefix.push_back(entry.value()); + } + return prefix; + }; + const auto committed_prefix = get_ledger_prefix(node_id, committed_seqno); + + auto is_present = [&](const auto& it) { + const auto& [id, _] = it; + return get_ledger_prefix(id, committed_seqno) == committed_prefix; + }; + + const auto present_count = + std::count_if(_nodes.begin(), _nodes.end(), is_present); + + const auto quorum = (_nodes.size() / 2) + 1; + if (present_count < quorum) + { + RAFT_DRIVER_OUT << fmt::format( + " Note over {}: Node has advanced commit to {}, " + "yet this entry is only present on {}/{} nodes " + "(need at least {} for safety)", + node_id, + committed_seqno, + present_count, + _nodes.size(), + quorum) + << std::endl; + throw std::runtime_error(fmt::format( + "Node ({}) at unsafe commit idx ({}) on line {}", + node_id, + committed_seqno, + lineno)); + } + } + void assert_commit_idx( ccf::NodeId node_id, const std::string& idx_s, const size_t lineno) { diff --git a/tests/raft_scenarios/suffix_collision.2 b/tests/raft_scenarios/suffix_collision.2 index 1e0bc069dd34..38c6721ff735 100644 --- a/tests/raft_scenarios/suffix_collision.2 +++ b/tests/raft_scenarios/suffix_collision.2 @@ -1,4 +1,7 @@ +# 3 nodes nodes,0,1,2 + +# Fully connected connect,0,1 connect,1,2 connect,0,2 @@ -25,7 +28,9 @@ dispatch_all periodic_all,10 dispatch_all +# All nodes agree so far state_all +assert_state_sync # Node 0 is partitioned, and replicates more items disconnect,0,1 @@ -100,6 +105,9 @@ connect,0,2 periodic_all,10 dispatch_all state_all +assert_commit_safety,0 +assert_commit_safety,1 +assert_commit_safety,2 # Node 1 tries to bring node 0 back in line # But we get unlucky with timing, and node 1 continually follows up with a heartbeat before hearing the response @@ -115,5 +123,6 @@ dispatch_all periodic_all,10 periodic_one,1,10 dispatch_all + state_all assert_state_sync diff --git a/tests/raft_scenarios/suffix_collision.3 b/tests/raft_scenarios/suffix_collision.3 new file mode 100644 index 000000000000..911ff7c8f443 --- /dev/null +++ b/tests/raft_scenarios/suffix_collision.3 @@ -0,0 +1,152 @@ +# 5 initial nodes +nodes,0,1,2,3,4 + +# fully connected +connect,0,1 +connect,0,2 +connect,0,3 +connect,0,4 +connect,1,2 +connect,1,3 +connect,1,4 +connect,2,3 +connect,2,4 +connect,3,4 + +# Node 0 starts first, and begins sending messages first +periodic_one,0,110 +dispatch_all + +periodic_all,10 +dispatch_all + +state_all + +replicate,1,hello world +emit_signature,1 +periodic_all,10 +dispatch_all + +replicate,1,saluton mondo +emit_signature,1 +periodic_all,10 +dispatch_all + +periodic_all,10 +dispatch_all + +# All nodes are in-sync +state_all +assert_state_sync + +# Nodes 0 and 1 are partitioned from the majority +disconnect,0,2 +disconnect,0,3 +disconnect,0,4 +disconnect,1,2 +disconnect,1,3 +disconnect,1,4 + +# Node 2 becomes primary, with votes from 3 and 4 +periodic_one,2,110 +dispatch_all + +periodic_all,10 +dispatch_all + +# Node 0 continues to produce a suffix in term 1 +replicate,1,branch a 0 +emit_signature,1 + +replicate,1,branch a 1 +emit_signature,1 + +replicate,1,branch a 2 +emit_signature,1 + +# Node 2 continues to produce a suffix in term 2 +replicate,2,branch b 0 +emit_signature,2 + +replicate,2,branch b 1 +emit_signature,2 + +replicate,2,branch b 2 +emit_signature,2 + +replicate,2,branch b 3 +emit_signature,2 + +# AEs describing these suffices are produced, but lost in transit +periodic_all,10 +drop_pending,0 +drop_pending,2 +dispatch_all + +# Node 0 calls a new election, hears from node 4 just long enough to become primary in term 3 +periodic_one,0,110 +reconnect,0,4 +periodic_one,0,110 +dispatch_all +periodic_one,0,110 +dispatch_all +disconnect,0,4 + +# Node 2 calls a new election, hears from node 4 just long enough to become primary in term 4 +periodic_one,2,110 +reconnect,2,4 +periodic_one,2,110 +dispatch_all +periodic_one,2,110 +dispatch_all +disconnect,2,4 + +# Node 0 continues to produce a suffix in term 3 +replicate,3,branch a 10 +emit_signature,3 + +replicate,3,branch a 11 +emit_signature,3 + +replicate,3,branch a 12 +emit_signature,3 + +# Node 2 continues to produce a suffix in term 4 +replicate,4,branch b 10 +emit_signature,4 + +replicate,4,branch b 11 +emit_signature,4 + +replicate,4,branch b 12 +emit_signature,4 + +# AEs describing these suffices are produced, and delivered, +# but due to partition they cannot be committed +state_all + +periodic_all,10 +dispatch_all + +periodic_all,10 +dispatch_all + +# Connection between 0 and 2 is restored +reconnect,0,2 + +# Node 0 hears about 2's progress (and later term), rolls back, +# produces a probe NACK +periodic_one,2,10 +dispatch_one,2 + +# Node 2 receives NACK from node 0 +dispatch_one,0 + +# Node 2 receives ACK from node 3 +dispatch_one,3 + +# Debug print current state +state_all + +# It must not advance commit based on the NACK index from node 0! +assert_commit_safety,2 From d7cf63486e68fccb23557bbda98a39d3b49a46db Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:43:25 +0100 Subject: [PATCH 025/135] [release/4.x] Cherry pick: Do not start if a PID file is present (#5361) (#5392) --- CHANGELOG.md | 6 ++++++ src/host/main.cpp | 8 ++++++++ tests/e2e_operations.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b1bdce08a4..95771fc1bcdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.3] + +[4.0.3]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.3 + +- If a pid file path is configured, `cchost` will no longer start if a file is present at that path. + ## [4.0.2] [4.0.2]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.2 diff --git a/src/host/main.cpp b/src/host/main.cpp index 303683ecdd00..bfde85ad50fd 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -206,6 +206,14 @@ int main(int argc, char** argv) return static_cast(CLI::ExitCodes::ValidationError); } + std::filesystem::path pid_file_path{config.output_files.pid_file}; + if (std::filesystem::exists(pid_file_path)) + { + LOG_FATAL_FMT( + "PID file {} already exists. Exiting.", pid_file_path.string()); + return static_cast(CLI::ExitCodes::FileError); + } + // Write PID to disk files::dump(fmt::format("{}", ::getpid()), config.output_files.pid_file); diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 19230d56b791..1a3fcb70f32d 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -16,6 +16,7 @@ import infra.proc import random import json +import time from loguru import logger as LOG @@ -375,7 +376,43 @@ def run_configuration_file_checks(args): assert rc == 0, f"Failed to check configuration: {rc}" +def run_pid_file_check(args): + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + args.perf_nodes, + pdb=args.pdb, + ) as network: + args.common_read_only_ledger_dir = None # Reset from previous test + network.start_and_open(args) + LOG.info("Check that pid file exists") + node = network.nodes[0] + node.stop() + # Delete ledger directory, since that too would prevent a restart + shutil.rmtree( + os.path.join(node.remote.remote.root, node.remote.ledger_dir_name) + ) + node.remote.start() + timeout = 10 + start = time.time() + LOG.info("Wait for node to shut down") + while time.time() - start < timeout: + if node.remote.check_done(): + break + time.sleep(0.1) + out, _ = node.remote.get_logs() + with open(out, "r") as outf: + last_line = outf.readlines()[-1].strip() + assert last_line.endswith( + "PID file node.pid already exists. Exiting." + ), last_line + LOG.info("Node shut down for the right reason") + network.ignoring_shutdown_errors = True + + def run(args): run_file_operations(args) run_tls_san_checks(args) run_configuration_file_checks(args) + run_pid_file_check(args) From 5a3aafbcbba358dc9c0c4d90584fd459e4a7d509 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 22 Jun 2023 17:23:25 +0100 Subject: [PATCH 026/135] [release/4.x] Cherry pick: Avoid taking too-early dependency on `previous_signature_seqno` (#5389) (#5393) --- src/kv/kv_types.h | 2 +- src/node/history.h | 30 +++++++----------------------- src/node/test/snapshotter.cpp | 4 ++-- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index e6b8f912a291..86bdfae30667 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -430,7 +430,7 @@ namespace kv const kv::TxID& tx_id, kv::Term term_of_next_version_) = 0; virtual void compact(Version v) = 0; virtual void set_term(kv::Term) = 0; - virtual std::vector serialise_tree(size_t from, size_t to) = 0; + virtual std::vector serialise_tree(size_t to) = 0; virtual void set_endorsed_certificate(const crypto::Pem& cert) = 0; virtual void start_signature_emit_timer() = 0; }; diff --git a/src/node/history.h b/src/node/history.h index 44c3f6d06bdc..1f9103854d89 100644 --- a/src/node/history.h +++ b/src/node/history.h @@ -211,7 +211,7 @@ namespace ccf return true; } - std::vector serialise_tree(size_t, size_t) override + std::vector serialise_tree(size_t) override { return {}; } @@ -285,7 +285,6 @@ namespace ccf class MerkleTreeHistoryPendingTx : public kv::PendingTx { kv::TxID txid; - ccf::SeqNo previous_signature_seqno; kv::Store& store; kv::TxHistory& history; NodeId id; @@ -295,14 +294,12 @@ namespace ccf public: MerkleTreeHistoryPendingTx( kv::TxID txid_, - ccf::SeqNo previous_signature_seqno_, kv::Store& store_, kv::TxHistory& history_, const NodeId& id_, crypto::KeyPair& kp_, crypto::Pem& endorsed_cert_) : txid(txid_), - previous_signature_seqno(previous_signature_seqno_), store(store_), history(history_), id(id_), @@ -320,7 +317,6 @@ namespace ccf crypto::Sha256Hash root = history.get_replicated_state_root(); std::vector primary_sig; - auto consensus = store.get_consensus(); primary_sig = kp.sign_hash(root.h.data(), root.h.size()); @@ -334,8 +330,7 @@ namespace ccf endorsed_cert); signatures->put(sig_value); - serialised_tree->put( - history.serialise_tree(previous_signature_seqno, txid.version - 1)); + serialised_tree->put(history.serialise_tree(txid.version - 1)); return sig.commit_reserved(); } }; @@ -679,10 +674,11 @@ namespace ccf return verify_node_signature(tx, sig_value.node, sig_value.sig, root); } - std::vector serialise_tree(size_t from, size_t to) override + std::vector serialise_tree(size_t to) override { std::lock_guard guard(state_lock); - return replicated_state_tree.serialise(from, to); + return replicated_state_tree.serialise( + replicated_state_tree.begin_index(), to); } void set_term(kv::Term t) override @@ -749,26 +745,14 @@ namespace ccf fmt::format("No endorsed certificate set to emit signature")); } - auto previous_signature_seqno = - consensus->get_previous_committable_seqno(); auto txid = store.next_txid(); - LOG_DEBUG_FMT( - "Signed at {} in view: {}, previous signature was at {}", - txid.version, - txid.term, - previous_signature_seqno); + LOG_DEBUG_FMT("Signed at {} in view: {}", txid.version, txid.term); store.commit( txid, std::make_unique>( - txid, - previous_signature_seqno, - store, - *this, - id, - kp, - endorsed_cert.value()), + txid, store, *this, id, kp, endorsed_cert.value()), true); } diff --git a/src/node/test/snapshotter.cpp b/src/node/test/snapshotter.cpp index 8ea03f113185..5c48d2ebdda0 100644 --- a/src/node/test/snapshotter.cpp +++ b/src/node/test/snapshotter.cpp @@ -118,7 +118,7 @@ bool record_signature( bool requires_snapshot = snapshotter->record_committable(idx); snapshotter->record_signature( idx, dummy_signature, kv::test::PrimaryNodeId, node_cert); - snapshotter->record_serialised_tree(idx, history->serialise_tree(1, idx)); + snapshotter->record_serialised_tree(idx, history->serialise_tree(idx)); return requires_snapshot; } @@ -464,7 +464,7 @@ TEST_CASE("Rekey ledger while snapshot is in progress") auto trees = tx.rw(ccf::Tables::SERIALISED_MERKLE_TREE); sigs->put({kv::test::PrimaryNodeId, 0, 0, {}, {}, {}, {}}); - auto tree = history->serialise_tree(1, snapshot_idx - 1); + auto tree = history->serialise_tree(snapshot_idx - 1); trees->put(tree); tx.commit(); From 959ed5de32c1cf17cae557f461cf0980c1891301 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 23 Jun 2023 18:23:48 +0100 Subject: [PATCH 027/135] [release/4.x] Cherry pick: Add config timeout option (#5379) (#5397) --- CHANGELOG.md | 1 + src/host/main.cpp | 45 +++++++++++++++++++++++++++++++---------- tests/e2e_operations.py | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95771fc1bcdd..3257c27f9d94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.3]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.3 +- User can now pass a `--config-timeout` option to `cchost` on startup. For example, a user wanting to start a `cchost` that may need to wait up 10 seconds for a valid config to appear under `/cfg/path` can invoke `./cchost --config-timeout 10s --config /path/cfg`. - If a pid file path is configured, `cchost` will no longer start if a file is present at that path. ## [4.0.2] diff --git a/src/host/main.cpp b/src/host/main.cpp index bfde85ad50fd..2e4d0cb40ed5 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -81,10 +81,14 @@ int main(int argc, char** argv) CLI::App app{"ccf"}; std::string config_file_path = "config.json"; - app - .add_option( - "-c,--config", config_file_path, "Path to JSON configuration file") - ->check(CLI::ExistingFile); + app.add_option( + "-c,--config", config_file_path, "Path to JSON configuration file"); + + ds::TimeString config_timeout = {"0s"}; + app.add_option( + "--config-timeout", + config_timeout, + "Configuration file read timeout, for example 5s or 1min"); bool check_config_only = false; app.add_flag( @@ -102,16 +106,35 @@ int main(int argc, char** argv) return app.exit(e); } - std::string config_str = files::slurp_string(config_file_path); + std::string config_str = files::slurp_string( + config_file_path, + true /* return an empty string if the file does not exist */); nlohmann::json config_json; - try + auto config_timeout_end = std::chrono::high_resolution_clock::now() + + std::chrono::microseconds(config_timeout); + std::string config_parsing_error = ""; + do { - config_json = nlohmann::json::parse(config_str); - } - catch (const std::exception& e) + std::string config_str = files::slurp_string( + config_file_path, + true /* return an empty string if the file does not exist */); + try + { + config_json = nlohmann::json::parse(config_str); + config_parsing_error = ""; + break; + } + catch (const std::exception& e) + { + config_parsing_error = fmt::format( + "Error parsing configuration file {}: {}", config_file_path, e.what()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } while (std::chrono::high_resolution_clock::now() < config_timeout_end); + + if (!config_parsing_error.empty()) { - throw std::logic_error(fmt::format( - "Error parsing configuration file {}: {}", config_file_path, e.what())); + throw std::logic_error(config_parsing_error); } auto schema_json = nlohmann::json::parse(host::host_config_schema); diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 1a3fcb70f32d..ce4433243062 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -16,6 +16,7 @@ import infra.proc import random import json +import subprocess import time from loguru import logger as LOG @@ -354,6 +355,45 @@ def run_tls_san_checks(args): assert len(sans) == 1, "Expected exactly one SAN" assert sans[0].value == ipaddress.ip_address(dummy_public_rpc_host) + # This is relatively direct test to make sure the config timeout feature + # works as intended. It is difficult to do with the existing framework + # as is because of the indirections and the fact that start() is a + # synchronous call. + start_node_path = network.nodes[0].remote.remote.root + # Remove ledger and pid file to allow a restart + shutil.rmtree(os.path.join(start_node_path, "0.ledger")) + os.remove(os.path.join(start_node_path, "node.pid")) + os.remove(os.path.join(start_node_path, "service_cert.pem")) + # Move configuration + shutil.move( + os.path.join(start_node_path, "0.config.json"), + os.path.join(start_node_path, "0.config.json.bak"), + ) + LOG.info("No config at all") + assert not os.path.exists(os.path.join(start_node_path, "0.config.json")) + LOG.info(f"Attempt to start node without a config under {start_node_path}") + proc = subprocess.Popen( + ["./cchost", "--config", "0.config.json", "--config-timeout", "10s"], + cwd=start_node_path, + ) + time.sleep(2) + LOG.info("Copy a partial config") + # Replace it with a prefix + with open(os.path.join(start_node_path, "0.config.json"), "w") as f: + f.write("{") + time.sleep(2) + LOG.info("Move a full config back") + shutil.copy( + os.path.join(start_node_path, "0.config.json.bak"), + os.path.join(start_node_path, "0.config.json"), + ) + time.sleep(10) + LOG.info("Wait out the rest of the timeout") + assert proc.poll() is None, "Node process should still be running" + assert os.path.exists(os.path.join(start_node_path, "service_cert.pem")) + proc.terminate() + proc.wait() + def run_configuration_file_checks(args): LOG.info( From 92d08bc4f913c991fdb81176103f66cdbbc9d68b Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:59:18 +0100 Subject: [PATCH 028/135] [release/4.x] Cherry pick: Update TypeScript to expose COSE authentication policies (#5403) (#5404) --- CHANGELOG.md | 6 +++++ doc/build_apps/api.rst | 18 ++++++++++++++ include/ccf/endpoint.h | 3 +++ js/ccf-app/src/endpoints.ts | 14 ++++++++++- src/apps/js_generic/named_auth_policies.h | 4 ++++ tests/js-modules/modules.py | 24 +++++++++++++++++++ tests/npm-app/app.json | 29 +++++++++++++++++++++++ tests/npm-app/src/endpoints/all.ts | 1 + tests/npm-app/src/endpoints/auth.ts | 19 +++++++++++++++ 9 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/npm-app/src/endpoints/auth.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3257c27f9d94..c98ad0688d51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.4] + +[4.0.4]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.4 + +- Added TypeScript interfaces `UserCOSESign1AuthnIdentity` and `MemberCOSESign1AuthnIdentity`, to be used with `user_cose_sign1` and `member_cose_sign1` authentication policies. + ## [4.0.3] [4.0.3]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.3 diff --git a/doc/build_apps/api.rst b/doc/build_apps/api.rst index 4c3a1e60ec5b..b541988fa8b6 100644 --- a/doc/build_apps/api.rst +++ b/doc/build_apps/api.rst @@ -60,6 +60,12 @@ Policies .. doxygenvariable:: ccf::user_cert_auth_policy :project: CCF +.. doxygenvariable:: ccf::member_cert_auth_policy + :project: CCF + +.. doxygenvariable:: ccf::member_cose_sign1_auth_policy + :project: CCF + .. doxygenvariable:: ccf::user_cose_sign1_auth_policy :project: CCF @@ -73,6 +79,18 @@ Identities :project: CCF :members: +.. doxygenstruct:: ccf::MemberCertAuthnIdentity + :project: CCF + :members: + +.. doxygenstruct:: ccf::UserCOSESign1AuthnIdentity + :project: CCF + :members: + +.. doxygenstruct:: ccf::MemberCOSESign1AuthnIdentity + :project: CCF + :members: + .. doxygenstruct:: ccf::JwtAuthnIdentity :project: CCF :members: diff --git a/include/ccf/endpoint.h b/include/ccf/endpoint.h index 09026539b584..1203b21e1ea8 100644 --- a/include/ccf/endpoint.h +++ b/include/ccf/endpoint.h @@ -132,6 +132,9 @@ namespace ccf::endpoints * Identity of the caller to be used by the endpoint. This can be * retrieved inside the endpoint with ctx.get_caller(), * @see ccf::UserCertAuthnIdentity + * @see ccf::MemberCertAuthnIdentity + * @see ccf::UserCOSESign1tAuthnIdentity + * @see ccf::MemberCOSESign1AuthnIdentity * @see ccf::JwtAuthnIdentity * * @see ccf::empty_auth_policy diff --git a/js/ccf-app/src/endpoints.ts b/js/ccf-app/src/endpoints.ts index a44cce6f793f..6dad593b207c 100644 --- a/js/ccf-app/src/endpoints.ts +++ b/js/ccf-app/src/endpoints.ts @@ -141,6 +141,16 @@ export interface MemberCertAuthnIdentity extends UserMemberAuthnIdentityCommon { policy: "member_cert"; } +export interface MemberCOSESign1AuthnIdentity + extends UserMemberAuthnIdentityCommon { + policy: "member_cose_sign1"; +} + +export interface UserCOSESign1AuthnIdentity + extends UserMemberAuthnIdentityCommon { + policy: "user_cose_sign1"; +} + export interface JwtAuthnIdentity extends AuthnIdentityCommon { policy: "jwt"; @@ -175,7 +185,9 @@ export type AuthnIdentity = | EmptyAuthnIdentity | UserCertAuthnIdentity | MemberCertAuthnIdentity - | JwtAuthnIdentity; + | JwtAuthnIdentity + | MemberCOSESign1AuthnIdentity + | UserCOSESign1AuthnIdentity; /** See {@linkcode Response.body}. */ export type ResponseBodyType = string | ArrayBuffer | JsonCompatible; diff --git a/src/apps/js_generic/named_auth_policies.h b/src/apps/js_generic/named_auth_policies.h index a1b757fe7a9c..82f6b5a26f8b 100644 --- a/src/apps/js_generic/named_auth_policies.h +++ b/src/apps/js_generic/named_auth_policies.h @@ -70,6 +70,10 @@ namespace ccfapp { return ccf::UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME; } + else if constexpr (std::is_same_v) + { + return ccf::MemberCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME; + } else if constexpr (std::is_same_v) { return ccf::EmptyAuthnPolicy::SECURITY_SCHEME_NAME; diff --git a/tests/js-modules/modules.py b/tests/js-modules/modules.py index d3921db302e1..b945956e8d05 100644 --- a/tests/js-modules/modules.py +++ b/tests/js-modules/modules.py @@ -1077,6 +1077,29 @@ def test_js_exception_output(network, args): return network +@reqs.description("Test User Cose authentication") +def test_user_cose_authentication(network, args): + primary, _ = network.find_nodes() + + with primary.client() as c: + r = c.put("/app/cose", {}) + assert r.status_code == http.HTTPStatus.UNAUTHORIZED, r + + with primary.client("user0") as c: + r = c.put("/app/cose", {}) + assert r.status_code == http.HTTPStatus.UNAUTHORIZED, r + + with primary.client("user0", headers={"content-type": "application/cose"}) as c: + r = c.put("/app/cose", {}) + assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r + + with primary.client(None, None, "user0") as c: + r = c.put("/app/cose", {}) + assert r.status_code == http.HTTPStatus.OK, r + assert r.body.text() == network.users[0].service_id + return network + + def run(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb @@ -1090,6 +1113,7 @@ def run(args): network = test_npm_app(network, args) network = test_js_execution_time(network, args) network = test_js_exception_output(network, args) + network = test_user_cose_authentication(network, args) if __name__ == "__main__": diff --git a/tests/npm-app/app.json b/tests/npm-app/app.json index 8b13849225d5..6ca670991ce5 100644 --- a/tests/npm-app/app.json +++ b/tests/npm-app/app.json @@ -1284,6 +1284,35 @@ } } } + }, + "/cose": { + "put": { + "js_module": "endpoints/auth.js", + "js_function": "checkUserCOSESign1Auth", + "forwarding_required": "sometimes", + "authn_policies": ["user_cose_sign1"], + "mode": "readwrite", + "openapi": { + "requestBody": { + "required": true, + "content": { + "application/cose": { + "schema": {} + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } } } } diff --git a/tests/npm-app/src/endpoints/all.ts b/tests/npm-app/src/endpoints/all.ts index eaf126c954c6..6e8d782ba7d7 100644 --- a/tests/npm-app/src/endpoints/all.ts +++ b/tests/npm-app/src/endpoints/all.ts @@ -5,3 +5,4 @@ export * from "./partition"; export * from "./proto"; export * from "./log"; export * from "./rpc"; +export * from "./auth"; diff --git a/tests/npm-app/src/endpoints/auth.ts b/tests/npm-app/src/endpoints/auth.ts new file mode 100644 index 000000000000..c51f826b8356 --- /dev/null +++ b/tests/npm-app/src/endpoints/auth.ts @@ -0,0 +1,19 @@ +import * as ccfapp from "@microsoft/ccf-app"; + +// Note: this is also tested more generically on the multi_auth endpoint +// of the logging application, but not with TypeScript types. +export function checkUserCOSESign1Auth( + request: ccfapp.Request +): ccfapp.Response { + if (request.caller === null || request.caller === undefined) { + return { status: 401 }; + } + + const caller = request.caller; + if (caller.policy !== "user_cose_sign1") { + return { status: 401 }; + } + + const id: ccfapp.UserCOSESign1AuthnIdentity = caller; + return { status: 200, body: id.id }; +} From 4dbbd7918687a68cff848da02a73c50d2551d19c Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 4 Jul 2023 09:04:44 +0100 Subject: [PATCH 029/135] [release/4.x] Cherry pick: Fix unsafe inline assembly usage (#5408) (#5409) --- include/ccf/crypto/entropy.h | 62 ++++++++++++------------------------ 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/include/ccf/crypto/entropy.h b/include/ccf/crypto/entropy.h index a35f5a84f01f..e211d7a88121 100644 --- a/include/ccf/crypto/entropy.h +++ b/include/ccf/crypto/entropy.h @@ -83,48 +83,28 @@ namespace crypto return drng_features; } - // The attribute below prevents ASAN error - // (Internal ticket: https://github.com/microsoft/CCF/issues/5050). - // ASAN with Debug mode causes invalid memory access. - // These suppressions can be removed after - // https://github.com/google/sanitizers/issues/1629 is resolved. -#if defined(__has_feature) -# if __has_feature(address_sanitizer) - __attribute__((no_sanitize("address"))) -# endif -#endif - static int - rdrand16_step(uint16_t* rand) + static bool rdrand16_step(uint16_t* rand) { unsigned char ok; - asm volatile("rdrand %0; setc %1" : "=r"(*rand), "=qm"(ok)); - return (int)ok; + // @ccc allows placing a constraint on the carry flag ('@cc c') without + // having to write to a separate register, see + // https://gcc.gnu.org/onlinedocs/gcc/Extended-Asm.html for more details. + asm volatile("rdrand %0" : "=r"(*rand), "=@ccc"(ok)); + return ok; } -#if defined(__has_feature) -# if __has_feature(address_sanitizer) - __attribute__((no_sanitize("address"))) -# endif -#endif - static int - rdrand32_step(uint32_t* rand) + static bool rdrand32_step(uint32_t* rand) { unsigned char ok; - asm volatile("rdrand %0; setc %1" : "=r"(*rand), "=qm"(ok)); - return (int)ok; + asm volatile("rdrand %0" : "=r"(*rand), "=@ccc"(ok)); + return ok; } -#if defined(__has_feature) -# if __has_feature(address_sanitizer) - __attribute__((no_sanitize("address"))) -# endif -#endif - static int - rdrand64_step(uint64_t* rand) + static bool rdrand64_step(uint64_t* rand) { unsigned char ok; - asm volatile("rdrand %0; setc %1" : "=r"(*rand), "=qm"(ok)); - return (int)ok; + asm volatile("rdrand %0" : "=r"(*rand), "=@ccc"(ok)); + return ok; } static int rdrand16_retry(unsigned int retries, uint16_t* rand) @@ -237,25 +217,25 @@ namespace crypto // The following three functions should be used to generate // randomness that will be used as seed for another RNG - static int rdseed16_step(uint16_t* seed) + static bool rdseed16_step(uint16_t* seed) { unsigned char ok; - asm volatile("rdseed %0; setc %1" : "=r"(*seed), "=qm"(ok)); - return (int)ok; + asm volatile("rdseed %0" : "=r"(*seed), "=@ccc"(ok)); + return ok; } - static int rdseed32_step(uint32_t* seed) + static bool rdseed32_step(uint32_t* seed) { unsigned char ok; - asm volatile("rdseed %0; setc %1" : "=r"(*seed), "=qm"(ok)); - return (int)ok; + asm volatile("rdseed %0" : "=r"(*seed), "=@ccc"(ok)); + return ok; } - static int rdseed64_step(uint64_t* seed) + static bool rdseed64_step(uint64_t* seed) { unsigned char ok; - asm volatile("rdseed %0; setc %1" : "=r"(*seed), "=qm"(ok)); - return (int)ok; + asm volatile("rdseed %0" : "=r"(*seed), "=@ccc"(ok)); + return ok; } public: From 450ed9a4eb70b2d0e54d9fac42f88cb57c9c1a1a Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 7 Jul 2023 17:52:17 +0100 Subject: [PATCH 030/135] [release/4.x] Cherry pick: Move side-effects out of common.cmake and into CMakeLists.txt (#5287) (#5423) Co-authored-by: Amaury Chamayou --- .threading_canary | 2 +- CMakeLists.txt | 509 ++++++++++++++++++++++++++++++++++++++++++++ cmake/common.cmake | 511 --------------------------------------------- 3 files changed, 510 insertions(+), 512 deletions(-) diff --git a/.threading_canary b/.threading_canary index bc32cd175b55..635f3470ece8 100644 --- a/.threading_canary +++ b/.threading_canary @@ -1 +1 @@ -THIS looks like a job for Threading Canary!! +THIS looks like a job for Threading Canar!y!! diff --git a/CMakeLists.txt b/CMakeLists.txt index b4ebb6693db3..f5efab06bd72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,515 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/preproject.cmake include(GNUInstallDirs) +set(CMAKE_MODULE_PATH "${CCF_DIR}/cmake;${CMAKE_MODULE_PATH}") + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +find_package(Threads REQUIRED) + +function(message) + if(NOT MESSAGE_QUIET) + _message(${ARGN}) + endif() +endfunction() + +option(PROFILE_TESTS "Profile tests" OFF) +set(PYTHON unbuffer python3) + +set(DISTRIBUTE_PERF_TESTS + "" + CACHE + STRING + "Hosts to which performance tests should be distributed, for example -n ssh://x.x.x.x -n ssh://x.x.x.x -n ssh://x.x.x.x" +) + +if(DISTRIBUTE_PERF_TESTS) + separate_arguments(NODES UNIX_COMMAND ${DISTRIBUTE_PERF_TESTS}) +else() + unset(NODES) +endif() + +option(VERBOSE_LOGGING "Enable verbose, unsafe logging of enclave code" OFF) +set(TEST_HOST_LOGGING_LEVEL "info") +if(VERBOSE_LOGGING) + set(TEST_HOST_LOGGING_LEVEL "trace") + add_compile_definitions(VERBOSE_LOGGING) +endif() + +option(USE_NULL_ENCRYPTOR "Turn off encryption of ledger updates - debug only" + OFF +) +if(USE_NULL_ENCRYPTOR) + add_compile_definitions(USE_NULL_ENCRYPTOR) +endif() + +option(SAN "Enable Address and Undefined Behavior Sanitizers" OFF) +option(TSAN "Enable Thread Sanitizers" OFF) +option(BUILD_END_TO_END_TESTS "Build end to end tests" ON) +option(COVERAGE "Enable coverage mapping" OFF) +option(SHUFFLE_SUITE "Shuffle end to end test suite" OFF) +option(LONG_TESTS "Enable long end-to-end tests" OFF) +option(KV_STATE_RB "Enable RBMap as underlying KV state implementation" OFF) +if(KV_STATE_RB) + add_compile_definitions(KV_STATE_RB) +endif() + +# This option controls whether to link virtual builds against snmalloc rather +# than use the system allocator. In builds using Open Enclave, enclave +# allocation is managed separately and enabling snmalloc is done by linking +# openenclave::oesnmalloc +option(USE_SNMALLOC "Link virtual build against snmalloc" ON) + +enable_language(ASM) + +set(CCF_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated) +include_directories(${CCF_DIR}/include) +include_directories(${CCF_DIR}/src) + +set(CCF_3RD_PARTY_EXPORTED_DIR "${CCF_DIR}/3rdparty/exported") +set(CCF_3RD_PARTY_INTERNAL_DIR "${CCF_DIR}/3rdparty/internal") + +include_directories(SYSTEM ${CCF_3RD_PARTY_EXPORTED_DIR}) +include_directories(SYSTEM ${CCF_3RD_PARTY_INTERNAL_DIR}) + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tools.cmake) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/tools.cmake DESTINATION cmake) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccf_app.cmake) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccf_app.cmake DESTINATION cmake) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/open_enclave.cmake + DESTINATION cmake +) + +if(SAN AND LVI_MITIGATIONS) + message( + FATAL_ERROR + "Building with both SAN and LVI mitigations is unsafe and deadlocks - choose one" + ) +endif() + +if(TSAN AND LVI_MITIGATIONS) + message( + FATAL_ERROR + "Building with both TSAN and LVI mitigations is unsafe and deadlocks - choose one" + ) +endif() + +add_custom_command( + COMMAND + openenclave::oeedger8r ${CCF_DIR}/edl/ccf.edl --search-path ${OE_INCLUDEDIR} + --trusted --trusted-dir ${CCF_GENERATED_DIR} --untrusted --untrusted-dir + ${CCF_GENERATED_DIR} + COMMAND mv ${CCF_GENERATED_DIR}/ccf_t.c ${CCF_GENERATED_DIR}/ccf_t.cpp + COMMAND mv ${CCF_GENERATED_DIR}/ccf_u.c ${CCF_GENERATED_DIR}/ccf_u.cpp + DEPENDS ${CCF_DIR}/edl/ccf.edl + OUTPUT ${CCF_GENERATED_DIR}/ccf_t.cpp ${CCF_GENERATED_DIR}/ccf_u.cpp + COMMENT "Generating code from EDL, and renaming to .cpp" +) + +# Copy and install CCF utilities +set(CCF_UTILITIES keygenerator.sh scurl.sh submit_recovery_share.sh + verify_quote.sh +) +foreach(UTILITY ${CCF_UTILITIES}) + configure_file( + ${CCF_DIR}/python/utils/${UTILITY} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY + ) + install(PROGRAMS ${CCF_DIR}/python/utils/${UTILITY} DESTINATION bin) +endforeach() + +# Copy utilities from tests directory +set(CCF_TEST_UTILITIES + tests.sh + cimetrics_env.sh + upload_pico_metrics.py + test_install.sh + docker_wrap.sh + config.jinja + recovery_benchmark.sh +) +foreach(UTILITY ${CCF_TEST_UTILITIES}) + configure_file( + ${CCF_DIR}/tests/${UTILITY} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY + ) +endforeach() + +# Install additional utilities +install(PROGRAMS ${CCF_DIR}/samples/scripts/sgxinfo.sh DESTINATION bin) +install(PROGRAMS ${CCF_DIR}/samples/scripts/snpinfo.sh DESTINATION bin) +install(FILES ${CCF_DIR}/tests/config.jinja DESTINATION bin) + +if(SAN) + install(FILES ${CCF_DIR}/src/san_common.suppressions DESTINATION bin) +endif() + +# Install getting_started scripts for VM creation and setup +install( + DIRECTORY ${CCF_DIR}/getting_started/ + DESTINATION getting_started + USE_SOURCE_PERMISSIONS +) + +if(COMPILE_TARGET STREQUAL "sgx") + # While virtual libraries need to be built for sgx for unit tests, these do + # not get installed to minimise installation size + set(INSTALL_VIRTUAL_LIBRARIES OFF) + + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(DEFAULT_ENCLAVE_TYPE debug) + endif() +elseif(COMPILE_TARGET STREQUAL "snp") + set(INSTALL_VIRTUAL_LIBRARIES OFF) +else() + set(INSTALL_VIRTUAL_LIBRARIES ON) +endif() + +set(HTTP_PARSER_SOURCES + ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/api.c + ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/http.c + ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/llhttp.c +) + +set(CCF_ENDPOINTS_SOURCES + ${CCF_DIR}/src/endpoints/endpoint.cpp + ${CCF_DIR}/src/endpoints/endpoint_registry.cpp + ${CCF_DIR}/src/endpoints/base_endpoint_registry.cpp + ${CCF_DIR}/src/endpoints/common_endpoint_registry.cpp + ${CCF_DIR}/src/endpoints/json_handler.cpp + ${CCF_DIR}/src/endpoints/authentication/cose_auth.cpp + ${CCF_DIR}/src/endpoints/authentication/cert_auth.cpp + ${CCF_DIR}/src/endpoints/authentication/empty_auth.cpp + ${CCF_DIR}/src/endpoints/authentication/jwt_auth.cpp + ${CCF_DIR}/src/enclave/enclave_time.cpp + ${CCF_DIR}/src/indexing/strategies/seqnos_by_key_bucketed.cpp + ${CCF_DIR}/src/indexing/strategies/seqnos_by_key_in_memory.cpp + ${CCF_DIR}/src/indexing/strategies/visit_each_entry_in_map.cpp + ${CCF_DIR}/src/node/historical_queries_adapter.cpp + ${CCF_DIR}/src/node/historical_queries_utils.cpp + ${CCF_DIR}/src/node/receipt.cpp +) + +find_library(CRYPTO_LIBRARY crypto) +find_library(TLS_LIBRARY ssl) + +include(${CCF_DIR}/cmake/crypto.cmake) +include(${CCF_DIR}/cmake/quickjs.cmake) +include(${CCF_DIR}/cmake/sss.cmake) +include(${CCF_DIR}/cmake/nghttp2.cmake) +include(${CCF_DIR}/cmake/qcbor.cmake) +include(${CCF_DIR}/cmake/t_cose.cmake) +set(MESSAGE_QUIET ON) +include(${CCF_DIR}/cmake/protobuf.cmake) +unset(MESSAGE_QUIET) + +# Host Executable +if(SAN + OR TSAN + OR NOT USE_SNMALLOC +) + set(SNMALLOC_COMPILE_OPTIONS "") +else() + set(SNMALLOC_HEADER_ONLY_LIBRARY ON) + add_subdirectory(3rdparty/exported/snmalloc EXCLUDE_FROM_ALL) + set(SNMALLOC_COMPILE_OPTIONS "-mcx16") + list(APPEND CCHOST_SOURCES src/host/snmalloc.cpp) +endif() + +list(APPEND CCHOST_SOURCES ${CCF_DIR}/src/host/main.cpp) + +if(COMPILE_TARGET STREQUAL "sgx") + list(APPEND CCHOST_SOURCES ${CCF_GENERATED_DIR}/ccf_u.cpp) +endif() + +add_executable(cchost ${CCHOST_SOURCES}) + +add_warning_checks(cchost) +add_san(cchost) + +target_compile_options( + cchost PRIVATE ${COMPILE_LIBCXX} ${SNMALLOC_COMPILE_OPTIONS} +) +target_include_directories(cchost PRIVATE ${CCF_GENERATED_DIR}) + +# Host is always built with verbose logging enabled, regardless of CMake option +target_compile_definitions(cchost PRIVATE VERBOSE_LOGGING) + +if(COMPILE_TARGET STREQUAL "sgx") + target_compile_definitions(cchost PUBLIC PLATFORM_SGX) +elseif(COMPILE_TARGET STREQUAL "snp") + target_compile_definitions(cchost PUBLIC PLATFORM_SNP) +elseif(COMPILE_TARGET STREQUAL "virtual") + target_compile_definitions(cchost PUBLIC PLATFORM_VIRTUAL) +endif() + +target_link_libraries( + cchost PRIVATE uv ${TLS_LIBRARY} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} + ${LINK_LIBCXX} ccfcrypto.host +) +if(COMPILE_TARGET STREQUAL "sgx") + target_link_libraries(cchost PRIVATE openenclave::oehost) +endif() + +install(TARGETS cchost DESTINATION bin) + +# Perf scenario executable +add_executable( + scenario_perf_client ${CCF_DIR}/src/clients/perf/scenario_perf_client.cpp +) +target_link_libraries( + scenario_perf_client PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser.host + ccfcrypto.host +) +if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9) + target_link_libraries(scenario_perf_client PRIVATE c++fs) +endif() +install(TARGETS scenario_perf_client DESTINATION bin) + +# HTTP parser +if(COMPILE_TARGET STREQUAL "sgx") + add_enclave_library_c(http_parser.enclave "${HTTP_PARSER_SOURCES}") + install( + TARGETS http_parser.enclave + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "snp") + add_library(http_parser.snp "${HTTP_PARSER_SOURCES}") + set_property(TARGET http_parser.snp PROPERTY POSITION_INDEPENDENT_CODE ON) + install( + TARGETS http_parser.snp + EXPORT ccf + DESTINATION lib + ) +endif() + +add_library(http_parser.host "${HTTP_PARSER_SOURCES}") +set_property(TARGET http_parser.host PROPERTY POSITION_INDEPENDENT_CODE ON) +if(INSTALL_VIRTUAL_LIBRARIES) + install( + TARGETS http_parser.host + EXPORT ccf + DESTINATION lib + ) +endif() + +# CCF kv libs +set(CCF_KV_SOURCES + ${CCF_DIR}/src/kv/tx.cpp ${CCF_DIR}/src/kv/untyped_map_handle.cpp + ${CCF_DIR}/src/kv/untyped_map_diff.cpp +) + +if(COMPILE_TARGET STREQUAL "sgx") + add_enclave_library(ccf_kv.enclave "${CCF_KV_SOURCES}") + add_warning_checks(ccf_kv.enclave) + install( + TARGETS ccf_kv.enclave + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "snp") + add_host_library(ccf_kv.snp "${CCF_KV_SOURCES}") + add_san(ccf_kv.snp) + add_warning_checks(ccf_kv.snp) + install( + TARGETS ccf_kv.snp + EXPORT ccf + DESTINATION lib + ) +endif() + +add_host_library(ccf_kv.host "${CCF_KV_SOURCES}") +add_san(ccf_kv.host) +add_warning_checks(ccf_kv.host) +if(INSTALL_VIRTUAL_LIBRARIES) + install( + TARGETS ccf_kv.host + EXPORT ccf + DESTINATION lib + ) +endif() + +# CCF endpoints libs +if(COMPILE_TARGET STREQUAL "sgx") + add_enclave_library(ccf_endpoints.enclave "${CCF_ENDPOINTS_SOURCES}") + target_link_libraries( + ccf_endpoints.enclave + PUBLIC qcbor.enclave t_cose.enclave http_parser.enclave ccfcrypto.enclave + ccf_kv.enclave + ) + add_warning_checks(ccf_endpoints.enclave) + install( + TARGETS ccf_endpoints.enclave + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "snp") + add_host_library(ccf_endpoints.snp "${CCF_ENDPOINTS_SOURCES}") + target_link_libraries( + ccf_endpoints.snp PUBLIC qcbor.snp t_cose.snp http_parser.snp ccfcrypto.snp + ccf_kv.snp + ) + add_san(ccf_endpoints.snp) + add_warning_checks(ccf_endpoints.snp) + install( + TARGETS ccf_endpoints.snp + EXPORT ccf + DESTINATION lib + ) +endif() + +add_host_library(ccf_endpoints.host "${CCF_ENDPOINTS_SOURCES}") +target_link_libraries( + ccf_endpoints.host PUBLIC qcbor.host t_cose.host http_parser.host + ccfcrypto.host ccf_kv.host +) +add_san(ccf_endpoints.host) +add_warning_checks(ccf_endpoints.host) + +if(INSTALL_VIRTUAL_LIBRARIES) + install( + TARGETS ccf_endpoints.host + EXPORT ccf + DESTINATION lib + ) +endif() + +# Common test args for Python scripts starting up CCF networks +set(WORKER_THREADS + 0 + CACHE STRING "Number of worker threads to start on each CCF node" +) + +set(CCF_NETWORK_TEST_DEFAULT_CONSTITUTION + --constitution + ${CCF_DIR}/samples/constitutions/default/actions.js + --constitution + ${CCF_DIR}/samples/constitutions/default/validate.js + --constitution + ${CCF_DIR}/samples/constitutions/default/resolve.js + --constitution + ${CCF_DIR}/samples/constitutions/default/apply.js +) +set(CCF_NETWORK_TEST_ARGS --host-log-level ${TEST_HOST_LOGGING_LEVEL} + --worker-threads ${WORKER_THREADS} +) + +if(COMPILE_TARGET STREQUAL "sgx") + add_enclave_library(js_openenclave.enclave ${CCF_DIR}/src/js/openenclave.cpp) + target_link_libraries(js_openenclave.enclave PUBLIC ccf.enclave) + add_lvi_mitigations(js_openenclave.enclave) + install( + TARGETS js_openenclave.enclave + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "snp") + add_library(js_openenclave.snp STATIC ${CCF_DIR}/src/js/openenclave.cpp) + add_san(js_openenclave.snp) + target_link_libraries(js_openenclave.snp PUBLIC ccf.snp) + target_compile_options(js_openenclave.snp PRIVATE ${COMPILE_LIBCXX}) + target_compile_definitions( + js_openenclave.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE + _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP + ) + set_property(TARGET js_openenclave.snp PROPERTY POSITION_INDEPENDENT_CODE ON) + install( + TARGETS js_openenclave.snp + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "virtual") + add_library(js_openenclave.virtual STATIC ${CCF_DIR}/src/js/openenclave.cpp) + add_san(js_openenclave.virtual) + target_link_libraries(js_openenclave.virtual PUBLIC ccf.virtual) + target_compile_options(js_openenclave.virtual PRIVATE ${COMPILE_LIBCXX}) + target_compile_definitions( + js_openenclave.virtual + PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE _LIBCPP_HAS_THREAD_API_PTHREAD + PLATFORM_VIRTUAL + ) + set_property( + TARGET js_openenclave.virtual PROPERTY POSITION_INDEPENDENT_CODE ON + ) + install( + TARGETS js_openenclave.virtual + EXPORT ccf + DESTINATION lib + ) +endif() + +if(COMPILE_TARGET STREQUAL "sgx") + add_enclave_library( + js_generic_base.enclave ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp + ) + target_link_libraries(js_generic_base.enclave PUBLIC ccf.enclave) + add_lvi_mitigations(js_generic_base.enclave) + install( + TARGETS js_generic_base.enclave + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "snp") + add_library( + js_generic_base.snp STATIC + ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp + ) + add_san(js_generic_base.snp) + add_warning_checks(js_generic_base.snp) + target_link_libraries(js_generic_base.snp PUBLIC ccf.snp) + target_compile_options(js_generic_base.snp PRIVATE ${COMPILE_LIBCXX}) + target_compile_definitions( + js_generic_base.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE + _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP + ) + set_property(TARGET js_generic_base.snp PROPERTY POSITION_INDEPENDENT_CODE ON) + install( + TARGETS js_generic_base.snp + EXPORT ccf + DESTINATION lib + ) +elseif(COMPILE_TARGET STREQUAL "virtual") + add_library( + js_generic_base.virtual STATIC + ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp + ) + add_san(js_generic_base.virtual) + add_warning_checks(js_generic_base.virtual) + target_link_libraries(js_generic_base.virtual PUBLIC ccf.virtual) + target_compile_options(js_generic_base.virtual PRIVATE ${COMPILE_LIBCXX}) + target_compile_definitions( + js_openenclave.virtual + PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE _LIBCPP_HAS_THREAD_API_PTHREAD + PLATFORM_VIRTUAL + ) + set_property( + TARGET js_generic_base.virtual PROPERTY POSITION_INDEPENDENT_CODE ON + ) + install( + TARGETS js_generic_base.virtual + EXPORT ccf + DESTINATION lib + ) +endif() +# SNIPPET_START: JS generic application +add_ccf_app( + js_generic + SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp + LINK_LIBS_ENCLAVE js_generic_base.enclave js_openenclave.enclave + LINK_LIBS_VIRTUAL js_generic_base.virtual js_openenclave.virtual + LINK_LIBS_SNP js_generic_base.snp js_openenclave.snp INSTALL_LIBS ON +) +sign_app_library( + js_generic.enclave ${CCF_DIR}/src/apps/js_generic/oe_sign.conf + ${CMAKE_CURRENT_BINARY_DIR}/signing_key.pem INSTALL_LIBS ON +) +# SNIPPET_END: JS generic application + +include(${CCF_DIR}/cmake/quictls.cmake) + +install(DIRECTORY ${CCF_DIR}/samples/apps/logging/js + DESTINATION samples/logging +) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake) # 3.0 temporary way to identify platform from Debian package diff --git a/cmake/common.cmake b/cmake/common.cmake index edd723abdf86..427cc7af79d2 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1,206 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -set(CMAKE_MODULE_PATH "${CCF_DIR}/cmake;${CMAKE_MODULE_PATH}") - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -find_package(Threads REQUIRED) - -function(message) - if(NOT MESSAGE_QUIET) - _message(${ARGN}) - endif() -endfunction() - -option(PROFILE_TESTS "Profile tests" OFF) -set(PYTHON unbuffer python3) - -set(DISTRIBUTE_PERF_TESTS - "" - CACHE - STRING - "Hosts to which performance tests should be distributed, for example -n ssh://x.x.x.x -n ssh://x.x.x.x -n ssh://x.x.x.x" -) - -if(DISTRIBUTE_PERF_TESTS) - separate_arguments(NODES UNIX_COMMAND ${DISTRIBUTE_PERF_TESTS}) -else() - unset(NODES) -endif() - -option(VERBOSE_LOGGING "Enable verbose, unsafe logging of enclave code" OFF) -set(TEST_HOST_LOGGING_LEVEL "info") -if(VERBOSE_LOGGING) - set(TEST_HOST_LOGGING_LEVEL "trace") - add_compile_definitions(VERBOSE_LOGGING) -endif() - -option(USE_NULL_ENCRYPTOR "Turn off encryption of ledger updates - debug only" - OFF -) -if(USE_NULL_ENCRYPTOR) - add_compile_definitions(USE_NULL_ENCRYPTOR) -endif() - -option(SAN "Enable Address and Undefined Behavior Sanitizers" OFF) -option(TSAN "Enable Thread Sanitizers" OFF) -option(BUILD_END_TO_END_TESTS "Build end to end tests" ON) -option(COVERAGE "Enable coverage mapping" OFF) -option(SHUFFLE_SUITE "Shuffle end to end test suite" OFF) -option(LONG_TESTS "Enable long end-to-end tests" OFF) -option(KV_STATE_RB "Enable RBMap as underlying KV state implementation" OFF) -if(KV_STATE_RB) - add_compile_definitions(KV_STATE_RB) -endif() - -# This option controls whether to link virtual builds against snmalloc rather -# than use the system allocator. In builds using Open Enclave, enclave -# allocation is managed separately and enabling snmalloc is done by linking -# openenclave::oesnmalloc -option(USE_SNMALLOC "Link virtual build against snmalloc" ON) - -enable_language(ASM) - -set(CCF_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated) -include_directories(${CCF_DIR}/include) -include_directories(${CCF_DIR}/src) - -set(CCF_3RD_PARTY_EXPORTED_DIR "${CCF_DIR}/3rdparty/exported") -set(CCF_3RD_PARTY_INTERNAL_DIR "${CCF_DIR}/3rdparty/internal") - -include_directories(SYSTEM ${CCF_3RD_PARTY_EXPORTED_DIR}) -include_directories(SYSTEM ${CCF_3RD_PARTY_INTERNAL_DIR}) - -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tools.cmake) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/tools.cmake DESTINATION cmake) -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccf_app.cmake) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccf_app.cmake DESTINATION cmake) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/open_enclave.cmake - DESTINATION cmake -) - -if(SAN AND LVI_MITIGATIONS) - message( - FATAL_ERROR - "Building with both SAN and LVI mitigations is unsafe and deadlocks - choose one" - ) -endif() - -if(TSAN AND LVI_MITIGATIONS) - message( - FATAL_ERROR - "Building with both TSAN and LVI mitigations is unsafe and deadlocks - choose one" - ) -endif() - -add_custom_command( - COMMAND - openenclave::oeedger8r ${CCF_DIR}/edl/ccf.edl --search-path ${OE_INCLUDEDIR} - --trusted --trusted-dir ${CCF_GENERATED_DIR} --untrusted --untrusted-dir - ${CCF_GENERATED_DIR} - COMMAND mv ${CCF_GENERATED_DIR}/ccf_t.c ${CCF_GENERATED_DIR}/ccf_t.cpp - COMMAND mv ${CCF_GENERATED_DIR}/ccf_u.c ${CCF_GENERATED_DIR}/ccf_u.cpp - DEPENDS ${CCF_DIR}/edl/ccf.edl - OUTPUT ${CCF_GENERATED_DIR}/ccf_t.cpp ${CCF_GENERATED_DIR}/ccf_u.cpp - COMMENT "Generating code from EDL, and renaming to .cpp" -) - -# Copy and install CCF utilities -set(CCF_UTILITIES keygenerator.sh scurl.sh submit_recovery_share.sh - verify_quote.sh -) -foreach(UTILITY ${CCF_UTILITIES}) - configure_file( - ${CCF_DIR}/python/utils/${UTILITY} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY - ) - install(PROGRAMS ${CCF_DIR}/python/utils/${UTILITY} DESTINATION bin) -endforeach() - -# Copy utilities from tests directory -set(CCF_TEST_UTILITIES - tests.sh - cimetrics_env.sh - upload_pico_metrics.py - test_install.sh - docker_wrap.sh - config.jinja - recovery_benchmark.sh -) -foreach(UTILITY ${CCF_TEST_UTILITIES}) - configure_file( - ${CCF_DIR}/tests/${UTILITY} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY - ) -endforeach() - -# Install additional utilities -install(PROGRAMS ${CCF_DIR}/samples/scripts/sgxinfo.sh DESTINATION bin) -install(PROGRAMS ${CCF_DIR}/samples/scripts/snpinfo.sh DESTINATION bin) -install(FILES ${CCF_DIR}/tests/config.jinja DESTINATION bin) - -if(SAN) - install(FILES ${CCF_DIR}/src/san_common.suppressions DESTINATION bin) -endif() - -# Install getting_started scripts for VM creation and setup -install( - DIRECTORY ${CCF_DIR}/getting_started/ - DESTINATION getting_started - USE_SOURCE_PERMISSIONS -) - -if(COMPILE_TARGET STREQUAL "sgx") - # While virtual libraries need to be built for sgx for unit tests, these do - # not get installed to minimise installation size - set(INSTALL_VIRTUAL_LIBRARIES OFF) - - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - set(DEFAULT_ENCLAVE_TYPE debug) - endif() -elseif(COMPILE_TARGET STREQUAL "snp") - set(INSTALL_VIRTUAL_LIBRARIES OFF) -else() - set(INSTALL_VIRTUAL_LIBRARIES ON) -endif() - -set(HTTP_PARSER_SOURCES - ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/api.c - ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/http.c - ${CCF_3RD_PARTY_EXPORTED_DIR}/llhttp/llhttp.c -) - -set(CCF_ENDPOINTS_SOURCES - ${CCF_DIR}/src/endpoints/endpoint.cpp - ${CCF_DIR}/src/endpoints/endpoint_registry.cpp - ${CCF_DIR}/src/endpoints/base_endpoint_registry.cpp - ${CCF_DIR}/src/endpoints/common_endpoint_registry.cpp - ${CCF_DIR}/src/endpoints/json_handler.cpp - ${CCF_DIR}/src/endpoints/authentication/cose_auth.cpp - ${CCF_DIR}/src/endpoints/authentication/cert_auth.cpp - ${CCF_DIR}/src/endpoints/authentication/empty_auth.cpp - ${CCF_DIR}/src/endpoints/authentication/jwt_auth.cpp - ${CCF_DIR}/src/enclave/enclave_time.cpp - ${CCF_DIR}/src/indexing/strategies/seqnos_by_key_bucketed.cpp - ${CCF_DIR}/src/indexing/strategies/seqnos_by_key_in_memory.cpp - ${CCF_DIR}/src/indexing/strategies/visit_each_entry_in_map.cpp - ${CCF_DIR}/src/node/historical_queries_adapter.cpp - ${CCF_DIR}/src/node/historical_queries_utils.cpp - ${CCF_DIR}/src/node/receipt.cpp -) - -find_library(CRYPTO_LIBRARY crypto) -find_library(TLS_LIBRARY ssl) - -include(${CCF_DIR}/cmake/crypto.cmake) -include(${CCF_DIR}/cmake/quickjs.cmake) -include(${CCF_DIR}/cmake/sss.cmake) -include(${CCF_DIR}/cmake/nghttp2.cmake) -include(${CCF_DIR}/cmake/qcbor.cmake) -include(${CCF_DIR}/cmake/t_cose.cmake) -set(MESSAGE_QUIET ON) -include(${CCF_DIR}/cmake/protobuf.cmake) -unset(MESSAGE_QUIET) - # Unit test wrapper function(add_unit_test name) add_executable(${name} ${CCF_DIR}/src/enclave/thread_local.cpp ${ARGN}) @@ -247,317 +47,6 @@ function(add_test_bin name) add_san(${name}) endfunction() -# Host Executable -if(SAN - OR TSAN - OR NOT USE_SNMALLOC -) - set(SNMALLOC_COMPILE_OPTIONS "") -else() - set(SNMALLOC_HEADER_ONLY_LIBRARY ON) - add_subdirectory(3rdparty/exported/snmalloc EXCLUDE_FROM_ALL) - set(SNMALLOC_COMPILE_OPTIONS "-mcx16") - list(APPEND CCHOST_SOURCES src/host/snmalloc.cpp) -endif() - -list(APPEND CCHOST_SOURCES ${CCF_DIR}/src/host/main.cpp) - -if(COMPILE_TARGET STREQUAL "sgx") - list(APPEND CCHOST_SOURCES ${CCF_GENERATED_DIR}/ccf_u.cpp) -endif() - -add_executable(cchost ${CCHOST_SOURCES}) - -add_warning_checks(cchost) -add_san(cchost) - -target_compile_options( - cchost PRIVATE ${COMPILE_LIBCXX} ${SNMALLOC_COMPILE_OPTIONS} -) -target_include_directories(cchost PRIVATE ${CCF_GENERATED_DIR}) - -# Host is always built with verbose logging enabled, regardless of CMake option -target_compile_definitions(cchost PRIVATE VERBOSE_LOGGING) - -if(COMPILE_TARGET STREQUAL "sgx") - target_compile_definitions(cchost PUBLIC PLATFORM_SGX) -elseif(COMPILE_TARGET STREQUAL "snp") - target_compile_definitions(cchost PUBLIC PLATFORM_SNP) -elseif(COMPILE_TARGET STREQUAL "virtual") - target_compile_definitions(cchost PUBLIC PLATFORM_VIRTUAL) -endif() - -target_link_libraries( - cchost PRIVATE uv ${TLS_LIBRARY} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} - ${LINK_LIBCXX} ccfcrypto.host -) -if(COMPILE_TARGET STREQUAL "sgx") - target_link_libraries(cchost PRIVATE openenclave::oehost) -endif() - -install(TARGETS cchost DESTINATION bin) - -# Perf scenario executable -add_executable( - scenario_perf_client ${CCF_DIR}/src/clients/perf/scenario_perf_client.cpp -) -target_link_libraries( - scenario_perf_client PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser.host - ccfcrypto.host -) -if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9) - target_link_libraries(scenario_perf_client PRIVATE c++fs) -endif() -install(TARGETS scenario_perf_client DESTINATION bin) - -# HTTP parser -if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library_c(http_parser.enclave "${HTTP_PARSER_SOURCES}") - install( - TARGETS http_parser.enclave - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "snp") - add_library(http_parser.snp "${HTTP_PARSER_SOURCES}") - set_property(TARGET http_parser.snp PROPERTY POSITION_INDEPENDENT_CODE ON) - install( - TARGETS http_parser.snp - EXPORT ccf - DESTINATION lib - ) -endif() - -add_library(http_parser.host "${HTTP_PARSER_SOURCES}") -set_property(TARGET http_parser.host PROPERTY POSITION_INDEPENDENT_CODE ON) -if(INSTALL_VIRTUAL_LIBRARIES) - install( - TARGETS http_parser.host - EXPORT ccf - DESTINATION lib - ) -endif() - -# CCF kv libs -set(CCF_KV_SOURCES - ${CCF_DIR}/src/kv/tx.cpp ${CCF_DIR}/src/kv/untyped_map_handle.cpp - ${CCF_DIR}/src/kv/untyped_map_diff.cpp -) - -if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library(ccf_kv.enclave "${CCF_KV_SOURCES}") - add_warning_checks(ccf_kv.enclave) - install( - TARGETS ccf_kv.enclave - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "snp") - add_host_library(ccf_kv.snp "${CCF_KV_SOURCES}") - add_san(ccf_kv.snp) - add_warning_checks(ccf_kv.snp) - install( - TARGETS ccf_kv.snp - EXPORT ccf - DESTINATION lib - ) -endif() - -add_host_library(ccf_kv.host "${CCF_KV_SOURCES}") -add_san(ccf_kv.host) -add_warning_checks(ccf_kv.host) -if(INSTALL_VIRTUAL_LIBRARIES) - install( - TARGETS ccf_kv.host - EXPORT ccf - DESTINATION lib - ) -endif() - -# CCF endpoints libs -if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library(ccf_endpoints.enclave "${CCF_ENDPOINTS_SOURCES}") - target_link_libraries( - ccf_endpoints.enclave - PUBLIC qcbor.enclave t_cose.enclave http_parser.enclave ccfcrypto.enclave - ccf_kv.enclave - ) - add_warning_checks(ccf_endpoints.enclave) - install( - TARGETS ccf_endpoints.enclave - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "snp") - add_host_library(ccf_endpoints.snp "${CCF_ENDPOINTS_SOURCES}") - target_link_libraries( - ccf_endpoints.snp PUBLIC qcbor.snp t_cose.snp http_parser.snp ccfcrypto.snp - ccf_kv.snp - ) - add_san(ccf_endpoints.snp) - add_warning_checks(ccf_endpoints.snp) - install( - TARGETS ccf_endpoints.snp - EXPORT ccf - DESTINATION lib - ) -endif() - -add_host_library(ccf_endpoints.host "${CCF_ENDPOINTS_SOURCES}") -target_link_libraries( - ccf_endpoints.host PUBLIC qcbor.host t_cose.host http_parser.host - ccfcrypto.host ccf_kv.host -) -add_san(ccf_endpoints.host) -add_warning_checks(ccf_endpoints.host) - -if(INSTALL_VIRTUAL_LIBRARIES) - install( - TARGETS ccf_endpoints.host - EXPORT ccf - DESTINATION lib - ) -endif() - -# Common test args for Python scripts starting up CCF networks -set(WORKER_THREADS - 0 - CACHE STRING "Number of worker threads to start on each CCF node" -) - -set(CCF_NETWORK_TEST_DEFAULT_CONSTITUTION - --constitution - ${CCF_DIR}/samples/constitutions/default/actions.js - --constitution - ${CCF_DIR}/samples/constitutions/default/validate.js - --constitution - ${CCF_DIR}/samples/constitutions/default/resolve.js - --constitution - ${CCF_DIR}/samples/constitutions/default/apply.js -) -set(CCF_NETWORK_TEST_ARGS --host-log-level ${TEST_HOST_LOGGING_LEVEL} - --worker-threads ${WORKER_THREADS} -) - -if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library(js_openenclave.enclave ${CCF_DIR}/src/js/openenclave.cpp) - target_link_libraries(js_openenclave.enclave PUBLIC ccf.enclave) - add_lvi_mitigations(js_openenclave.enclave) - install( - TARGETS js_openenclave.enclave - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "snp") - add_library(js_openenclave.snp STATIC ${CCF_DIR}/src/js/openenclave.cpp) - add_san(js_openenclave.snp) - target_link_libraries(js_openenclave.snp PUBLIC ccf.snp) - target_compile_options(js_openenclave.snp PRIVATE ${COMPILE_LIBCXX}) - target_compile_definitions( - js_openenclave.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE - _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP - ) - set_property(TARGET js_openenclave.snp PROPERTY POSITION_INDEPENDENT_CODE ON) - install( - TARGETS js_openenclave.snp - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "virtual") - add_library(js_openenclave.virtual STATIC ${CCF_DIR}/src/js/openenclave.cpp) - add_san(js_openenclave.virtual) - target_link_libraries(js_openenclave.virtual PUBLIC ccf.virtual) - target_compile_options(js_openenclave.virtual PRIVATE ${COMPILE_LIBCXX}) - target_compile_definitions( - js_openenclave.virtual - PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE _LIBCPP_HAS_THREAD_API_PTHREAD - PLATFORM_VIRTUAL - ) - set_property( - TARGET js_openenclave.virtual PROPERTY POSITION_INDEPENDENT_CODE ON - ) - install( - TARGETS js_openenclave.virtual - EXPORT ccf - DESTINATION lib - ) -endif() - -if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library( - js_generic_base.enclave ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp - ) - target_link_libraries(js_generic_base.enclave PUBLIC ccf.enclave) - add_lvi_mitigations(js_generic_base.enclave) - install( - TARGETS js_generic_base.enclave - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "snp") - add_library( - js_generic_base.snp STATIC - ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp - ) - add_san(js_generic_base.snp) - add_warning_checks(js_generic_base.snp) - target_link_libraries(js_generic_base.snp PUBLIC ccf.snp) - target_compile_options(js_generic_base.snp PRIVATE ${COMPILE_LIBCXX}) - target_compile_definitions( - js_generic_base.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE - _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP - ) - set_property(TARGET js_generic_base.snp PROPERTY POSITION_INDEPENDENT_CODE ON) - install( - TARGETS js_generic_base.snp - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "virtual") - add_library( - js_generic_base.virtual STATIC - ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp - ) - add_san(js_generic_base.virtual) - add_warning_checks(js_generic_base.virtual) - target_link_libraries(js_generic_base.virtual PUBLIC ccf.virtual) - target_compile_options(js_generic_base.virtual PRIVATE ${COMPILE_LIBCXX}) - target_compile_definitions( - js_openenclave.virtual - PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE _LIBCPP_HAS_THREAD_API_PTHREAD - PLATFORM_VIRTUAL - ) - set_property( - TARGET js_generic_base.virtual PROPERTY POSITION_INDEPENDENT_CODE ON - ) - install( - TARGETS js_generic_base.virtual - EXPORT ccf - DESTINATION lib - ) -endif() -# SNIPPET_START: JS generic application -add_ccf_app( - js_generic - SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp - LINK_LIBS_ENCLAVE js_generic_base.enclave js_openenclave.enclave - LINK_LIBS_VIRTUAL js_generic_base.virtual js_openenclave.virtual - LINK_LIBS_SNP js_generic_base.snp js_openenclave.snp INSTALL_LIBS ON -) -sign_app_library( - js_generic.enclave ${CCF_DIR}/src/apps/js_generic/oe_sign.conf - ${CMAKE_CURRENT_BINARY_DIR}/signing_key.pem INSTALL_LIBS ON -) -# SNIPPET_END: JS generic application - -include(${CCF_DIR}/cmake/quictls.cmake) - -install(DIRECTORY ${CCF_DIR}/samples/apps/logging/js - DESTINATION samples/logging -) - -# Samples - # Helper for building clients inheriting from perf_client function(add_client_exe name) From 0c41373366db9893061e68534546ab2e6552ee1a Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 7 Jul 2023 18:26:24 +0100 Subject: [PATCH 031/135] [release/4.x] Cherry pick: Permit Trace+Debug logging in all non-SGX builds (#5375) (#5424) Co-authored-by: Eddy Ashton --- CHANGELOG.md | 6 ++++ CMakeLists.txt | 32 +++++++++++------ cmake/version.cmake | 6 +++- doc/build_apps/logging.rst | 2 +- doc/contribute/build_ccf.rst | 1 - doc/host_config_schema/cchost_config.json | 2 +- edl/ccf.edl | 1 + include/ccf/ds/logger.h | 44 +++++++++++------------ include/ccf/ds/logger_level.h | 13 +++++++ scripts/scan-build.sh | 3 -- src/common/configuration.h | 17 ++++----- src/common/enclave_interface_types.h | 2 ++ src/consensus/aft/test/driver.cpp | 2 +- src/consensus/aft/test/main.cpp | 2 +- src/ds/test/logger_bench.cpp | 8 ++--- src/ds/test/logger_json_test.cpp | 2 +- src/enclave/interface.h | 2 +- src/enclave/main.cpp | 20 +++++++++++ src/enclave/virtual_enclave.h | 3 ++ src/host/configuration.h | 2 +- src/host/enclave.h | 6 ++-- src/host/main.cpp | 16 +++++++++ src/http/test/http_test.cpp | 2 +- src/kv/test/kv_bench.cpp | 10 +++--- src/kv/test/kv_contention.cpp | 2 +- src/node/rpc/gov_logging.h | 2 +- src/node/test/history_bench.cpp | 2 +- tests/infra/e2e_args.py | 12 +++++-- tests/infra/network.py | 1 + tests/infra/remote.py | 22 +++++++++++- 30 files changed, 171 insertions(+), 74 deletions(-) create mode 100644 include/ccf/ds/logger_level.h diff --git a/CHANGELOG.md b/CHANGELOG.md index c98ad0688d51..f98360181f51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.5] + +[4.0.5]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.5 + +- Debug logging is now available in non-SGX builds by default, and controlled by a run-time CLI argument (`--enclave-log-level`). On SGX this remains a build-time decision (#5375). + ## [4.0.4] [4.0.4]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.4 diff --git a/CMakeLists.txt b/CMakeLists.txt index f5efab06bd72..d561aa8fb7f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,11 +74,25 @@ else() unset(NODES) endif() -option(VERBOSE_LOGGING "Enable verbose, unsafe logging of enclave code" OFF) -set(TEST_HOST_LOGGING_LEVEL "info") +option( + VERBOSE_LOGGING + "Enable verbose, potentially unsafe logging of enclave code. Affects logging level passed at run-time to end-to-end-tests, and compile-time max verbosity on SGX." + OFF +) +set(TEST_LOGGING_LEVEL "info") if(VERBOSE_LOGGING) - set(TEST_HOST_LOGGING_LEVEL "trace") - add_compile_definitions(VERBOSE_LOGGING) + set(TEST_LOGGING_LEVEL "trace") +endif() + +# NB: Toggling VERBOSE_LOGGING on non-SGX platforms causes no build change, so +# should not cause a rebuild +if(COMPILE_TARGET STREQUAL "sgx" AND NOT VERBOSE_LOGGING) + # Disable verbose, unsafe logging of enclave code. On some platforms it is + # safe to build with this logging enabled, and then it can be disabled at + # run-time. However this run-time control is not possible on SGX, so to ensure + # a given MRENCLAVE cannot leak via debug logging it must be removed at + # build-time, with this option. + add_compile_definitions(CCF_DISABLE_VERBOSE_LOGGING) endif() option(USE_NULL_ENCRYPTOR "Turn off encryption of ledger updates - debug only" @@ -275,9 +289,6 @@ target_compile_options( ) target_include_directories(cchost PRIVATE ${CCF_GENERATED_DIR}) -# Host is always built with verbose logging enabled, regardless of CMake option -target_compile_definitions(cchost PRIVATE VERBOSE_LOGGING) - if(COMPILE_TARGET STREQUAL "sgx") target_compile_definitions(cchost PUBLIC PLATFORM_SGX) elseif(COMPILE_TARGET STREQUAL "snp") @@ -434,8 +445,9 @@ set(CCF_NETWORK_TEST_DEFAULT_CONSTITUTION --constitution ${CCF_DIR}/samples/constitutions/default/apply.js ) -set(CCF_NETWORK_TEST_ARGS --host-log-level ${TEST_HOST_LOGGING_LEVEL} - --worker-threads ${WORKER_THREADS} +set(CCF_NETWORK_TEST_ARGS + --host-log-level ${TEST_LOGGING_LEVEL} --enclave-log-level + ${TEST_LOGGING_LEVEL} --worker-threads ${WORKER_THREADS} ) if(COMPILE_TARGET STREQUAL "sgx") @@ -851,7 +863,6 @@ if(BUILD_TESTS) logger_json_test ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/logger_json_test.cpp ) - target_compile_definitions(logger_json_test PUBLIC VERBOSE_LOGGING) add_unit_test( kv_test @@ -1108,7 +1119,6 @@ if(BUILD_TESTS) # Picobench benchmarks add_picobench(map_bench SRCS src/ds/test/map_bench.cpp) add_picobench(logger_bench SRCS src/ds/test/logger_bench.cpp) - target_compile_definitions(logger_bench PUBLIC VERBOSE_LOGGING) add_picobench(json_bench SRCS src/ds/test/json_bench.cpp) add_picobench(ring_buffer_bench SRCS src/ds/test/ring_buffer_bench.cpp) add_picobench( diff --git a/cmake/version.cmake b/cmake/version.cmake index 55bf7fa50499..b44770950f94 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -5,7 +5,11 @@ unset(CCF_VERSION) unset(CCF_RELEASE_VERSION) unset(CCF_VERSION_SUFFIX) -option(UNSAFE_VERSION "Produce build with unsafe logging levels" OFF) +option( + UNSAFE_VERSION + "Append unsafe suffix to project and targets. Should be used on platforms where log level is determined at build-time, to distinguish builds which are unsafely verbose." + OFF +) set(CCF_PROJECT "ccf_${COMPILE_TARGET}") if(UNSAFE_VERSION) diff --git a/doc/build_apps/logging.rst b/doc/build_apps/logging.rst index 5bef089e0159..7ef2485d1fc8 100644 --- a/doc/build_apps/logging.rst +++ b/doc/build_apps/logging.rst @@ -28,7 +28,7 @@ These logging functions do several things: - Prefix formatted metadata. The produced log line will include a timestamp, the name and line number where the line was produced, and an ``[app]`` tag - Write without an ECALL. The final write must be handled by the host, so writing directly from the enclave would require an expensive ECALL. Instead these macros will queue writes to a ringbuffer for the host to process, so diagnostic logging should not cause significant performance drops -By default, the ``CCF_APP_TRACE`` and ``CCF_APP_DEBUG`` macros are compiled out, to avoid leaking confidential information during execution. To enable these, define ``VERBOSE_LOGGING`` before including ``logger.h`` in your application code. The unsafe build variants of CCF include verbose logging of the framework itself, but this can be used independently of verbose logging in the application code. +The most-verbose lines which are emitted may be restricted at build-time, if required for security on that platform. On SGX, the ``CCF_APP_TRACE`` and ``CCF_APP_DEBUG`` macros are disabled at compile-time, to avoid leaking confidential information during execution. To enable these, define ``VERBOSE_LOGGING`` before including ``logger.h`` in your application code. The unsafe build variants of CCF include verbose logging of the framework itself, but this can be used independently of verbose logging in the application code. .. note:: The app's logging entries will be interleaved (line-by-line) with the framework's logging messages. Filter for entries containing ``[app]`` to extract only application log lines. diff --git a/doc/contribute/build_ccf.rst b/doc/contribute/build_ccf.rst index 8cacba2a71d4..580e368b7d04 100644 --- a/doc/contribute/build_ccf.rst +++ b/doc/contribute/build_ccf.rst @@ -51,7 +51,6 @@ The most common build switches include: * **BUILD_TESTS**: Boolean. Build all tests for CCF. Default to ON. * **SAN**: Boolean. Build unit tests with Address and Undefined behaviour sanitizers enabled. Default to OFF. * **COMPILE_TARGET**: String. Target compilation platform. Defaults to ``sgx``. Supported values are ``sgx``, ``snp``, or ``virtual``. -* **VERBOSE_LOGGING**: Boolean. Enable all logging levels. Default to OFF. Run Tests --------- diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json index 26bdd6254808..8023769fdc70 100644 --- a/doc/host_config_schema/cchost_config.json +++ b/doc/host_config_schema/cchost_config.json @@ -534,7 +534,7 @@ "type": "string", "enum": ["Trace", "Debug", "Info", "Fail", "Fatal"], "default": "Info", - "description": "Logging level for the untrusted host. Note: while it is possible to set the host log level at startup, it is deliberately not possible to change the log level of the enclave without rebuilding it and changing its code identity" + "description": "Logging level for the untrusted host." }, "format": { "type": "string", diff --git a/edl/ccf.edl b/edl/ccf.edl index 7dae8bc96b94..c1d65e04fac0 100644 --- a/edl/ccf.edl +++ b/edl/ccf.edl @@ -25,6 +25,7 @@ enclave { size_t enclave_version_size, [out] size_t* enclave_version_len, StartType start_type, + LoggerLevel enclave_log_level, size_t num_worker_thread, [user_check] void* time_location, ); diff --git a/include/ccf/ds/logger.h b/include/ccf/ds/logger.h index 5f0bf33574a5..2d1d1ce0d02f 100644 --- a/include/ccf/ds/logger.h +++ b/include/ccf/ds/logger.h @@ -3,6 +3,7 @@ #pragma once #include "ccf/ds/enum_formatter.h" +#include "ccf/ds/logger_level.h" #include "ccf/ds/thread_ids.h" #define FMT_HEADER_ONLY @@ -16,22 +17,18 @@ namespace logger { - enum Level - { - TRACE, - DEBUG, // events useful for debugging - INFO, // important events that should be logged even in release mode - FAIL, // survivable failures that should always be logged - FATAL, // fatal errors that may be non-recoverable - MAX_LOG_LEVEL - }; - - static constexpr Level MOST_VERBOSE = static_cast(0); + static constexpr LoggerLevel MOST_VERBOSE = +#ifdef CCF_DISABLE_VERBOSE_LOGGING + LoggerLevel::INFO +#else + LoggerLevel::TRACE +#endif + ; static constexpr const char* LevelNames[] = { "trace", "debug", "info", "fail", "fatal"}; - static constexpr const char* to_string(Level l) + static constexpr const char* to_string(LoggerLevel l) { return LevelNames[static_cast(l)]; } @@ -48,7 +45,7 @@ namespace logger { public: friend struct Out; - Level log_level; + LoggerLevel log_level; std::string tag; std::string file_name; size_t line_number; @@ -58,7 +55,7 @@ namespace logger std::string msg; LogLine( - Level level_, + LoggerLevel level_, std::string_view tag_, std::string_view file_name_, size_t line_number_, @@ -295,14 +292,14 @@ namespace logger } #endif - static inline Level& level() + static inline LoggerLevel& level() { - static Level the_level = MOST_VERBOSE; + static LoggerLevel the_level = MOST_VERBOSE; return the_level; } - static inline bool ok(Level l) + static inline bool ok(LoggerLevel l) { return l >= level(); } @@ -327,7 +324,7 @@ namespace logger } #ifndef INSIDE_ENCLAVE - if (line.log_level == Level::FATAL) + if (line.log_level == LoggerLevel::FATAL) { throw std::logic_error("Fatal: " + format_to_text(line)); } @@ -358,8 +355,9 @@ namespace logger // This allows: // CCF_LOG_OUT(DEBUG, "foo") << "this " << "msg"; #define CCF_LOG_OUT(LVL, TAG) \ - logger::config::ok(logger::LVL) && \ - logger::Out() == logger::LogLine(logger::LVL, TAG, __FILE__, __LINE__) + logger::config::ok(LoggerLevel::LVL) && \ + logger::Out() == \ + logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__) // To avoid repeating the (s, ...) args for every macro, we cheat with a curried // macro here by ending the macro with another macro name, which then accepts @@ -382,7 +380,7 @@ namespace logger # define CCF_LOGGER_DEPRECATE(MACRO) #endif -#ifdef VERBOSE_LOGGING +#ifndef CCF_DISABLE_VERBOSE_LOGGING # define LOG_TRACE_FMT \ CCF_LOGGER_DEPRECATE(LOG_TRACE_FMT) CCF_LOG_FMT(TRACE, "") # define LOG_DEBUG_FMT \ @@ -391,8 +389,8 @@ namespace logger # define CCF_APP_TRACE CCF_LOG_FMT(TRACE, "app") # define CCF_APP_DEBUG CCF_LOG_FMT(DEBUG, "app") #else -// Without compile-time VERBOSE_LOGGING option, these logging macros are -// compile-time nops (and cannot be enabled by accident or malice) +// With verbose logging disabled by compile-time definition, these logging +// macros are compile-time nops (and cannot be enabled by accident or malice) # define LOG_TRACE_FMT(...) CCF_LOGGER_DEPRECATE(LOG_TRACE_FMT)((void)0) # define LOG_DEBUG_FMT(...) CCF_LOGGER_DEPRECATE(LOG_DEBUG_FMT)((void)0) diff --git a/include/ccf/ds/logger_level.h b/include/ccf/ds/logger_level.h new file mode 100644 index 000000000000..2b77d1a8395b --- /dev/null +++ b/include/ccf/ds/logger_level.h @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +enum LoggerLevel +{ + TRACE, + DEBUG, // events useful for debugging + INFO, // important events that should be logged even in release mode + FAIL, // survivable failures that should always be logged + FATAL, // fatal errors that may be non-recoverable + MAX_LOG_LEVEL +}; diff --git a/scripts/scan-build.sh b/scripts/scan-build.sh index b5f6488fa1b6..7444e64d177a 100755 --- a/scripts/scan-build.sh +++ b/scripts/scan-build.sh @@ -9,8 +9,5 @@ export CCC_CXX="clang++-$CLANG_VERSION" SCAN="scan-build-$CLANG_VERSION --exclude 3rdparty --exclude test" -# VERBOSE_LOGGING=ON is important, without it scan-build will report values as unused -# everywhere we compile out the logging statements that would otherwise read them -$SCAN cmake -GNinja -DCOMPILE_TARGET=virtual -DVERBOSE_LOGGING=ON -DCMAKE_BUILD_TYPE=Debug .. # Fails on the current build of clang, because of false positives in doctest, WIP $SCAN ninja || true \ No newline at end of file diff --git a/src/common/configuration.h b/src/common/configuration.h index a4140f699914..c53fd01bd29a 100644 --- a/src/common/configuration.h +++ b/src/common/configuration.h @@ -22,16 +22,13 @@ #include #include -namespace logger -{ - DECLARE_JSON_ENUM( - Level, - {{Level::TRACE, "Trace"}, - {Level::DEBUG, "Debug"}, - {Level::INFO, "Info"}, - {Level::FAIL, "Fail"}, - {Level::FATAL, "Fatal"}}); -} +DECLARE_JSON_ENUM( + LoggerLevel, + {{LoggerLevel::TRACE, "Trace"}, + {LoggerLevel::DEBUG, "Debug"}, + {LoggerLevel::INFO, "Info"}, + {LoggerLevel::FAIL, "Fail"}, + {LoggerLevel::FATAL, "Fatal"}}); DECLARE_JSON_ENUM( StartType, diff --git a/src/common/enclave_interface_types.h b/src/common/enclave_interface_types.h index 3d376f087f47..27cf3fc08bfb 100644 --- a/src/common/enclave_interface_types.h +++ b/src/common/enclave_interface_types.h @@ -2,6 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once +#include "ccf/ds/logger_level.h" + enum CreateNodeStatus { /** Call was successful and the node was successfully created */ diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 2ee47f24a102..6a2542458e67 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -44,7 +44,7 @@ int main(int argc, char** argv) #else logger::config::add_text_console_logger(); #endif - logger::config::level() = logger::DEBUG; + logger::config::level() = LoggerLevel::DEBUG; threading::ThreadMessaging::init(1); diff --git a/src/consensus/aft/test/main.cpp b/src/consensus/aft/test/main.cpp index b55328cdfccc..48ab8335b989 100644 --- a/src/consensus/aft/test/main.cpp +++ b/src/consensus/aft/test/main.cpp @@ -769,7 +769,7 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) DOCTEST_TEST_CASE("Exceed append entries limit") { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; ccf::NodeId node_id0 = kv::test::PrimaryNodeId; ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; diff --git a/src/ds/test/logger_bench.cpp b/src/ds/test/logger_bench.cpp index 183895ce3d92..6e54b15bdde5 100644 --- a/src/ds/test/logger_bench.cpp +++ b/src/ds/test/logger_bench.cpp @@ -51,7 +51,7 @@ static void log_accepted(picobench::state& s) { prepare_loggers(); - logger::config::level() = logger::DEBUG; + logger::config::level() = LoggerLevel::DEBUG; { picobench::scope scope(s); @@ -69,7 +69,7 @@ static void log_accepted_fmt(picobench::state& s) { prepare_loggers(); - logger::config::level() = logger::DEBUG; + logger::config::level() = LoggerLevel::DEBUG; { picobench::scope scope(s); @@ -87,7 +87,7 @@ static void log_rejected(picobench::state& s) { prepare_loggers(); - logger::config::level() = logger::FAIL; + logger::config::level() = LoggerLevel::FAIL; { picobench::scope scope(s); @@ -105,7 +105,7 @@ static void log_rejected_fmt(picobench::state& s) { prepare_loggers(); - logger::config::level() = logger::FAIL; + logger::config::level() = LoggerLevel::FAIL; { picobench::scope scope(s); diff --git a/src/ds/test/logger_json_test.cpp b/src/ds/test/logger_json_test.cpp index 40613e9a3d0c..7d77149b0767 100644 --- a/src/ds/test/logger_json_test.cpp +++ b/src/ds/test/logger_json_test.cpp @@ -12,7 +12,7 @@ TEST_CASE("Test custom log format") std::string test_log_file = "./test_json_logger.txt"; remove(test_log_file.c_str()); logger::config::add_json_console_logger(); - logger::config::level() = logger::DEBUG; + logger::config::level() = LoggerLevel::DEBUG; std::string log_msg_dbg = "log_msg_dbg"; std::string log_msg_fail = "log_msg_fail"; diff --git a/src/enclave/interface.h b/src/enclave/interface.h index cf6f3e815d0a..096385e96d57 100644 --- a/src/enclave/interface.h +++ b/src/enclave/interface.h @@ -38,7 +38,7 @@ DECLARE_RINGBUFFER_MESSAGE_PAYLOAD( std::chrono::microseconds::rep, std::string, size_t, - logger::Level, + LoggerLevel, std::string, uint16_t, std::string); diff --git a/src/enclave/main.cpp b/src/enclave/main.cpp index e1f9d5c66906..4987c801469b 100644 --- a/src/enclave/main.cpp +++ b/src/enclave/main.cpp @@ -72,6 +72,7 @@ extern "C" size_t enclave_version_size, size_t* enclave_version_len, StartType start_type, + LoggerLevel enclave_log_level, size_t num_worker_threads, void* time_location) { @@ -229,6 +230,25 @@ extern "C" return CreateNodeStatus::ReconfigurationMethodNotSupported; } + // Warn if run-time logging level is unsupported. SGX enclaves have their + // minimum logging level (maximum verbosity) restricted at compile-time, + // while other platforms can permit any level at compile-time and then bind + // the run-time choice in attestations. + const auto mv = logger::MOST_VERBOSE; + const auto requested = enclave_log_level; + const auto permitted = std::max(mv, requested); + if (requested != permitted) + { + LOG_FAIL_FMT( + "Unable to set requested enclave logging level '{}'. Most verbose " + "permitted level is '{}', so setting level to '{}'.", + logger::to_string(requested), + logger::to_string(mv), + logger::to_string(permitted)); + } + + logger::config::level() = permitted; + ccf::Enclave* enclave = nullptr; try diff --git a/src/enclave/virtual_enclave.h b/src/enclave/virtual_enclave.h index 22f72e106b6f..9bafc2b92d15 100644 --- a/src/enclave/virtual_enclave.h +++ b/src/enclave/virtual_enclave.h @@ -115,6 +115,7 @@ extern "C" size_t enclave_version_size, size_t* enclave_version_len, StartType start_type, + LoggerLevel enclave_log_level, size_t num_worker_thread, void* time_location) { @@ -134,6 +135,7 @@ extern "C" size_t, size_t*, StartType, + LoggerLevel, size_t, void*); @@ -157,6 +159,7 @@ extern "C" enclave_version_size, enclave_version_len, start_type, + enclave_log_level, num_worker_thread, time_location); diff --git a/src/host/configuration.h b/src/host/configuration.h index 99c09ed878ca..90d43e7481fb 100644 --- a/src/host/configuration.h +++ b/src/host/configuration.h @@ -111,7 +111,7 @@ namespace host struct Logging { - logger::Level host_level = logger::Level::INFO; + LoggerLevel host_level = LoggerLevel::INFO; LogFormat format = LogFormat::TEXT; bool operator==(const Logging&) const = default; diff --git a/src/host/enclave.h b/src/host/enclave.h index 48f51a55b5d5..6a243c1745af 100644 --- a/src/host/enclave.h +++ b/src/host/enclave.h @@ -128,7 +128,7 @@ namespace host expect_enclave_file_suffix(path, ".enclave.so.signed", type); } -# ifndef VERBOSE_LOGGING +# ifdef CCF_DISABLE_VERBOSE_LOGGING oe_log_set_callback(nullptr, nop_oe_logger); # endif @@ -220,6 +220,7 @@ namespace host std::vector& node_cert, std::vector& service_cert, StartType start_type, + LoggerLevel enclave_log_level, size_t num_worker_thread, void* time_location) { @@ -258,7 +259,8 @@ namespace host snapshot_aligned_size, node_cert.data(), node_cert.size(), &node_cert_len, \ service_cert.data(), service_cert.size(), &service_cert_len, \ enclave_version_buf.data(), enclave_version_buf.size(), \ - &enclave_version_len, start_type, num_worker_thread, time_location + &enclave_version_len, start_type, enclave_log_level, num_worker_thread, \ + time_location oe_result_t err = OE_FAILURE; diff --git a/src/host/main.cpp b/src/host/main.cpp index 2e4d0cb40ed5..04d7104b4f6d 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -97,6 +97,21 @@ int main(int argc, char** argv) app.add_flag( "-v, --version", print_version, "Display CCF host version and exit"); + LoggerLevel enclave_log_level = LoggerLevel::INFO; + std::map log_level_options; + for (size_t i = logger::MOST_VERBOSE; i < LoggerLevel::MAX_LOG_LEVEL; ++i) + { + const auto l = (LoggerLevel)i; + log_level_options[logger::to_string(l)] = l; + } + + app + .add_option( + "--enclave-log-level", + enclave_log_level, + "Logging level for the enclave code") + ->transform(CLI::CheckedTransformer(log_level_options, CLI::ignore_case)); + try { app.parse(argc, argv); @@ -634,6 +649,7 @@ int main(int argc, char** argv) node_cert, service_cert, config.command.type, + enclave_log_level, config.worker_threads, time_updater->behaviour.get_value()); ecall_completed.store(true); diff --git a/src/http/test/http_test.cpp b/src/http/test/http_test.cpp index 898567dec327..b3e93f355fb7 100644 --- a/src/http/test/http_test.cpp +++ b/src/http/test/http_test.cpp @@ -277,7 +277,7 @@ DOCTEST_TEST_CASE("URL parsing") DOCTEST_TEST_CASE("Pessimal transport") { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; const http::HeaderMap h1 = {{"foo", "bar"}, {"baz", "42"}}; const http::HeaderMap h2 = { diff --git a/src/kv/test/kv_bench.cpp b/src/kv/test/kv_bench.cpp index c4d1977912c4..992cb10e3aa2 100644 --- a/src/kv/test/kv_bench.cpp +++ b/src/kv/test/kv_bench.cpp @@ -52,7 +52,7 @@ std::string build_map_name(const std::string& core_name, kv::SecurityDomain sd) template static void serialise(picobench::state& s) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; kv::Store kv_store; auto secrets = create_ledger_secrets(); @@ -84,7 +84,7 @@ static void serialise(picobench::state& s) template static void deserialise(picobench::state& s) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; kv::Store kv_store; kv::Store kv_store2; @@ -125,7 +125,7 @@ static void deserialise(picobench::state& s) template static void commit_latency(picobench::state& s) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; kv::Store kv_store; auto secrets = create_ledger_secrets(); @@ -163,7 +163,7 @@ static void commit_latency(picobench::state& s) template static void ser_snap(picobench::state& s) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; kv::Store kv_store; auto secrets = create_ledger_secrets(); @@ -201,7 +201,7 @@ static void ser_snap(picobench::state& s) template static void des_snap(picobench::state& s) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; kv::Store kv_store; kv::Store kv_store2; diff --git a/src/kv/test/kv_contention.cpp b/src/kv/test/kv_contention.cpp index 04ab90dd5cc3..167b8c4667ae 100644 --- a/src/kv/test/kv_contention.cpp +++ b/src/kv/test/kv_contention.cpp @@ -35,7 +35,7 @@ class SlowStubConsensus : public kv::test::StubConsensus DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) { - logger::config::level() = logger::INFO; + logger::config::level() = LoggerLevel::INFO; // Multiple threads write random entries into random tables, and attempt to // commit them. A single thread continually compacts the kv to the latest diff --git a/src/node/rpc/gov_logging.h b/src/node/rpc/gov_logging.h index 3cd8fbe806ca..07448d220a12 100644 --- a/src/node/rpc/gov_logging.h +++ b/src/node/rpc/gov_logging.h @@ -4,7 +4,7 @@ #include "ccf/ds/logger.h" -#ifdef VERBOSE_LOGGING +#ifndef CCF_DISABLE_VERBOSE_LOGGING # define GOV_TRACE_FMT CCF_LOG_FMT(TRACE, "gov") # define GOV_DEBUG_FMT CCF_LOG_FMT(DEBUG, "gov") #else diff --git a/src/node/test/history_bench.cpp b/src/node/test/history_bench.cpp index f01e6e978408..69e4d2ff1736 100644 --- a/src/node/test/history_bench.cpp +++ b/src/node/test/history_bench.cpp @@ -160,7 +160,7 @@ PICOBENCH(append_compact<1000>).iterations(sizes).samples(10); int main(int argc, char* argv[]) { - logger::config::level() = logger::FATAL; + logger::config::level() = LoggerLevel::FATAL; threading::ThreadMessaging::init(1); picobench::runner runner; diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index eb004f92cc3e..2616e009ca19 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -107,11 +107,19 @@ def cli_args(add=lambda x: None, parser=None, accept_unknown=False): default=os.getenv("TEST_ENCLAVE", os.getenv("DEFAULT_ENCLAVE_PLATFORM", "sgx")), choices=("sgx", "snp", "virtual"), ) + log_level_choices = ("trace", "debug", "info", "fail", "fatal") + default_log_level = "info" parser.add_argument( "--host-log-level", help="Runtime host log level", - default="info", - choices=("trace", "debug", "info", "fail", "fatal"), + default=default_log_level, + choices=log_level_choices, + ) + parser.add_argument( + "--enclave-log-level", + help="Runtime enclave log level", + default=default_log_level, + choices=log_level_choices, ) parser.add_argument( "--log-format-json", diff --git a/tests/infra/network.py b/tests/infra/network.py index fc983d89d2e6..75ea88437895 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -165,6 +165,7 @@ class Network: "enclave_type", "enclave_platform", "host_log_level", + "enclave_log_level", "sig_tx_interval", "sig_ms_interval", "election_timeout_ms", diff --git a/tests/infra/remote.py b/tests/infra/remote.py index e0bc6772d42b..fa82bb416268 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -15,6 +15,10 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape import json import infra.snp as snp +import ccf._versionifier +from setuptools.extern.packaging.version import ( # type: ignore + Version, +) from loguru import logger as LOG @@ -591,6 +595,7 @@ def __init__( curve_id=None, version=None, host_log_level="Info", + enclave_log_level="Info", major_version=None, node_address=None, config_file=None, @@ -814,7 +819,22 @@ def __init__( if major_version is None or major_version > 1: # use the relative path to the config file so that it works on remotes too - cmd = [bin_path, "--config", os.path.basename(config_file)] + cmd = [ + bin_path, + "--config", + os.path.basename(config_file), + ] + + v = ( + ccf._versionifier.to_python_version(version) + if version is not None + else None + ) + if v is None or v >= Version("4.0.5"): + cmd += [ + "--enclave-log-level", + enclave_log_level, + ] if start_type == StartType.start: members_info = kwargs.get("members_info") From dbf70e2bc0bc0764edb5b3bc9790d7e4660e24fe Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:09:20 +0100 Subject: [PATCH 032/135] [release/4.x] Cherry pick: Update npm dependencies (#5429) (#5432) --- tests/npm-app/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/npm-app/package.json b/tests/npm-app/package.json index f098d770b8b0..873ea59f1f5a 100644 --- a/tests/npm-app/package.json +++ b/tests/npm-app/package.json @@ -16,7 +16,7 @@ "jsrsasign-util": "^1.0.2", "jwt-decode": "^3.0.0", "lodash-es": "^4.17.15", - "protobufjs": "^6.10.1" + "protobufjs": "^7.2.4" }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", @@ -24,7 +24,7 @@ "@rollup/plugin-typescript": "^8.2.0", "@types/jsrsasign": "^8.0.7", "@types/lodash-es": "^4.17.3", - "del-cli": "^3.0.1", + "del-cli": "^5.0.0", "http-server": "^0.13.0", "rollup": "^2.41.0", "tslib": "^2.0.1", From 3f1e7516eb2b014bddb8b5841af8c927837bf47b Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:12:45 +0100 Subject: [PATCH 033/135] [release/4.x] Cherry pick: Fix `logger_json_test` (#5426) (#5427) --- src/ds/test/logger_json_test.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ds/test/logger_json_test.cpp b/src/ds/test/logger_json_test.cpp index 7d77149b0767..3d6e2b8daef4 100644 --- a/src/ds/test/logger_json_test.cpp +++ b/src/ds/test/logger_json_test.cpp @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. +#undef CCF_DISABLE_VERBOSE_LOGGING + #include "ccf/ds/logger.h" #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN @@ -14,16 +16,16 @@ TEST_CASE("Test custom log format") logger::config::add_json_console_logger(); logger::config::level() = LoggerLevel::DEBUG; std::string log_msg_dbg = "log_msg_dbg"; - std::string log_msg_fail = "log_msg_fail"; + std::string log_msg_trace = "log_msg_trace"; std::ofstream out(test_log_file.c_str()); std::streambuf* coutbuf = std::cout.rdbuf(); std::cout.rdbuf(out.rdbuf()); LOG_DEBUG_FMT("{}", log_msg_dbg); - LOG_TRACE_FMT("{}", log_msg_fail); + LOG_TRACE_FMT("{}", log_msg_trace); LOG_DEBUG_FMT("{}", log_msg_dbg); - LOG_TRACE_FMT("{}", log_msg_fail); + LOG_TRACE_FMT("{}", log_msg_trace); LOG_DEBUG_FMT("{}", log_msg_dbg); out.flush(); From a9a5ea35c1b3d0f2859b764e294c6077978f8ce6 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:13:42 +0100 Subject: [PATCH 034/135] [release/4.x] Cherry pick: Fix historical queries issue - sequence of operations could result in permanently missing receipts (#5418) (#5441) --- src/node/historical_queries.h | 7 ++----- src/node/test/historical_queries.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h index 684c5537a4d9..84da7f22f5cf 100644 --- a/src/node/historical_queries.h +++ b/src/node/historical_queries.h @@ -314,15 +314,12 @@ namespace ccf::historical return; } - const auto newly_requested_receipts = - should_include_receipts && !include_receipts; - include_receipts = should_include_receipts; HISTORICAL_LOG( "Clearing {} supporting signatures", supporting_signatures.size()); supporting_signatures.clear(); - if (newly_requested_receipts) + if (should_include_receipts) { // If requesting signatures, populate receipts for each entry that we // already have. Normally this would be done when each entry was @@ -458,7 +455,7 @@ namespace ccf::historical details->get_commit_evidence(), details->claims_digest); HISTORICAL_LOG( - "Assigned a sig for {} after given signature at {}", + "Assigned a receipt for {} after given signature at {}", seqno, sig_details->transaction_id.to_str()); diff --git a/src/node/test/historical_queries.cpp b/src/node/test/historical_queries.cpp index 1231aa6bf03b..f7e238845edd 100644 --- a/src/node/test/historical_queries.cpp +++ b/src/node/test/historical_queries.cpp @@ -1527,6 +1527,33 @@ TEST_CASE("StateCache concurrent access") previously_requested.push_back("B"); query_random_range_states(20, 23, handle, error_printer); } + { + INFO("Problem case 7"); + const auto handle_a = 42; + const auto handle_b = 43; + std::vector messages; + auto error_printer = [&]() { + for (const auto& msg : messages) + { + std::cout << msg << std::endl; + } + }; + { + messages.push_back(fmt::format( + "Handle {} requested stores (no sigs) from 3 to 8", handle_a)); + query_random_range_stores(3, 8, handle_a, error_printer); + } + { + messages.push_back(fmt::format( + "Handle {} requested states (with sigs) from 5 to 8", handle_b)); + query_random_range_states(5, 8, handle_b, error_printer); + } + { + messages.push_back(fmt::format( + "Handle {} requested states (with sigs) from 3 to 8", handle_b)); + query_random_range_states(3, 8, handle_b, error_printer); + } + } const auto seed = time(NULL); INFO("Using seed: ", seed); From a03e0d9bb16a6edf15d2c532cf0efc1b7a14f640 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:16:23 +0100 Subject: [PATCH 035/135] [release/4.x] Cherry pick: Fix `channels_test` (#5433) (#5437) --- include/ccf/ds/logger.h | 3 ++- src/node/test/channels.cpp | 16 ++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/include/ccf/ds/logger.h b/include/ccf/ds/logger.h index 2d1d1ce0d02f..acce0c84cc0f 100644 --- a/include/ccf/ds/logger.h +++ b/include/ccf/ds/logger.h @@ -116,7 +116,8 @@ namespace logger virtual void emit(const std::string& s) { - std::cout << s << std::flush; + std::cout.write(s.c_str(), s.size()); + std::cout.flush(); } virtual void write( diff --git a/src/node/test/channels.cpp b/src/node/test/channels.cpp index cba3ee51004c..320333c42f34 100644 --- a/src/node/test/channels.cpp +++ b/src/node/test/channels.cpp @@ -1467,16 +1467,12 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Key rotation") expected_results->emplace_back(msg_body); } - // Sometimes, sleep for a while and let the channel threads catch up. - // If we do not sleep here, we may occasionally produce submit too many - // messages at once, resulting in a node's pending send queue filling up and - // a failure result when it calls `send_encrypted`. A real application - // should handle this error code, and set parameters so it is extremely - // rare. - if (i % 6 == 0) - { - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } + // If we do not sleep here, we submit many messages at once, resulting in a + // node's pending send queue filling up and a failure result when it calls + // `send_encrypted`. A real application should handle this error code, and + // set parameters so it is extremely rare. Here we simply sleep so that the + // sends happen one-at-a-time. + std::this_thread::sleep_for(std::chrono::milliseconds(20)); } // Assume this sleep is long enough for channel threads to fully catch up From dd4b3a18ed8f39e0a2cf785ebb77ffb86e352cf3 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:17:10 +0100 Subject: [PATCH 036/135] [release/4.x] Cherry pick: Channels connections fix - clear shared secret when peer key changes (#5443) (#5446) --- src/crypto/key_exchange.h | 2 + src/node/test/channels.cpp | 77 +++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/crypto/key_exchange.h b/src/crypto/key_exchange.h index a294da737dca..a6886374f4de 100644 --- a/src/crypto/key_exchange.h +++ b/src/crypto/key_exchange.h @@ -50,6 +50,7 @@ namespace tls if (!own_key) { own_key = make_key_pair(curve); + shared_secret.clear(); } // For backwards compatibility we need to keep the format we used with @@ -99,6 +100,7 @@ namespace tls } peer_key = std::make_shared(pk); + shared_secret.clear(); } const std::vector& get_shared_secret() diff --git a/src/node/test/channels.cpp b/src/node/test/channels.cpp index 320333c42f34..b42e3fd51f41 100644 --- a/src/node/test/channels.cpp +++ b/src/node/test/channels.cpp @@ -918,7 +918,6 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Interrupted key exchange") NoDrops, }; - DropStage drop_stage; for (const auto drop_stage : { DropStage::NoDrops, DropStage::FinalMessage, @@ -1035,6 +1034,82 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Interrupted key exchange") } } +TEST_CASE_FIXTURE(IORingbuffersFixture, "Stuttering handshake") +{ + MsgType aad; + aad.fill(0x10); + + auto network_kp = crypto::make_key_pair(default_curve); + auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); + + auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_cert = + generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); + + auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_cert = + generate_endorsed_cert(channel2_kp, "CN=Node1", network_kp, service_cert); + + auto channels1 = NodeToNodeChannelManager(wf1); + channels1.initialize(nid1, service_cert, channel1_kp, channel1_cert); + auto channels2 = NodeToNodeChannelManager(wf2); + channels2.initialize(nid2, service_cert, channel2_kp, channel2_cert); + + std::vector msg_body; + msg_body.push_back(0x1); + msg_body.push_back(0x2); + msg_body.push_back(0x10); + msg_body.push_back(0x42); + + INFO("Send an initial request, starting a handshake"); + REQUIRE(channels1.send_encrypted( + nid2, NodeMsgType::forwarded_msg, {aad.begin(), aad.size()}, msg_body)); + + INFO("Send a second request, triggering a second handshake"); + REQUIRE(channels1.send_encrypted( + nid2, NodeMsgType::forwarded_msg, {aad.begin(), aad.size()}, msg_body)); + + INFO("Receive first init message"); + auto q = read_outbound_msgs(eio1); + REQUIRE(q.size() == 2); + + const auto init1 = q[0]; + REQUIRE(init1.type == NodeMsgType::channel_msg); + REQUIRE(channels2.recv_channel_message(init1.from, init1.data())); + + INFO("Receive response to first handshake"); + const auto resp1 = get_first(eio2, NodeMsgType::channel_msg); + REQUIRE_FALSE(channels1.recv_channel_message(resp1.from, resp1.data())); + + INFO("Receive second init message"); + const auto init2 = q[1]; + REQUIRE(init2.type == NodeMsgType::channel_msg); + REQUIRE(channels2.recv_channel_message(init2.from, init2.data())); + + INFO("Receive response to second handshake"); + const auto resp2 = get_first(eio2, NodeMsgType::channel_msg); + REQUIRE(channels1.recv_channel_message(resp2.from, resp2.data())); + + INFO("Receive final"); + q = read_outbound_msgs(eio1); + REQUIRE(q.size() == 3); + + const auto fin = q[0]; + REQUIRE(fin.type == NodeMsgType::channel_msg); + REQUIRE(channels2.recv_channel_message(fin.from, fin.data())); + + INFO("Decrypt original message"); + const auto received = q[1]; + REQUIRE(received.type == NodeMsgType::forwarded_msg); + const auto decrypted = channels2.recv_encrypted( + received.from, + {received.authenticated_hdr.data(), received.authenticated_hdr.size()}, + received.payload.data(), + received.payload.size()); + + REQUIRE(decrypted == msg_body); +} + TEST_CASE_FIXTURE(IORingbuffersFixture, "Expired certs") { auto network_kp = crypto::make_key_pair(default_curve); From afe05d824dc996791b97c63a6ee15777689ec7cc Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:29:34 +0100 Subject: [PATCH 037/135] [release/4.x] Cherry pick: Add nested configurations field (#5445) (#5447) --- doc/operations/certificates.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/operations/certificates.rst b/doc/operations/certificates.rst index 56ea8cd8c25a..90155c8fe83b 100644 --- a/doc/operations/certificates.rst +++ b/doc/operations/certificates.rst @@ -116,15 +116,17 @@ To be able to bind to that port, the ``cchost`` binary may need to be given spec } }, "acme": { - "my-acme-cfg": { - "ca_certs": [ "-----BEGIN CERTIFICATE-----\nMIIBg ..." ], - "directory_url": "https://...", - "service_dns_name": "my-ccf.example.com", - "alternative_names": [ "www.my-ccf.example.com", ... ] - "contact": ["mailto:john@example.com"], - "terms_of_service_agreed": true, - "challenge_type": "http-01", - "challenge_server_interface": "acme_challenge_server_interface" + "configurations": { + "my-acme-cfg": { + "ca_certs": [ "-----BEGIN CERTIFICATE-----\nMIIBg ..." ], + "directory_url": "https://...", + "service_dns_name": "my-ccf.example.com", + "alternative_names": [ "www.my-ccf.example.com", ... ], + "contact": ["mailto:john@example.com"], + "terms_of_service_agreed": true, + "challenge_type": "http-01", + "challenge_server_interface": "acme_challenge_server_interface" + } } } } From 91b674996fe78ae0ebffa28c57cd35d7d22fa74f Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 13 Jul 2023 10:54:46 +0100 Subject: [PATCH 038/135] [release/4.x] Cherry pick: Store deadlock fix (#5413) (#5450) --- src/kv/apply_changes.h | 25 ++++++- src/kv/committable_tx.h | 12 +++- src/kv/kv_types.h | 5 +- src/kv/store.h | 97 ++++++++++++++++------------ src/kv/test/stub_consensus.h | 1 - src/node/history.h | 27 ++++++-- src/node/test/historical_queries.cpp | 8 +++ src/node/test/history.cpp | 64 +++++++++--------- src/node/test/snapshot.cpp | 4 +- src/node/test/snapshotter.cpp | 3 + 10 files changed, 162 insertions(+), 84 deletions(-) diff --git a/src/kv/apply_changes.h b/src/kv/apply_changes.h index bc39fc438d7b..ce6bf58be3bc 100644 --- a/src/kv/apply_changes.h +++ b/src/kv/apply_changes.h @@ -41,7 +41,8 @@ namespace kv const MapCollection& new_maps, const std::optional& new_maps_conflict_version, bool track_read_versions, - bool track_deletes_on_missing_keys) + bool track_deletes_on_missing_keys, + const std::optional& expected_rollback_count = std::nullopt) { // All maps with pending writes are locked, transactions are prepared // and possibly committed, and then all maps with pending writes are @@ -70,7 +71,27 @@ namespace kv } bool ok = true; - if (has_writes) + + if (expected_rollback_count.has_value() && !changes.empty()) + { + // expected_rollback_count is only set on signature transactions + // which always contain some writes, and on which all the maps + // point to the same store. + auto store = changes.begin()->second.map->get_store(); + if (store != nullptr) + { + // Note that this is done when holding the lock on at least some maps + // through the combination of the changes not being empty, and the + // acquisition of the map locks on line 69. This guarantees atomicity + // with respect to rollbacks, which would acquire the map lock on all + // maps at once to truncate their roll. The net result is that the + // transaction becomes a noop if a rollback occurred between it being + // committed, and the side effects being applied. + ok = store->check_rollback_count(expected_rollback_count.value()); + } + } + + if (ok && has_writes) { for (auto it = views.begin(); it != views.end(); ++it) { diff --git a/src/kv/committable_tx.h b/src/kv/committable_tx.h index 671abb6308ff..924955bf9e8c 100644 --- a/src/kv/committable_tx.h +++ b/src/kv/committable_tx.h @@ -388,14 +388,21 @@ namespace kv // never conflict. class ReservedTx : public CommittableTx { + private: + Version rollback_count = 0; + public: ReservedTx( - AbstractStore* _store, Term read_term, const TxID& reserved_tx_id) : + AbstractStore* _store, + Term read_term, + const TxID& reserved_tx_id, + Version rollback_count_) : CommittableTx(_store) { version = reserved_tx_id.version; pimpl->commit_view = reserved_tx_id.term; pimpl->read_txid = TxID(read_term, reserved_tx_id.version - 1); + rollback_count = rollback_count_; } // Used by frontend to commit reserved transactions @@ -417,7 +424,8 @@ namespace kv pimpl->created_maps, version, track_read_versions, - track_deletes_on_missing_keys); + track_deletes_on_missing_keys, + rollback_count); success = c.has_value(); if (!success) diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index 86bdfae30667..e58de2273f53 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -425,7 +425,9 @@ namespace kv const std::vector& hash_at_snapshot) = 0; virtual std::vector get_raw_leaf(uint64_t index) = 0; virtual void append(const std::vector& data) = 0; - virtual void append_entry(const crypto::Sha256Hash& digest) = 0; + virtual void append_entry( + const crypto::Sha256Hash& digest, + std::optional expected_term = std::nullopt) = 0; virtual void rollback( const kv::TxID& tx_id, kv::Term term_of_next_version_) = 0; virtual void compact(Version v) = 0; @@ -731,6 +733,7 @@ namespace kv const TxID& txid, std::unique_ptr pending_tx, bool globally_committable) = 0; + virtual bool check_rollback_count(Version count) = 0; virtual std::unique_ptr snapshot_unsafe_maps( Version v) = 0; diff --git a/src/kv/store.h b/src/kv/store.h index 675a9dae1975..30a8e962529a 100644 --- a/src/kv/store.h +++ b/src/kv/store.h @@ -944,6 +944,10 @@ namespace kv Version previous_rollback_count = 0; ccf::View replication_view = 0; + std::vector, bool>> + contiguous_pending_txs; + auto h = get_history(); + { std::lock_guard vguard(version_lock); if (txid.term != term_of_next_version && get_consensus()->is_primary()) @@ -968,8 +972,6 @@ namespace kv std::make_tuple(std::move(pending_tx), globally_committable)}); LOG_TRACE_FMT("Inserting pending tx at {}", txid.version); - - auto h = get_history(); auto c = get_consensus(); for (Version offset = 1; true; ++offset) @@ -988,52 +990,61 @@ namespace kv break; } - auto& [pending_tx_, committable_] = search->second; - auto - [success_, data_, claims_digest_, commit_evidence_digest_, hooks_] = - pending_tx_->call(); - auto data_shared = - std::make_shared>(std::move(data_)); - auto hooks_shared = - std::make_shared(std::move(hooks_)); - - // NB: this cannot happen currently. Regular Tx only make it here if - // they did succeed, and signatures cannot conflict because they - // execute in order with a read_version that's version - 1, so even - // two contiguous signatures are fine - if (success_ != CommitResult::SUCCESS) - { - LOG_DEBUG_FMT("Failed Tx commit {}", last_replicated + offset); - } + contiguous_pending_txs.emplace_back(std::move(search->second)); + pending_txs.erase(search); + } - if (h) - { - h->append_entry(ccf::entry_leaf( - *data_shared, commit_evidence_digest_, claims_digest_)); - } + previous_rollback_count = rollback_count; + previous_last_replicated = last_replicated; + next_last_replicated = last_replicated + contiguous_pending_txs.size(); - LOG_DEBUG_FMT( - "Batching {} ({}) during commit of {}.{}", - last_replicated + offset, - data_shared->size(), - txid.term, - txid.version); + replication_view = term_of_next_version; + } + // Release version lock - batch.emplace_back( - last_replicated + offset, data_shared, committable_, hooks_shared); - pending_txs.erase(search); + if (contiguous_pending_txs.size() == 0) + { + return CommitResult::SUCCESS; + } + + size_t offset = 1; + for (auto& [pending_tx_, committable_] : contiguous_pending_txs) + { + auto + [success_, data_, claims_digest_, commit_evidence_digest_, hooks_] = + pending_tx_->call(); + auto data_shared = + std::make_shared>(std::move(data_)); + auto hooks_shared = + std::make_shared(std::move(hooks_)); + + // NB: this cannot happen currently. Regular Tx only make it here if + // they did succeed, and signatures cannot conflict because they + // execute in order with a read_version that's version - 1, so even + // two contiguous signatures are fine + if (success_ != CommitResult::SUCCESS) + { + LOG_DEBUG_FMT("Failed Tx commit {}", last_replicated + offset); } - if (batch.size() == 0) + if (h) { - return CommitResult::SUCCESS; + h->append_entry( + ccf::entry_leaf( + *data_shared, commit_evidence_digest_, claims_digest_), + replication_view); } - previous_rollback_count = rollback_count; - previous_last_replicated = last_replicated; - next_last_replicated = last_replicated + batch.size(); + LOG_DEBUG_FMT( + "Batching {} ({}) during commit of {}.{}", + last_replicated + offset, + data_shared->size(), + txid.term, + txid.version); - replication_view = term_of_next_version; + batch.emplace_back( + last_replicated + offset, data_shared, committable_, hooks_shared); + offset++; } if (c->replicate(batch, replication_view)) @@ -1087,6 +1098,12 @@ namespace kv maps_lock.unlock(); } + bool check_rollback_count(Version count) override + { + std::lock_guard vguard(version_lock); + return rollback_count == count; + } + std::tuple next_version(bool commit_new_map) override { std::lock_guard vguard(version_lock); @@ -1288,7 +1305,7 @@ namespace kv { // version_lock should already been acquired in case term_of_last_version // is incremented. - return ReservedTx(this, term_of_last_version, tx_id); + return ReservedTx(this, term_of_last_version, tx_id, rollback_count); } virtual void set_flag(Flag f) override diff --git a/src/kv/test/stub_consensus.h b/src/kv/test/stub_consensus.h index 0f26bca4c1ad..978634741924 100644 --- a/src/kv/test/stub_consensus.h +++ b/src/kv/test/stub_consensus.h @@ -23,7 +23,6 @@ namespace kv::test std::vector replica; ccf::TxID committed_txid = {}; ccf::View current_view = 0; - ccf::SeqNo last_signature = 0; aft::ViewHistory view_history; diff --git a/src/node/history.h b/src/node/history.h index 1f9103854d89..2608aa9e9060 100644 --- a/src/node/history.h +++ b/src/node/history.h @@ -134,7 +134,9 @@ namespace ccf version++; } - void append_entry(const crypto::Sha256Hash& digest) override + void append_entry( + const crypto::Sha256Hash& digest, + std::optional term_of_next_version = std::nullopt) override { version++; } @@ -677,8 +679,15 @@ namespace ccf std::vector serialise_tree(size_t to) override { std::lock_guard guard(state_lock); - return replicated_state_tree.serialise( - replicated_state_tree.begin_index(), to); + if (to <= replicated_state_tree.end_index()) + { + return replicated_state_tree.serialise( + replicated_state_tree.begin_index(), to); + } + else + { + return {}; + } } void set_term(kv::Term t) override @@ -784,10 +793,20 @@ namespace ccf replicated_state_tree.append(rh); } - void append_entry(const crypto::Sha256Hash& digest) override + void append_entry( + const crypto::Sha256Hash& digest, + std::optional expected_term_of_next_version = + std::nullopt) override { log_hash(digest, APPEND); std::lock_guard guard(state_lock); + if (expected_term_of_next_version.has_value()) + { + if (expected_term_of_next_version.value() != term_of_next_version) + { + return; + } + } replicated_state_tree.append(digest); } diff --git a/src/node/test/historical_queries.cpp b/src/node/test/historical_queries.cpp index f7e238845edd..c4fc595006ec 100644 --- a/src/node/test/historical_queries.cpp +++ b/src/node/test/historical_queries.cpp @@ -58,6 +58,7 @@ TestState create_and_init_state(bool initialise_ledger_rekey = true) std::make_shared(*ts.kv_store, node_id, *ts.node_kp); h->set_endorsed_certificate({}); ts.kv_store->set_history(h); + ts.kv_store->initialise_term(2); { INFO("Store the signing node's key"); @@ -122,6 +123,7 @@ kv::Version write_transactions(kv::Store& kv_store, size_t tx_count) REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); } + REQUIRE(kv_store.current_version() == end); return kv_store.current_version(); } @@ -219,6 +221,7 @@ std::map> construct_host_ledger( std::map> ledger; auto next_ledger_entry = consensus->pop_oldest_entry(); + auto version = std::get<0>(next_ledger_entry.value()); while (next_ledger_entry.has_value()) { const auto ib = ledger.insert(std::make_pair( @@ -226,6 +229,11 @@ std::map> construct_host_ledger( *std::get<1>(next_ledger_entry.value()))); REQUIRE(ib.second); next_ledger_entry = consensus->pop_oldest_entry(); + if (next_ledger_entry.has_value()) + { + REQUIRE(version + 1 == std::get<0>(next_ledger_entry.value())); + version = std::get<0>(next_ledger_entry.value()); + } } return ledger; diff --git a/src/node/test/history.cpp b/src/node/test/history.cpp index 117a624592ba..1939290a868f 100644 --- a/src/node/test/history.cpp +++ b/src/node/test/history.cpp @@ -3,6 +3,7 @@ #include "node/history.h" #include "ccf/app_interface.h" +#include "ccf/ds/logger.h" #include "ccf/service/tables/nodes.h" #include "kv/kv_types.h" #include "kv/store.h" @@ -70,38 +71,39 @@ TEST_CASE("Check signature verification") { auto encryptor = std::make_shared(); + auto kp = crypto::make_key_pair(); + const auto self_signed = kp->self_sign("CN=Node", valid_from, valid_to); + kv::Store primary_store; primary_store.set_encryptor(encryptor); + constexpr auto store_term = 2; + std::shared_ptr primary_history = + std::make_shared( + primary_store, kv::test::PrimaryNodeId, *kp); + primary_history->set_endorsed_certificate(self_signed); + primary_store.set_history(primary_history); + primary_store.initialise_term(store_term); kv::Store backup_store; backup_store.set_encryptor(encryptor); + std::shared_ptr backup_history = + std::make_shared( + backup_store, kv::test::FirstBackupNodeId, *kp); + backup_history->set_endorsed_certificate(self_signed); + backup_store.set_history(backup_history); + backup_store.initialise_term(store_term); ccf::Nodes nodes(ccf::Tables::NODES); ccf::Signatures signatures(ccf::Tables::SIGNATURES); - auto kp = crypto::make_key_pair(); - - const auto self_signed = kp->self_sign("CN=Node", valid_from, valid_to); - std::shared_ptr consensus = std::make_shared(&backup_store); primary_store.set_consensus(consensus); + std::shared_ptr null_consensus = std::make_shared(nullptr); backup_store.set_consensus(null_consensus); - std::shared_ptr primary_history = - std::make_shared( - primary_store, kv::test::PrimaryNodeId, *kp); - primary_history->set_endorsed_certificate(self_signed); - primary_store.set_history(primary_history); - - std::shared_ptr backup_history = - std::make_shared( - backup_store, kv::test::FirstBackupNodeId, *kp); - backup_history->set_endorsed_certificate(self_signed); - backup_store.set_history(backup_history); - INFO("Write certificate"); { auto txs = primary_store.create_tx(); @@ -133,21 +135,31 @@ TEST_CASE("Check signature verification") TEST_CASE("Check signing works across rollback") { auto encryptor = std::make_shared(); + + auto kp = crypto::make_key_pair(); + const auto self_signed = kp->self_sign("CN=Node", valid_from, valid_to); + kv::Store primary_store; primary_store.set_encryptor(encryptor); constexpr auto store_term = 2; + std::shared_ptr primary_history = + std::make_shared( + primary_store, kv::test::PrimaryNodeId, *kp); + primary_history->set_endorsed_certificate(self_signed); + primary_store.set_history(primary_history); primary_store.initialise_term(store_term); kv::Store backup_store; + std::shared_ptr backup_history = + std::make_shared( + backup_store, kv::test::FirstBackupNodeId, *kp); + backup_history->set_endorsed_certificate(self_signed); + backup_store.set_history(backup_history); backup_store.set_encryptor(encryptor); backup_store.initialise_term(store_term); ccf::Nodes nodes(ccf::Tables::NODES); - auto kp = crypto::make_key_pair(); - - const auto self_signed = kp->self_sign("CN=Node", valid_from, valid_to); - std::shared_ptr consensus = std::make_shared(&backup_store); primary_store.set_consensus(consensus); @@ -155,18 +167,6 @@ TEST_CASE("Check signing works across rollback") std::make_shared(nullptr); backup_store.set_consensus(null_consensus); - std::shared_ptr primary_history = - std::make_shared( - primary_store, kv::test::PrimaryNodeId, *kp); - primary_history->set_endorsed_certificate(self_signed); - primary_store.set_history(primary_history); - - std::shared_ptr backup_history = - std::make_shared( - backup_store, kv::test::FirstBackupNodeId, *kp); - backup_history->set_endorsed_certificate(self_signed); - backup_store.set_history(backup_history); - INFO("Write certificate"); { auto txs = primary_store.create_tx(); diff --git a/src/node/test/snapshot.cpp b/src/node/test/snapshot.cpp index 367b3b6d554e..139c8365004c 100644 --- a/src/node/test/snapshot.cpp +++ b/src/node/test/snapshot.cpp @@ -30,8 +30,8 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) auto source_history = std::make_shared( source_store, source_node_id, *source_node_kp); source_history->set_endorsed_certificate({}); - source_store.set_history(source_history); + source_store.initialise_term(2); kv::Map string_map("public:string_map"); @@ -79,7 +79,7 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) target_tree.append(ccf::entry_leaf( serialised_signature, - crypto::Sha256Hash("ce:0.4:"), + crypto::Sha256Hash("ce:2.4:"), ccf::empty_claims())); REQUIRE( target_tree.get_root() == source_history->get_replicated_state_root()); diff --git a/src/node/test/snapshotter.cpp b/src/node/test/snapshotter.cpp index 5c48d2ebdda0..5ea06f53ddff 100644 --- a/src/node/test/snapshotter.cpp +++ b/src/node/test/snapshotter.cpp @@ -142,6 +142,7 @@ TEST_CASE("Regular snapshotting") auto history = std::make_shared( *network.tables.get(), kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); + network.tables->initialise_term(2); network.tables->set_consensus(consensus); auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); @@ -304,6 +305,7 @@ TEST_CASE("Rollback before snapshot is committed") auto history = std::make_shared( *network.tables.get(), kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); + network.tables->initialise_term(2); network.tables->set_consensus(consensus); auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); @@ -434,6 +436,7 @@ TEST_CASE("Rekey ledger while snapshot is in progress") auto history = std::make_shared( *network.tables.get(), kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); + network.tables->initialise_term(2); network.tables->set_consensus(consensus); auto ledger_secrets = std::make_shared(); ledger_secrets->init(); From 4a5d13a8b06c68f9cb9b0536f63a834bd9973180 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Jul 2023 10:32:32 +0100 Subject: [PATCH 039/135] [release/4.x] Cherry pick: Remove shadowing instances from the code (#5449) (#5454) --- cmake/preproject.cmake | 1 + src/apps/tpcc/app/tpcc_setup.h | 20 ++++++------- src/ds/state_machine.h | 30 ++++++++++--------- src/host/ledger.h | 4 +-- src/host/lfs_file_handler.h | 7 +++-- src/host/main.cpp | 2 +- src/host/tcp.h | 28 ++++++++--------- src/host/udp.h | 17 +++++------ .../strategies/seqnos_by_key_bucketed.cpp | 3 +- src/kv/store.h | 5 ++-- src/kv/untyped_map.h | 16 +++++----- src/node/acme_client.h | 4 +-- src/node/historical_queries.h | 14 ++++----- src/node/history.h | 2 +- src/node/node_state.h | 15 +++++----- src/node/rpc/member_frontend.h | 28 ++++++++--------- src/node/rpc/node_frontend.h | 2 -- src/node/share_manager.h | 4 +-- 18 files changed, 101 insertions(+), 101 deletions(-) diff --git a/cmake/preproject.cmake b/cmake/preproject.cmake index 462f199138c1..a682bd061357 100644 --- a/cmake/preproject.cmake +++ b/cmake/preproject.cmake @@ -76,6 +76,7 @@ function(add_warning_checks name) -Wpedantic -Wno-unused -Wno-unused-parameter + -Wshadow ) endfunction() diff --git a/src/apps/tpcc/app/tpcc_setup.h b/src/apps/tpcc/app/tpcc_setup.h index 5bc4716b7a62..89dce4d82ada 100644 --- a/src/apps/tpcc/app/tpcc_setup.h +++ b/src/apps/tpcc/app/tpcc_setup.h @@ -73,10 +73,10 @@ namespace tpcc } std::unordered_set select_unique_ids( - uint32_t num_items, uint32_t num_unique) + uint32_t num_items_, uint32_t num_unique) { std::unordered_set r; - for (uint32_t i = 0; i < num_items; ++i) + for (uint32_t i = 0; i < num_items_; ++i) { r.insert(i); } @@ -408,18 +408,18 @@ namespace tpcc if (new_order) { - tpcc::TpccTables::DistributeKey table_key; - table_key.v.w_id = w_id; - table_key.v.d_id = d_id; - auto it = tpcc::TpccTables::new_orders.find(table_key.k); - if (it == tpcc::TpccTables::new_orders.end()) + tpcc::TpccTables::DistributeKey table_key_; + table_key_.v.w_id = w_id; + table_key_.v.d_id = d_id; + auto it_ = tpcc::TpccTables::new_orders.find(table_key_.k); + if (it_ == tpcc::TpccTables::new_orders.end()) { std::string tbl_name = fmt::format("new_orders_{}_{}", w_id, d_id); auto r = tpcc::TpccTables::new_orders.insert( - {table_key.k, + {table_key_.k, TpccMap(tbl_name.c_str())}); - it = r.first; + it_ = r.first; } NewOrder no; @@ -427,7 +427,7 @@ namespace tpcc no.d_id = d_id; no.o_id = o_id; - auto new_orders = args.tx.rw(it->second); + auto new_orders = args.tx.rw(it_->second); new_orders->put(no.get_key(), no); } } diff --git a/src/ds/state_machine.h b/src/ds/state_machine.h index 65a01085e262..8769ba7393fd 100644 --- a/src/ds/state_machine.h +++ b/src/ds/state_machine.h @@ -13,36 +13,38 @@ namespace ds class StateMachine { const std::string label; - std::atomic s; + std::atomic state; public: - StateMachine(std::string&& l, T s) : label(std::move(l)), s(s) {} + StateMachine(const std::string& label_, T state_) : + label(label_), + state(state_) + {} - void expect(T s) const + void expect(T state_) const { - auto state = this->s.load(); - if (s != state) + auto state_snapshot = state.load(); + if (state_ != state_snapshot) { - throw std::logic_error( - fmt::format("[{}] State is {}, but expected {}", label, state, s)); + throw std::logic_error(fmt::format( + "[{}] State is {}, but expected {}", label, state_snapshot, state_)); } } - bool check(T s) const + bool check(T state_) const { - return s == this->s.load(); + return state_ == state.load(); } T value() const { - return this->s.load(); + return state.load(); } - void advance(T s) + void advance(T state_) { - LOG_DEBUG_FMT( - "[{}] Advancing to state {} (from {})", label, s, this->s.load()); - this->s.store(s); + LOG_DEBUG_FMT("[{}] Advancing to state {}", label, state_); + state.store(state_); } }; } diff --git a/src/host/ledger.h b/src/host/ledger.h index ba82d7056b75..0e3d3d5e7f66 100644 --- a/src/host/ledger.h +++ b/src/host/ledger.h @@ -1530,9 +1530,9 @@ namespace asynchost DISPATCHER_SET_MESSAGE_HANDLER( disp, consensus::ledger_init, [this](const uint8_t* data, size_t size) { auto idx = serialized::read(data, size); - auto recovery_start_idx = + auto recovery_start_index = serialized::read(data, size); - init(idx, recovery_start_idx); + init(idx, recovery_start_index); }); DISPATCHER_SET_MESSAGE_HANDLER( diff --git a/src/host/lfs_file_handler.h b/src/host/lfs_file_handler.h index 1e971161e560..bb1ae6eca1b1 100644 --- a/src/host/lfs_file_handler.h +++ b/src/host/lfs_file_handler.h @@ -60,11 +60,12 @@ namespace asynchost { std::ifstream f(target_path, std::ios::binary); f.seekg(0, f.end); - const auto size = f.tellg(); - LOG_TRACE_FMT("Reading {} byte file from {}", size, target_path); + const auto file_size = f.tellg(); + LOG_TRACE_FMT( + "Reading {} byte file from {}", file_size, target_path); f.seekg(0, f.beg); - ccf::indexing::LFSEncryptedContents blob(size); + ccf::indexing::LFSEncryptedContents blob(file_size); f.read((char*)blob.data(), blob.size()); f.close(); RINGBUFFER_WRITE_MESSAGE( diff --git a/src/host/main.cpp b/src/host/main.cpp index 04d7104b4f6d..1d99007b4db6 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -130,7 +130,7 @@ int main(int argc, char** argv) std::string config_parsing_error = ""; do { - std::string config_str = files::slurp_string( + config_str = files::slurp_string( config_file_path, true /* return an empty string if the file does not exist */); try diff --git a/src/host/tcp.h b/src/host/tcp.h index 03b2132058a7..7b2f51a1d6a5 100644 --- a/src/host/tcp.h +++ b/src/host/tcp.h @@ -232,17 +232,17 @@ namespace asynchost void start(int64_t id) {} bool connect( - const std::string& host, - const std::string& port, - const std::optional& client_host = std::nullopt) + const std::string& host_, + const std::string& port_, + const std::optional& client_host_ = std::nullopt) { // If a client host is set, bind to this first. Otherwise, connect // straight away. - if (client_host.has_value()) + if (client_host_.has_value()) { - this->client_host = client_host; - this->host = host; - this->port = port; + client_host = client_host_; + host = host_; + port = port_; if (client_addr_base != nullptr) { @@ -262,7 +262,7 @@ namespace asynchost else { assert_status(FRESH, CONNECTING_RESOLVING); - return resolve(host, port, true); + return resolve(host_, port_, true); } return true; @@ -314,12 +314,12 @@ namespace asynchost } bool listen( - const std::string& host, - const std::string& port, + const std::string& host_, + const std::string& port_, const std::optional& name = std::nullopt) { assert_status(FRESH, LISTENING_RESOLVING); - bool ret = resolve(host, port, false); + bool ret = resolve(host_, port_, false); listen_name = name; return ret; } @@ -545,10 +545,10 @@ namespace asynchost } bool resolve( - const std::string& host, const std::string& port, bool async = true) + const std::string& host_, const std::string& port_, bool async = true) { - this->host = host; - this->port = port; + host = host_; + port = port_; if (addr_base != nullptr) { diff --git a/src/host/udp.h b/src/host/udp.h index 4805c5e00756..adc62219ce5f 100644 --- a/src/host/udp.h +++ b/src/host/udp.h @@ -148,14 +148,14 @@ namespace asynchost /// Listen to packets on host:port bool listen( - const std::string& host, - const std::string& port, + const std::string& host_, + const std::string& port_, const std::optional& name = std::nullopt) { listen_name = name; auto name_str = name.has_value() ? name.value() : ""; - LOG_TRACE_FMT("UDP listen on {}:{} [{}]", host, port, name_str); - return resolve(host, port, false); + LOG_TRACE_FMT("UDP listen on {}:{} [{}]", host_, port_, name_str); + return resolve(host_, port_, false); } /// Start the service via behaviour (register on ringbuffer, etc) @@ -164,8 +164,7 @@ namespace asynchost behaviour->on_start(id); } - /// Dummy for now - bool connect(const std::string& host, const std::string& port) + bool connect(const std::string& host_, const std::string& port_) { LOG_TRACE_FMT("UDP dummy connect to {}:{}", host, port); return true; @@ -327,10 +326,10 @@ namespace asynchost } bool resolve( - const std::string& host, const std::string& port, bool async = true) + const std::string& host_, const std::string& port_, bool async = true) { - this->host = host; - this->port = port; + host = host_; + port = port_; LOG_TRACE_FMT("UDP resolve {}:{}", host, port); if (addr_base != nullptr) diff --git a/src/indexing/strategies/seqnos_by_key_bucketed.cpp b/src/indexing/strategies/seqnos_by_key_bucketed.cpp index a1a2b59dcb01..020ca9a31eef 100644 --- a/src/indexing/strategies/seqnos_by_key_bucketed.cpp +++ b/src/indexing/strategies/seqnos_by_key_bucketed.cpp @@ -274,7 +274,8 @@ namespace ccf::indexing::strategies // strategies currently used, this re-indexes everything from // the start of time. { - std::lock_guard guard(current_txid_lock); + std::lock_guard current_txid_guard( + current_txid_lock); current_txid = {}; } old_results.clear(); diff --git a/src/kv/store.h b/src/kv/store.h index 30a8e962529a..edb71f32f26f 100644 --- a/src/kv/store.h +++ b/src/kv/store.h @@ -706,9 +706,9 @@ namespace kv } } - for (auto& it : maps) + for (auto& map_it : maps) { - auto& [_, map] = it.second; + auto& [_, map] = map_it.second; map->unlock(); } } @@ -972,7 +972,6 @@ namespace kv std::make_tuple(std::move(pending_tx), globally_committable)}); LOG_TRACE_FMT("Inserting pending tx at {}", txid.version); - auto c = get_consensus(); for (Version offset = 1; true; ++offset) { diff --git a/src/kv/untyped_map.h b/src/kv/untyped_map.h index ed494b36e5bd..b8fbe76a05ba 100644 --- a/src/kv/untyped_map.h +++ b/src/kv/untyped_map.h @@ -147,15 +147,15 @@ namespace kv::untyped bool prepare(bool track_read_versions) override { - auto& roll = map.get_roll(); + auto& map_roll = map.get_roll(); // If the parent map has rolled back since this transaction began, this // transaction must fail. - if (change_set.rollback_counter != roll.rollback_counter) + if (change_set.rollback_counter != map_roll.rollback_counter) return false; // If we have iterated over the map, check for a global version match. - auto current = roll.commits->get_tail(); + auto current = map_roll.commits->get_tail(); if ( (change_set.read_version != NoVersion) && (change_set.read_version != current->version)) @@ -211,8 +211,8 @@ namespace kv::untyped return; } - auto& roll = map.get_roll(); - auto state = roll.commits->get_tail()->state; + auto& map_roll = map.get_roll(); + auto state = map_roll.commits->get_tail()->state; // To track conflicts the read version of all keys that are read or // written within a transaction must be updated. @@ -590,11 +590,11 @@ namespace kv::untyped /** Set handler to be called on global transaction commit * - * @param hook function to be called on global transaction commit + * @param hook_ function to be called on global transaction commit */ - void set_global_hook(const CommitHook& hook) + void set_global_hook(const CommitHook& hook_) { - global_hook = hook; + global_hook = hook_; } /** Reset global transaction commit handler diff --git a/src/node/acme_client.h b/src/node/acme_client.h index 4bb40371a638..5305159efee5 100644 --- a/src/node/acme_client.h +++ b/src/node/acme_client.h @@ -74,7 +74,7 @@ namespace ACME virtual ~Client() {} void get_certificate( - std::shared_ptr service_key, bool override_time = false) + std::shared_ptr service_key_, bool override_time = false) { using namespace std::chrono_literals; using namespace std::chrono; @@ -111,7 +111,7 @@ namespace ACME if (ok) { - this->service_key = service_key; + service_key = service_key_; last_request = system_clock::now(); num_failed_attempts++; request_directory(); diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h index 84da7f22f5cf..3ab374396bc7 100644 --- a/src/node/historical_queries.h +++ b/src/node/historical_queries.h @@ -629,18 +629,18 @@ namespace ccf::historical auto my_stores_it = request.my_stores.begin(); while (my_stores_it != request.my_stores.end()) { - auto [seqno, _] = *my_stores_it; - auto it = all_stores.find(seqno); - auto details = + auto [store_seqno, _] = *my_stores_it; + auto it = all_stores.find(store_seqno); + auto store_details = it == all_stores.end() ? nullptr : it->second.lock(); - if (details == nullptr) + if (store_details == nullptr) { - details = std::make_shared(); - all_stores.insert_or_assign(it, seqno, details); + store_details = std::make_shared(); + all_stores.insert_or_assign(it, store_seqno, store_details); } - my_stores_it->second = details; + my_stores_it->second = store_details; ++my_stores_it; } } diff --git a/src/node/history.h b/src/node/history.h index 2608aa9e9060..4b5d495a8318 100644 --- a/src/node/history.h +++ b/src/node/history.h @@ -136,7 +136,7 @@ namespace ccf void append_entry( const crypto::Sha256Hash& digest, - std::optional term_of_next_version = std::nullopt) override + std::optional term_of_next_version_ = std::nullopt) override { version++; } diff --git a/src/node/node_state.h b/src/node/node_state.h index c983528b3bd9..c106e3703bee 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -240,12 +240,12 @@ namespace ccf QuoteVerificationResult verify_quote( kv::ReadOnlyTx& tx, - const QuoteInfo& quote_info, + const QuoteInfo& quote_info_, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) override { return AttestationProvider::verify_quote_against_store( - tx, quote_info, expected_node_public_key_der, measurement); + tx, quote_info_, expected_node_public_key_der, measurement); } // @@ -701,7 +701,7 @@ namespace ccf } View view = VIEW_UNKNOWN; - std::vector view_history = {}; + std::vector view_history_ = {}; if (startup_snapshot_info) { // It is only possible to deserialise the entire snapshot then, @@ -711,7 +711,7 @@ namespace ccf network.tables, startup_snapshot_info->raw, hooks, - &view_history, + &view_history_, resp.network_info->public_only, config.recover.previous_service_identity); @@ -748,7 +748,7 @@ namespace ccf consensus->init_as_backup( network.tables->current_version(), view, - view_history, + view_history_, last_recovered_signed_idx); snapshotter->set_last_snapshot_idx( @@ -1293,13 +1293,13 @@ namespace ccf if (startup_snapshot_info) { - std::vector view_history; + std::vector view_history_; kv::ConsensusHookPtrs hooks; deserialise_snapshot( recovery_store, startup_snapshot_info->raw, hooks, - &view_history, + &view_history_, false, config.recover.previous_service_identity); startup_snapshot_info.reset(); @@ -2559,7 +2559,6 @@ namespace ccf auto msg = std::make_unique>( [](std::unique_ptr> msg) { auto& state = msg->data.self; - auto& config = state.config; if (state.consensus && state.consensus->can_replicate()) { diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index ec5418a8fd3f..1c57a7c71c16 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -309,10 +309,10 @@ namespace ccf } if (pi_.value().state == ProposalState::ACCEPTED) { - js::Runtime rt(&tx); - js::Context js_context(rt, js::TxAccess::GOV_RW); - rt.add_ccf_classdefs(); - js::TxContext txctx{&tx}; + js::Runtime apply_rt(&tx); + js::Context apply_js_context(apply_rt, js::TxAccess::GOV_RW); + apply_rt.add_ccf_classdefs(); + js::TxContext apply_txctx{&tx}; auto gov_effects = context.get_subsystem(); @@ -323,7 +323,7 @@ namespace ccf } js::populate_global( - &txctx, + &apply_txctx, nullptr, nullptr, std::nullopt, @@ -333,23 +333,23 @@ namespace ccf &network, nullptr, this, - js_context); - auto apply_func = js_context.function( + apply_js_context); + auto apply_func = apply_js_context.function( constitution, "apply", "public:ccf.gov.constitution[0]"); - std::vector argv = { - js_context.new_string_len( + std::vector apply_argv = { + apply_js_context.new_string_len( (const char*)proposal.data(), proposal.size()), - js_context.new_string_len( + apply_js_context.new_string_len( proposal_id.c_str(), proposal_id.size())}; - auto val = js_context.call(apply_func, argv); + auto apply_val = apply_js_context.call(apply_func, apply_argv); - if (JS_IsException(val)) + if (JS_IsException(apply_val)) { pi_.value().state = ProposalState::FAILED; - auto [reason, trace] = js::js_error_message(js_context); - if (js_context.interrupt_data.request_timed_out) + auto [reason, trace] = js::js_error_message(apply_js_context); + if (apply_js_context.interrupt_data.request_timed_out) { reason = "Operation took too long to complete."; } diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 8d4097a93277..5a36da05cb62 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -464,7 +464,6 @@ namespace ccf auto primary_id = consensus->primary(); if (primary_id.has_value()) { - auto nodes = args.tx.ro(this->network.nodes); auto info = nodes->get(primary_id.value()); if (info) { @@ -556,7 +555,6 @@ namespace ccf auto primary_id = consensus->primary(); if (primary_id.has_value()) { - auto nodes = args.tx.ro(this->network.nodes); auto info = nodes->get(primary_id.value()); if (info) { diff --git a/src/node/share_manager.h b/src/node/share_manager.h index e9aa67deef9a..a62ff5c8176a 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -426,12 +426,12 @@ namespace ccf auto decrypted_ls_raw = decrypt_previous_ledger_secret_raw( latest_ls, it->previous_ledger_secret->encrypted_data); - auto s = restored_ledger_secrets.emplace( + auto secret = restored_ledger_secrets.emplace( it->previous_ledger_secret->version, std::make_shared( std::move(decrypted_ls_raw), it->previous_ledger_secret->previous_secret_stored_version)); - latest_ls = s.first->second; + latest_ls = secret.first->second; } return restored_ledger_secrets; From 176cf6a8d0d2c07e893e9888301b3647cdb24584 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 14 Jul 2023 12:04:23 +0100 Subject: [PATCH 040/135] [release/4.x] Cherry pick: Add `getVersionOfPreviousWrite` to TypeScript `TypedKvMap` (#5451) (#5455) --- CHANGELOG.md | 1 + js/ccf-app/src/kv.ts | 4 ++++ tests/js-modules/modules.py | 27 +++++++++++++++++++++++++ tests/npm-app/app.json | 31 +++++++++++++++++++++++++++++ tests/npm-app/src/endpoints/log.ts | 32 ++++++++++++++++++++++++------ 5 files changed, 89 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f98360181f51..b24a6dc7ee11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.5]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.5 - Debug logging is now available in non-SGX builds by default, and controlled by a run-time CLI argument (`--enclave-log-level`). On SGX this remains a build-time decision (#5375). +- Added `getVersionOfPreviousWrite` to TypeScript `TypedKvMap` interface (#5451). ## [4.0.4] diff --git a/js/ccf-app/src/kv.ts b/js/ccf-app/src/kv.ts index 023c88fad7f5..692af4a681bf 100644 --- a/js/ccf-app/src/kv.ts +++ b/js/ccf-app/src/kv.ts @@ -56,6 +56,10 @@ export class TypedKvMap { return v === undefined ? undefined : this.vt.decode(v); } + getVersionOfPreviousWrite(key: K): number | undefined { + return this.kv.getVersionOfPreviousWrite(this.kt.encode(key)); + } + set(key: K, value: V): TypedKvMap { this.kv.set(this.kt.encode(key), this.vt.encode(value)); return this; diff --git a/tests/js-modules/modules.py b/tests/js-modules/modules.py index b945956e8d05..d9ec9e082cfd 100644 --- a/tests/js-modules/modules.py +++ b/tests/js-modules/modules.py @@ -781,19 +781,41 @@ def test_npm_app(network, args): r = c.get("/app/log?id=42") assert r.status_code == http.HTTPStatus.NOT_FOUND, r.status_code + r = c.get("/app/log/version?id=42") + assert r.status_code == http.HTTPStatus.NOT_FOUND, r.status_code + r = c.post("/app/log?id=42", {"msg": "Hello!"}) assert r.status_code == http.HTTPStatus.OK, r.status_code r = c.get("/app/log?id=42") assert r.status_code == http.HTTPStatus.OK, r.status_code body = r.body.json() + assert body["id"] == 42, r.body assert body["msg"] == "Hello!", r.body + r = c.get("/app/log/version?id=42") + assert r.status_code == http.HTTPStatus.OK, r.status_code + v0 = r.body.json()["version"] + r = c.post("/app/log?id=42", {"msg": "Saluton!"}) assert r.status_code == http.HTTPStatus.OK, r.status_code + + r = c.get("/app/log/version?id=42") + assert r.status_code == http.HTTPStatus.OK, r.status_code + v1 = r.body.json()["version"] + assert v1 > v0 + + r = c.get("/app/log/version?id=43") + assert r.status_code == http.HTTPStatus.NOT_FOUND, r.status_code + r = c.post("/app/log?id=43", {"msg": "Bonjour!"}) assert r.status_code == http.HTTPStatus.OK, r.status_code + r = c.get("/app/log/version?id=43") + assert r.status_code == http.HTTPStatus.OK, r.status_code + v2 = r.body.json()["version"] + assert v2 > v1 + r = c.get("/app/log/all") assert r.status_code == http.HTTPStatus.OK, r.status_code body = r.body.json() @@ -802,6 +824,11 @@ def test_npm_app(network, args): assert {"id": 42, "msg": "Saluton!"} in body, body assert {"id": 43, "msg": "Bonjour!"} in body, body + r = c.get("/app/log/version?id=42") + assert r.status_code == http.HTTPStatus.OK, r.status_code + v3 = r.body.json()["version"] + assert v3 == v1 + test_apply_writes(c) r = c.get("/app/jwt") diff --git a/tests/npm-app/app.json b/tests/npm-app/app.json index 6ca670991ce5..576bd0cc239a 100644 --- a/tests/npm-app/app.json +++ b/tests/npm-app/app.json @@ -722,6 +722,37 @@ } } }, + "/log/version": { + "get": { + "js_module": "endpoints/log.js", + "js_function": "getLogItemVersion", + "forwarding_required": "sometimes", + "authn_policies": ["user_cert"], + "mode": "readonly", + "openapi": { + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "properties": { + "id": { + "type": "number" + }, + "version": { + "type": "number" + } + }, + "type": "object" + } + } + } + } + } + } + } + }, "/rpc/apply_writes": { "post": { "js_module": "endpoints/rpc.js", diff --git a/tests/npm-app/src/endpoints/log.ts b/tests/npm-app/src/endpoints/log.ts index 2e2ccd9fe61a..fcba84d2c662 100644 --- a/tests/npm-app/src/endpoints/log.ts +++ b/tests/npm-app/src/endpoints/log.ts @@ -1,16 +1,36 @@ import * as ccfapp from "@microsoft/ccf-app"; +type LogContent = string; + interface LogItem { - msg: string; + msg: LogContent; } interface LogEntry extends LogItem { id: number; } -const logMap = ccfapp.typedKv("log", ccfapp.uint32, ccfapp.json()); +interface LogVersion { + id: number; + version: number; +} + +const logMap = ccfapp.typedKv("log", ccfapp.uint32, ccfapp.json()); -export function getLogItem(request: ccfapp.Request): ccfapp.Response { +export function getLogItem(request: ccfapp.Request): ccfapp.Response { + const id = parseInt(request.query.split("=")[1]); + if (!logMap.has(id)) { + return { + statusCode: 404, + }; + } + return { + body: { id: id, msg: logMap.get(id) }, + }; +} +export function getLogItemVersion( + request: ccfapp.Request, +): ccfapp.Response { const id = parseInt(request.query.split("=")[1]); if (!logMap.has(id)) { return { @@ -18,13 +38,13 @@ export function getLogItem(request: ccfapp.Request): ccfapp.Response { }; } return { - body: logMap.get(id), + body: { id: id, version: logMap.getVersionOfPreviousWrite(id) }, }; } export function setLogItem(request: ccfapp.Request): ccfapp.Response { const id = parseInt(request.query.split("=")[1]); - logMap.set(id, request.body.json()); + logMap.set(id, request.body.json().msg); return {}; } @@ -33,7 +53,7 @@ export function getAllLogItems( ): ccfapp.Response> { let items: Array = []; logMap.forEach(function (item, id) { - items.push({ id: id, msg: item.msg }); + items.push({ id: id, msg: item }); }); return { body: items, From f22bda11a1685c81431c93d1be4199ae9a64c4c3 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Fri, 14 Jul 2023 13:53:50 +0100 Subject: [PATCH 041/135] [release/4.x] Cherry pick: Include intermediate certs in TLS handshake (#5453) (#5457) --- CHANGELOG.md | 1 + src/tls/cert.h | 28 +++++++++++++++++++++++----- tests/acme_endorsement.py | 21 ++++++++++----------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b24a6dc7ee11..a99e877c282e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.5]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.5 - Debug logging is now available in non-SGX builds by default, and controlled by a run-time CLI argument (`--enclave-log-level`). On SGX this remains a build-time decision (#5375). +- Supporting intermediate cert chain included in TLS handshake, where previously only server leaf certificate was present (#5453). - Added `getVersionOfPreviousWrite` to TypeScript `TypedKvMap` interface (#5451). ## [4.0.4] diff --git a/src/tls/cert.h b/src/tls/cert.h index fa76a0c523be..f6745edd0fe9 100644 --- a/src/tls/cert.h +++ b/src/tls/cert.h @@ -28,6 +28,7 @@ namespace tls bool auth_required; Unique_X509 own_cert; + Unique_STACK_OF_X509 chain; std::shared_ptr own_pkey; bool has_own_cert = false; @@ -44,10 +45,26 @@ namespace tls { if (own_cert_.has_value() && own_pkey_.has_value()) { - Unique_BIO certbio(*own_cert_); - own_cert = Unique_X509(certbio, true); - own_pkey = std::make_shared(*own_pkey_); + const auto certs = crypto::split_x509_cert_bundle(own_cert_->str()); has_own_cert = true; + + { + Unique_BIO certbio(certs[0]); + own_cert = Unique_X509(certbio, true); + own_pkey = std::make_shared(*own_pkey_); + } + + if (certs.size() > 1) + { + for (auto it = certs.begin() + 1; it != certs.end(); ++it) + { + Unique_BIO certbio(*it); + Unique_X509 cert(certbio, true); + + CHECK1(sk_X509_push(chain, cert)); + CHECK1(X509_up_ref(cert)); + } + } } } @@ -91,8 +108,9 @@ namespace tls if (has_own_cert) { - CHECK1(SSL_CTX_use_cert_and_key(ssl_ctx, own_cert, *own_pkey, NULL, 1)); - CHECK1(SSL_use_cert_and_key(ssl, own_cert, *own_pkey, NULL, 1)); + CHECK1( + SSL_CTX_use_cert_and_key(ssl_ctx, own_cert, *own_pkey, chain, 1)); + CHECK1(SSL_use_cert_and_key(ssl, own_cert, *own_pkey, chain, 1)); } } }; diff --git a/tests/acme_endorsement.py b/tests/acme_endorsement.py index 83d1a86d882f..ce99b4027f31 100644 --- a/tests/acme_endorsement.py +++ b/tests/acme_endorsement.py @@ -48,7 +48,7 @@ def wait_for_port_to_listen(host, port, timeout=10): @reqs.description("Start network and wait for ACME certificates") def wait_for_certificates( - args, network_name, ca_certs, interface_name, challenge_interface, timeout=5 * 60 + args, network_name, root_cert, interface_name, challenge_interface, timeout=5 ): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb @@ -76,8 +76,7 @@ def wait_for_certificates( context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True - for crt in ca_certs: - context.load_verify_locations(cadata=crt) + context.load_verify_locations(cadata=root_cert) s = socket.socket() s.settimeout(1) @@ -92,9 +91,12 @@ def wait_for_certificates( assert network_public_key == cert_public_key num_ok += 1 except Exception as ex: - LOG.trace(f"Likely expected exception: {ex}") + LOG.warning(f"Likely temporary exception: {ex}") if num_ok != len(args.nodes): + LOG.warning( + f"{num_ok}/{len(args.nodes)} nodes presenting endorsed cert" + ) time.sleep(1) # We can't run test_unsecured_interfaces against the ACME-endorsed interface @@ -195,12 +197,9 @@ def get_without_cert_check(url): return urllib.request.urlopen(url, context=ctx).read().decode("utf-8") -def get_pebble_ca_certs(mgmt_address): +def get_pebble_root_cert(mgmt_address): ca = get_without_cert_check("https://" + mgmt_address + "/roots/0") - intermediate = get_without_cert_check( - "https://" + mgmt_address + "/intermediates/0" - ) - return ca, intermediate + return ca @reqs.description("Test that secure content is not available on an unsecured interface") @@ -321,11 +320,11 @@ def run_pebble(args): ) try: - ca_certs = get_pebble_ca_certs(mgmt_address) + root_cert = get_pebble_root_cert(mgmt_address) wait_for_certificates( args, network_name, - ca_certs, + root_cert, "acme_endorsed_interface", "acme_challenge_server_if", ) From 62f572eb3af00928ddeb273794bcf87b6d67ca3a Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:39:30 +0100 Subject: [PATCH 042/135] [release/4.x] Cherry pick: Fix `lts_compatibility` in SGX Debug - don't pass unsafe `--enclave-log-level` (#5461) (#5462) --- tests/infra/remote.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/infra/remote.py b/tests/infra/remote.py index fa82bb416268..ef42f9293a37 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -831,10 +831,12 @@ def __init__( else None ) if v is None or v >= Version("4.0.5"): - cmd += [ - "--enclave-log-level", - enclave_log_level, - ] + # Avoid passing too-low level to debug SGX nodes + if not (enclave_type == "debug" and enclave_platform == "sgx"): + cmd += [ + "--enclave-log-level", + enclave_log_level, + ] if start_type == StartType.start: members_info = kwargs.get("members_info") From 82c00ded13b9f297ae06167be7e28864f7cc930a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 18 Jul 2023 17:44:41 +0100 Subject: [PATCH 043/135] [release/4.x] Cherry pick: Update to Open Enclave 0.19.3 (#5460) (#5466) --- .azure-pipelines-gh-pages.yml | 6 +++++- .azure-pipelines.yml | 6 +++--- .daily.yml | 6 +++--- .devcontainer/devcontainer.json | 2 +- .github/workflows/build-ci-container.yml | 4 ++-- .github/workflows/ci-checks.yml | 2 +- .multi-thread.yml | 2 +- .stress.yml | 2 +- CHANGELOG.md | 4 ++++ cmake/cpack_settings.cmake | 2 +- cmake/open_enclave.cmake | 2 +- docker/ccf_ci_built | 2 +- getting_started/setup_vm/roles/openenclave/vars/common.yml | 4 ++-- scripts/azure_deployment/arm_aci.py | 2 +- 14 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index 238fa09a591a..2386a1e0d701 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -7,7 +7,11 @@ trigger: jobs: - job: build_and_publish_docs - container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 + displayName: "Build and publish docs" + variables: + Codeql.SkipTaskAutoInjection: true + skipComponentGovernanceDetection: true + container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 31178b434ca8..5ce5abab716d 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,15 +29,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.daily.yml b/.daily.yml index cb15c22a6754..e21670eca6f5 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,15 +25,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b1e7ef63dda3..3589efcaa4f7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml index b0e7c5d0213c..baa35b00b860 100644 --- a/.github/workflows/build-ci-container.yml +++ b/.github/workflows/build-ci-container.yml @@ -45,10 +45,10 @@ jobs: run: docker login -u $ACR_TOKEN_NAME -p ${{ secrets.ACR_CI_PUSH_TOKEN_PASSWORD }} $ACR_REGISTRY - name: Pull CI container - run: docker pull $ACR_REGISTRY/ccf/ci:27-04-2023-snp-clang15 + run: docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.3-snp-clang15 - name: Build CCF CI SNP container - run: docker build -f docker/ccf_ci_built . --build-arg="base=ccfmsrc.azurecr.io/ccf/ci:27-04-2023-snp-clang15" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` + run: docker build -f docker/ccf_ci_built . --build-arg="base=ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` - name: Push CI container run: docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 58def75f64e0..33ae3e4ca09e 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.multi-thread.yml b/.multi-thread.yml index 19d96e67c6f9..abdca1aefaae 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -16,7 +16,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 7aa671ef8b53..828a7234451d 100644 --- a/.stress.yml +++ b/.stress.yml @@ -20,7 +20,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:27-04-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/CHANGELOG.md b/CHANGELOG.md index a99e877c282e..f28241e16d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## Unreleased + +- Updated Open Enclave to [0.19.3](https://github.com/openenclave/openenclave/releases/tag/v0.19.3). + ## [4.0.5] [4.0.5]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.5 diff --git a/cmake/cpack_settings.cmake b/cmake/cpack_settings.cmake index 22b539246ae5..3f9c16d971ae 100644 --- a/cmake/cpack_settings.cmake +++ b/cmake/cpack_settings.cmake @@ -21,7 +21,7 @@ message(STATUS "Debian package version: ${CPACK_DEBIAN_PACKAGE_VERSION}") set(CCF_DEB_BASE_DEPENDENCIES "libuv1 (>= 1.34.2);openssl (>=1.1.1)") set(CCF_DEB_DEPENDENCIES ${CCF_DEB_BASE_DEPENDENCIES}) -set(OE_VERSION "0.19.0") +set(OE_VERSION "0.19.3") if(COMPILE_TARGET STREQUAL "sgx") list(APPEND CCF_DEB_DEPENDENCIES "libc++1-11;libc++abi1-11;open-enclave (>=${OE_VERSION})" diff --git a/cmake/open_enclave.cmake b/cmake/open_enclave.cmake index cbe431276262..fa13387b42bd 100644 --- a/cmake/open_enclave.cmake +++ b/cmake/open_enclave.cmake @@ -6,7 +6,7 @@ if(NOT COMPILE_TARGET STREQUAL "sgx") endif() # Find OpenEnclave package -find_package(OpenEnclave 0.19.0 CONFIG REQUIRED) +find_package(OpenEnclave 0.19.3 CONFIG REQUIRED) # As well as pulling in openenclave:: targets, this sets variables which can be # used for our edge cases (eg - for virtual libraries). These do not follow the # standard naming patterns, for example use OE_INCLUDEDIR rather than diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 99115ff6e145..c16716712801 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:27-04-2023-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/getting_started/setup_vm/roles/openenclave/vars/common.yml b/getting_started/setup_vm/roles/openenclave/vars/common.yml index b8aaec7a5a92..74396a848341 100644 --- a/getting_started/setup_vm/roles/openenclave/vars/common.yml +++ b/getting_started/setup_vm/roles/openenclave/vars/common.yml @@ -1,6 +1,6 @@ -oe_ver: "0.19.0" +oe_ver: "0.19.3" # Usually the same, except for rc, where ver is -rc and ver_ is _rc -oe_ver_: "0.19.0" +oe_ver_: "0.19.3" # Source install workspace: "/tmp/" diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index eff9e5351ed2..77abd366cb90 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -178,7 +178,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:27-04-2023-snp", + default="ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp", ) parser.add_argument( "--aci-type", From 4941ad638b95e653f5ae00c3632197d58e8500e4 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 18 Jul 2023 18:19:53 +0100 Subject: [PATCH 044/135] Release notes for 4.0.6 (#5467) --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f28241e16d75..805a7ea6ef32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## Unreleased +## [4.0.6] + +[4.0.6]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.6 - Updated Open Enclave to [0.19.3](https://github.com/openenclave/openenclave/releases/tag/v0.19.3). From 72b6f35a28f51a082d41ffc2d2904f1d2af4dc13 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 19 Jul 2023 09:36:43 +0100 Subject: [PATCH 045/135] [release/4.x] Cherry pick: Fix SNP `code_update_test` - `ccf.digest` is now `ccf.crypto.digest` (#5464) (#5468) Co-authored-by: Eddy Ashton --- samples/constitutions/default/actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/constitutions/default/actions.js b/samples/constitutions/default/actions.js index 5fdbb52bf5b5..9f99ac5b9b41 100644 --- a/samples/constitutions/default/actions.js +++ b/samples/constitutions/default/actions.js @@ -1038,7 +1038,7 @@ const actions = new Map([ // SHA-256 digest is the specified host data if (args.security_policy != "") { const securityPolicyDigest = ccf.bufToStr( - ccf.digest("SHA-256", ccf.strToBuf(args.security_policy)) + ccf.crypto.digest("SHA-256", ccf.strToBuf(args.security_policy)) ); const hostData = ccf.bufToStr(hexStrToBuf(args.host_data)); if (securityPolicyDigest != hostData) { From b341e1498aafc00931957a4a05c004f1ea533e60 Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Thu, 20 Jul 2023 21:20:31 +0100 Subject: [PATCH 046/135] [release/4.x] Cherry pick: Expose COSE identity content in JS/TS apps (#5465) (#5473) --- CHANGELOG.md | 1 + js/ccf-app/src/endpoints.ts | 15 +++++++++++++-- samples/apps/logging/js/src/logging.js | 2 +- samples/apps/logging/logging.cpp | 4 ++++ src/apps/js_generic/js_generic_base.cpp | 7 +++++++ tests/e2e_logging.py | 2 +- tests/js-modules/modules.py | 7 +++++-- tests/npm-app/src/endpoints/auth.ts | 15 +++++++++++---- tests/npm-app/src/endpoints/rpc.ts | 2 +- 9 files changed, 44 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 805a7ea6ef32..14136e9ca1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.6]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.6 - Updated Open Enclave to [0.19.3](https://github.com/openenclave/openenclave/releases/tag/v0.19.3). +- Expose COSESign1 `content` for `user_cose_sign1` authenticated endpoints in JavaScript/TypeScript apps (#5465). ## [4.0.5] diff --git a/js/ccf-app/src/endpoints.ts b/js/ccf-app/src/endpoints.ts index 6dad593b207c..c2c4b5b75faa 100644 --- a/js/ccf-app/src/endpoints.ts +++ b/js/ccf-app/src/endpoints.ts @@ -141,13 +141,24 @@ export interface MemberCertAuthnIdentity extends UserMemberAuthnIdentityCommon { policy: "member_cert"; } +interface UserMemberCOSEAuthIdentityCommon { + cose: { + /** + * COSE content + */ + content: ArrayBuffer; + }; +} + export interface MemberCOSESign1AuthnIdentity - extends UserMemberAuthnIdentityCommon { + extends UserMemberAuthnIdentityCommon, + UserMemberCOSEAuthIdentityCommon { policy: "member_cose_sign1"; } export interface UserCOSESign1AuthnIdentity - extends UserMemberAuthnIdentityCommon { + extends UserMemberAuthnIdentityCommon, + UserMemberCOSEAuthIdentityCommon { policy: "user_cose_sign1"; } diff --git a/samples/apps/logging/js/src/logging.js b/samples/apps/logging/js/src/logging.js index 1f14e22b1b4e..622393d6a0d2 100644 --- a/samples/apps/logging/js/src/logging.js +++ b/samples/apps/logging/js/src/logging.js @@ -309,4 +309,4 @@ export function count_public(request) { const records = public_records(ccf.kv, parsedQuery.scope); const count = records.size; return { body: count }; -} +} \ No newline at end of file diff --git a/samples/apps/logging/logging.cpp b/samples/apps/logging/logging.cpp index 8224f60d21ec..d91db9f3ee3d 100644 --- a/samples/apps/logging/logging.cpp +++ b/samples/apps/logging/logging.cpp @@ -922,6 +922,10 @@ namespace loggingapp response += fmt::format( "\nThe caller is identified by a COSE Sign1 signed by kid: {}", cose_ident->user_id); + response += fmt::format( + "\nThe caller is identified by a COSE Sign1 with content of size: " + "{}", + cose_ident->content.size()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_body(std::move(response)); return; diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 823601a066a2..01f05f408d32 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -98,6 +98,13 @@ namespace ccfapp policy_name = get_policy_name_from_ident(user_cose_ident); id = user_cose_ident->user_id; is_member = false; + + auto cose = ctx.new_obj(); + cose.set( + "content", + ctx.new_array_buffer_copy( + user_cose_ident->content.data(), user_cose_ident->content.size())); + caller.set("cose", cose); } if (policy_name == nullptr) diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 5caeb641db96..8dd8d68e666c 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -640,7 +640,7 @@ def require_new_response(r): LOG.info("Authenticate via COSE Sign1 payload") with primary.client(None, None, "user1") as c: - r = c.post("/app/multi_auth") + r = c.post("/app/multi_auth", body={"some": "content"}) require_new_response(r) return network diff --git a/tests/js-modules/modules.py b/tests/js-modules/modules.py index d9ec9e082cfd..011cf704dc13 100644 --- a/tests/js-modules/modules.py +++ b/tests/js-modules/modules.py @@ -1121,9 +1121,12 @@ def test_user_cose_authentication(network, args): assert r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR, r with primary.client(None, None, "user0") as c: - r = c.put("/app/cose", {}) + r = c.put("/app/cose", body={"some": "content"}) assert r.status_code == http.HTTPStatus.OK, r - assert r.body.text() == network.users[0].service_id + body = r.body.json() + assert body["policy"] == "user_cose_sign1" + assert body["id"] == network.users[0].service_id + return network diff --git a/tests/npm-app/src/endpoints/auth.ts b/tests/npm-app/src/endpoints/auth.ts index c51f826b8356..d2ed600bce27 100644 --- a/tests/npm-app/src/endpoints/auth.ts +++ b/tests/npm-app/src/endpoints/auth.ts @@ -6,14 +6,21 @@ export function checkUserCOSESign1Auth( request: ccfapp.Request ): ccfapp.Response { if (request.caller === null || request.caller === undefined) { - return { status: 401 }; + return { statusCode: 401 }; } const caller = request.caller; if (caller.policy !== "user_cose_sign1") { - return { status: 401 }; + return { statusCode: 401 }; } - const id: ccfapp.UserCOSESign1AuthnIdentity = caller; - return { status: 200, body: id.id }; + const c: ccfapp.UserCOSESign1AuthnIdentity = caller; + if ( + request.body.arrayBuffer().byteLength > 0 && + c.cose.content.byteLength == 0 + ) { + return { statusCode: 401 }; + } + + return { statusCode: 200, body: c }; } diff --git a/tests/npm-app/src/endpoints/rpc.ts b/tests/npm-app/src/endpoints/rpc.ts index 1a99e557a21d..874a6ce56c24 100644 --- a/tests/npm-app/src/endpoints/rpc.ts +++ b/tests/npm-app/src/endpoints/rpc.ts @@ -28,7 +28,7 @@ export function getApplyWrites(request: ccfapp.Request): ccfapp.Response { }; } -export function throwError(request: ccfapp.Request): ccfapp.Response { +export function throwError(request: ccfapp.Request) { function nested(arg: number) { throw new Error(`test error: ${arg}`); } From 0e406e48409c819aea5139391a85f89dd090f0b5 Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:56:56 +0100 Subject: [PATCH 047/135] 4.0.7 Changelog (#5479) --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14136e9ca1d4..ea3726b575b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.7] + +[4.0.7]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.7 + +- Expose COSESign1 `content` for `user_cose_sign1` authenticated endpoints in JavaScript/TypeScript apps (#5465). + ## [4.0.6] [4.0.6]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.6 - Updated Open Enclave to [0.19.3](https://github.com/openenclave/openenclave/releases/tag/v0.19.3). -- Expose COSESign1 `content` for `user_cose_sign1` authenticated endpoints in JavaScript/TypeScript apps (#5465). ## [4.0.5] From 2866a8f0da0900461dce2564b8b82d5cca7445cb Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:39:50 +0100 Subject: [PATCH 048/135] [release/4.x] Cherry pick: Fix bug in TLS context - allow incoming write buffer to be relocated (#5482) (#5485) --- src/enclave/tls_session.h | 3 +-- src/tls/context.h | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/enclave/tls_session.h b/src/enclave/tls_session.h index 8242cf887022..58b502e66f1f 100644 --- a/src/enclave/tls_session.h +++ b/src/enclave/tls_session.h @@ -416,8 +416,7 @@ namespace ccf } else { - LOG_TRACE_FMT( - "TLS {} on flush: {}", session_id, tls::error_string(r)); + LOG_TRACE_FMT("TLS session {} error on flush: {}", session_id, -r); stop(error); } } diff --git a/src/tls/context.h b/src/tls/context.h index 54fafcd0fa82..ad68bcb26250 100644 --- a/src/tls/context.h +++ b/src/tls/context.h @@ -60,6 +60,15 @@ namespace tls SSL_CTX_set1_curves_list(cfg, "P-521:P-384:P-256"); SSL_set1_curves_list(ssl, "P-521:P-384:P-256"); + // Allow buffer to be relocated between WANT_WRITE retries, and do partial + // writes if possible + SSL_CTX_set_mode( + cfg, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); + SSL_set_mode( + ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); + // Initialise connection if (client) SSL_set_connect_state(ssl); From 25aa856305e1c3f0d84d3d1aeaf3263532e88cac Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 9 Aug 2023 13:44:00 +0100 Subject: [PATCH 049/135] [release/4.x] Cherry pick: Ruff lint update: Use `isinstance` (#5520) (#5523) --- tests/perf-system/generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/perf-system/generator/generator.py b/tests/perf-system/generator/generator.py index 1a2a90c6d2fc..abc1a37a2668 100644 --- a/tests/perf-system/generator/generator.py +++ b/tests/perf-system/generator/generator.py @@ -39,7 +39,7 @@ def append( headers["content-type"] = content_type # Convert body to bytes if we were given a string - if type(body) == str: + if isinstance(body, str): body = body.encode("utf-8") request_line = f"{verb.upper()} {path} {http_version}" From d27346d7bf43e089a6d51ab72df067015a8ae550 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 10 Aug 2023 16:08:33 +0100 Subject: [PATCH 050/135] [release/4.x] Cherry pick: Install vegeta and pebble and run corresponding tests in Daily (#5529) (#5530) --- .azure-pipelines-templates/common.yml | 3 +++ .azure-pipelines-templates/daily-matrix.yml | 6 +++++ .../install_extended_testing_tools.yml | 8 ++++++ .azure-pipelines-templates/matrix.yml | 8 ++++++ .azure-pipelines-templates/stress-matrix.yml | 1 + .multi-thread.yml | 2 ++ CMakeLists.txt | 25 +++++++++---------- getting_started/setup_vm/ccf-dev.yml | 6 ----- .../setup_vm/ccf-extended-testing.yml | 8 ++++++ 9 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 .azure-pipelines-templates/install_extended_testing_tools.yml create mode 100644 getting_started/setup_vm/ccf-extended-testing.yml diff --git a/.azure-pipelines-templates/common.yml b/.azure-pipelines-templates/common.yml index c71f529d7612..f553437ece0c 100644 --- a/.azure-pipelines-templates/common.yml +++ b/.azure-pipelines-templates/common.yml @@ -27,6 +27,9 @@ jobs: parameters: target: "${{ parameters.target }}" + - ${{ if eq(parameters.installExtendedTestingTools, true) }}: + - template: install_extended_testing_tools.yml + - ${{ if not(eq(parameters.suffix, 'ScanBuild')) }}: - template: cmake.yml parameters: diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index 4bc27077cc86..62290dda5a0f 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -63,6 +63,7 @@ jobs: ctest_filter: '-LE "benchmark|perf"' ctest_timeout: "1600" depends_on: configure + installExtendedTestingTools: true - template: common.yml parameters: @@ -72,6 +73,7 @@ jobs: suffix: "ScanBuild" artifact_name: "Virtual_ScanBuild" depends_on: configure + installExtendedTestingTools: true - template: common.yml parameters: @@ -82,6 +84,7 @@ jobs: artifact_name: "SGX_Release" ctest_filter: '-LE "benchmark|perf|rotation"' depends_on: configure + installExtendedTestingTools: true - template: common.yml parameters: @@ -93,6 +96,7 @@ jobs: artifact_name: "SNPCC_Release" ctest_filter: '-LE "benchmark|perf|rotation"' depends_on: configure + installExtendedTestingTools: true - template: common.yml parameters: @@ -103,6 +107,7 @@ jobs: artifact_name: "Virtual_Release" ctest_filter: '-LE "benchmark|perf|rotation"' depends_on: configure + installExtendedTestingTools: true - template: common.yml parameters: @@ -113,3 +118,4 @@ jobs: artifact_name: "SGX_Unsafe" ctest_filter: '-LE "benchmark|perf|rotation"' depends_on: configure + installExtendedTestingTools: true diff --git a/.azure-pipelines-templates/install_extended_testing_tools.yml b/.azure-pipelines-templates/install_extended_testing_tools.yml new file mode 100644 index 000000000000..882a73182d64 --- /dev/null +++ b/.azure-pipelines-templates/install_extended_testing_tools.yml @@ -0,0 +1,8 @@ +steps: + - script: | + set -ex + sudo apt-get -y update + sudo apt install ansible -y + cd getting_started/setup_vm + ansible-playbook ccf-extended-testing.yml + displayName: Install Extended Testing Tools diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index 4cdf0ce6413c..391076c911e6 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -67,6 +67,7 @@ jobs: artifact_name: "${{ target }}_Debug" ctest_filter: "${{ parameters.test[target].ctest_args }}" depends_on: configure + installExtendedTestingTools: false # Performance - ${{ if eq(parameters.perf_tests, 'run') }}: @@ -79,6 +80,7 @@ jobs: artifact_name: "SGX_Perf" ctest_filter: "${{ parameters.test.perf.ctest_args }}" depends_on: configure + installExtendedTestingTools: false - ${{ if eq(parameters.perf_tests, 'run') }}: - template: common.yml @@ -91,6 +93,7 @@ jobs: artifact_name: "Virtual_Perf" ctest_filter: "${{ parameters.test.virtual_perf.ctest_args }}" depends_on: configure + installExtendedTestingTools: false - ${{ if eq(parameters.perf_tests, 'run') }}: - template: cimetrics.yml @@ -98,6 +101,7 @@ jobs: target: Virtual env: ${{ parameters.env.Virtual }} suffix: "Perf" + installExtendedTestingTools: false depends_on: - configure - SGX_Perf @@ -118,6 +122,7 @@ jobs: artifact_name: "SGX_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure + installExtendedTestingTools: false - template: common.yml parameters: @@ -129,6 +134,7 @@ jobs: artifact_name: "SNPCC_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure + installExtendedTestingTools: false - template: common.yml parameters: @@ -140,6 +146,7 @@ jobs: artifact_name: "Virtual_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure + installExtendedTestingTools: false # Build that produces unsafe binaries for troubleshooting purposes - template: common.yml @@ -151,6 +158,7 @@ jobs: artifact_name: "SGX_Unsafe" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure + installExtendedTestingTools: false - template: release.yml parameters: diff --git a/.azure-pipelines-templates/stress-matrix.yml b/.azure-pipelines-templates/stress-matrix.yml index 017ec216516f..21c239116f1f 100644 --- a/.azure-pipelines-templates/stress-matrix.yml +++ b/.azure-pipelines-templates/stress-matrix.yml @@ -10,3 +10,4 @@ jobs: artifact_name: "StressTest" ctest_filter: '-L "vegeta" -C "long_stress"' depends_on: configure + installExtendedTestingTools: true diff --git a/.multi-thread.yml b/.multi-thread.yml index abdca1aefaae..22bede264ff3 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -34,6 +34,7 @@ jobs: artifact_name: "MultiThread" ctest_filter: '-LE "perf"' depends_on: configure + installExtendedTestingTools: false - template: .azure-pipelines-templates/common.yml parameters: @@ -46,3 +47,4 @@ jobs: suffix: "MultiThreadTsan" artifact_name: "MultiThreadTsan" depends_on: configure + installExtendedTestingTools: false diff --git a/CMakeLists.txt b/CMakeLists.txt index d561aa8fb7f4..cac0250f8a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1431,32 +1431,31 @@ if(BUILD_TESTS) CONSENSUS ${CONSENSUS_FILTER} ) - add_e2e_test( - NAME acme_endorsement_test - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/acme_endorsement.py - LABEL ACME - CONSENSUS cft - ) + if(LONG_TESTS) + add_e2e_test( + NAME acme_endorsement_test + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/acme_endorsement.py + CONSENSUS ${CONSENSUS_FILTER} + LABEL ACME + ) - foreach(CONSENSUS ${CONSENSUSES}) add_e2e_test( - NAME vegeta_stress_${CONSENSUS} + NAME vegeta_stress PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/vegeta_stress.py - CONSENSUS ${CONSENSUS} LABEL vegeta + CONSENSUS ${CONSENSUS_FILTER} ADDITIONAL_ARGS -p "samples/apps/logging/liblogging" ) add_e2e_test( - NAME vegeta_long_stress_${CONSENSUS} + NAME vegeta_long_stress PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/vegeta_stress.py - CONSENSUS ${CONSENSUS} LABEL vegeta CONFIGURATIONS long_stress + CONSENSUS ${CONSENSUS_FILTER} ADDITIONAL_ARGS -p "samples/apps/logging/liblogging" --duration 45m ) - - endforeach() + endif() add_perf_test( NAME ls diff --git a/getting_started/setup_vm/ccf-dev.yml b/getting_started/setup_vm/ccf-dev.yml index 686bdb489850..8db67cfd8823 100644 --- a/getting_started/setup_vm/ccf-dev.yml +++ b/getting_started/setup_vm/ccf-dev.yml @@ -29,12 +29,6 @@ - import_role: name: nodejs tasks_from: install.yml - - import_role: - name: vegeta - tasks_from: install.yml - - import_role: - name: pebble - tasks_from: install.yml - import_role: name: h2spec tasks_from: install.yml diff --git a/getting_started/setup_vm/ccf-extended-testing.yml b/getting_started/setup_vm/ccf-extended-testing.yml new file mode 100644 index 000000000000..93610045d5a1 --- /dev/null +++ b/getting_started/setup_vm/ccf-extended-testing.yml @@ -0,0 +1,8 @@ +- hosts: localhost + tasks: + - import_role: + name: vegeta + tasks_from: install.yml + - import_role: + name: pebble + tasks_from: install.yml From d5a4ba9d42375f7650f5ab11d7da5874f5f2ccaa Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 11 Aug 2023 09:54:47 +0100 Subject: [PATCH 051/135] [release/4.x] Cherry pick: Remove GHA SNP CI container builder (#5471) (#5534) --- .azure-pipelines-templates/azure_cli.yml | 4 +- .azure-pipelines-templates/deploy_aci.yml | 40 +++++++------- .../install_ssh_key.yml | 2 +- .azure_pipelines_snp.yml | 6 +++ .github/workflows/build-ci-container.yml | 54 ------------------- docker/ccf_ci_built | 6 ++- 6 files changed, 33 insertions(+), 79 deletions(-) delete mode 100644 .github/workflows/build-ci-container.yml diff --git a/.azure-pipelines-templates/azure_cli.yml b/.azure-pipelines-templates/azure_cli.yml index dfaaca8cf653..e045abf02871 100644 --- a/.azure-pipelines-templates/azure_cli.yml +++ b/.azure-pipelines-templates/azure_cli.yml @@ -5,9 +5,7 @@ steps: # After the extension being in public preview, we can install the latest version automatically # by `az extension update --name confcom`. # But for now we need to manually manage the version. - az extension add --source https://acccliazext.blob.core.windows.net/confcom/confcom-0.2.10-py3-none-any.whl -y - # Workaround of bug in v0.2.10 - sudo chmod +x /opt/az/azcliextensions/confcom/azext_confcom/bin/dmverity-vhd + az extension add --name confcom -y az login --service-principal -u ${{ parameters.app_id }} -p ${{ parameters.service_principal_password }} --tenant ${{ parameters.tenant }} name: setup_azure_cli displayName: "Install Azure CLI and login" diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 192e666b07e7..5d05c6ec2363 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -25,9 +25,13 @@ jobs: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true sshKey: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] - pool: - vmImage: ubuntu-20.04 + pool: ado-virtual-ccf-sub # To build CCF quickly steps: + - checkout: self + clean: true + fetchDepth: 0 + fetchTags: true + - script: | env name: print_env @@ -45,26 +49,21 @@ jobs: - script: | set -ex - RETRIES=100 - HEAD_SHA=`git rev-parse HEAD` - if [ $(Build.SourceBranchName) == "merge" ]; then - # If this is the case, we're running in a PR, and the SHA we really want is in the - # commit message of the last commit (which merges it into target) - echo "Running in a PR, getting the SHA from the commit message" - LAST_COMMIT=(`git log -1 --pretty=%B`) - HEAD_SHA=${LAST_COMMIT[1]} - fi - echo "##vso[task.setvariable variable=gitSha;isOutput=true]$HEAD_SHA" - until [ $RETRIES -eq 0 ] || [[ `az acr repository show-tags -n ccfmsrc --repository ccf/ci` == *"$HEAD_SHA"* ]] - do - sleep 60 - echo "$(( RETRIES-- )) tries remaining" - done - name: wait_for_image - displayName: "Wait for CI Container Building" + docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY + docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.3-snp-clang15 + docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` + docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` + name: build_ci_image + displayName: "Build CI SNP container" + env: + ACR_TOKEN_NAME: ci-push-token + ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) + ACR_REGISTRY: ccfmsrc.azurecr.io + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 - script: | set -ex + sudo apt-get install -y python3.8-venv python3.8 -m venv ./scripts/azure_deployment/.env source ./scripts/azure_deployment/.env/bin/activate pip install -r ./scripts/azure_deployment/requirements.txt @@ -74,7 +73,7 @@ jobs: --region northeurope \ --aci-type dynamic-agent \ --deployment-name ci-$(Build.BuildNumber) \ - --aci-image ccfmsrc.azurecr.io/ccf/ci:pr-$(wait_for_image.gitSha) \ + --aci-image ccfmsrc.azurecr.io/ccf/ci:pr-`git rev-parse HEAD` \ --ports 22 \ --aci-setup-timeout 300 \ --aci-private-key-b64 $(sshKey) \ @@ -110,6 +109,7 @@ jobs: service_principal_password: $(CCF_SNP_CI_SERVICE_PRINCIPAL_PASSWORD) tenant: $(CCF_SNP_CI_TENANT) + # TODO: Fix this too - script: | set -ex RETRIES=100 diff --git a/.azure-pipelines-templates/install_ssh_key.yml b/.azure-pipelines-templates/install_ssh_key.yml index b8736a37348e..0a9657a7ac26 100644 --- a/.azure-pipelines-templates/install_ssh_key.yml +++ b/.azure-pipelines-templates/install_ssh_key.yml @@ -1,7 +1,7 @@ steps: - script: | set -ex - mkdir ~/.ssh + mkdir -p ~/.ssh echo ${{ parameters.ssh_key }} | base64 -d > ~/.ssh/id_rsa sudo chmod 600 ~/.ssh/id_rsa sudo ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 155fc3989a01..d6e0993a3672 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -32,6 +32,12 @@ variables: - name: secondaryAcisPath value: "/home/agent/secondary_acis" +resources: + containers: + - container: virtual + image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + jobs: - template: .azure-pipelines-templates/configure.yml diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml deleted file mode 100644 index baa35b00b860..000000000000 --- a/.github/workflows/build-ci-container.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: "Build SNP CI Testing CCF Container" - -on: - workflow_dispatch: - pull_request_target: - branches: - - main - - release/3.x - paths: - - scripts/azure_deployment/* - - .github/workflows/build-ci-container.yml - - .azure_pipelines_snp.yml - - .azure-pipelines-templates/deploy_aci.yml - - .azure-pipelines-templates/test_on_remote.yml - - .snpcc_canary - push: - branches: - - main - - release/3.x - schedule: - - cron: "0 9 * * Mon-Fri" - -env: - ACR_REGISTRY: ccfmsrc.azurecr.io - ACR_TOKEN_NAME: ci-push-token - -jobs: - build: - name: "Build and Publish Pre-built SNP Container" - runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Debug - run: | - df -kh - sudo mkdir -p /etc/docker - sudo echo '{"data-root": "/mnt"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - - - name: Log in - run: docker login -u $ACR_TOKEN_NAME -p ${{ secrets.ACR_CI_PUSH_TOKEN_PASSWORD }} $ACR_REGISTRY - - - name: Pull CI container - run: docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.3-snp-clang15 - - - name: Build CCF CI SNP container - run: docker build -f docker/ccf_ci_built . --build-arg="base=ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` - - - name: Push CI container - run: docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index c16716712801..8332565d25e0 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -26,6 +26,10 @@ RUN useradd -m $user \ ARG platform=sgx RUN mkdir /CCF COPY . /CCF/ -RUN mkdir /CCF/build && cd /CCF/build && cmake -GNinja -DCOMPILE_TARGET=${platform} .. && ninja && chmod -R 777 /CCF +RUN mkdir /CCF/build \ + && cd /CCF/build \ + && cmake -GNinja -DCOMPILE_TARGET=${platform} .. \ + && ninja \ + && chmod -R 777 /CCF CMD ["/usr/sbin/sshd", "-D"] From f9b6cb25780382cf4b45e6de95b47b5898d47c0f Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 11 Aug 2023 10:56:45 +0100 Subject: [PATCH 052/135] [release/4.x] Cherry pick: Update CI jobs to 10-08-2023 base (#5532) (#5536) --- .azure-pipelines-gh-pages.yml | 2 +- .azure-pipelines-templates/daily-matrix.yml | 19 +++++++++++++++++++ .azure-pipelines-templates/deploy_aci.yml | 4 ++-- .azure-pipelines.yml | 6 +++--- .azure_pipelines_snp.yml | 2 +- .daily.yml | 6 +++--- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci-checks.yml | 2 +- .multi-thread.yml | 2 +- .stress.yml | 2 +- docker/ccf_ci_built | 2 +- scripts/azure_deployment/arm_aci.py | 2 +- 12 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index 2386a1e0d701..2e4b4571a89b 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -11,7 +11,7 @@ jobs: variables: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true - container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index 62290dda5a0f..f40d093cc2c8 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -57,9 +57,28 @@ jobs: target: Virtual env: "${{ parameters.env.Virtual }}" fetch_quictls: debug +<<<<<<< HEAD cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.SAN.cmake_args }} ${{ parameters.build.QUICTLS.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "Instrumented" artifact_name: "Virtual_Instrumented" +======= + cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.ASAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" + suffix: "ASAN" + artifact_name: "Virtual_ASAN" + ctest_filter: '-LE "benchmark|perf"' + ctest_timeout: "1600" + depends_on: configure + installExtendedTestingTools: true + + - template: common.yml + parameters: + target: Virtual + env: "${{ parameters.env.Virtual }}" + fetch_quictls: debug + cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.TSAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" + suffix: "TSAN" + artifact_name: "Virtual_TSAN" +>>>>>>> ad7f64d81... Update CI jobs to 10-08-2023 base (#5532) ctest_filter: '-LE "benchmark|perf"' ctest_timeout: "1600" depends_on: configure diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 5d05c6ec2363..e85993b7e1d3 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -50,7 +50,7 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.3-snp-clang15 + docker pull $ACR_REGISTRY/ccf/ci:10-08-2023-snp-clang15 docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image @@ -59,7 +59,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 - script: | set -ex diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 5ce5abab716d..b8d501caedfd 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,15 +29,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index d6e0993a3672..72a7426539ea 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -35,7 +35,7 @@ variables: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.daily.yml b/.daily.yml index e21670eca6f5..6ceb24c08cee 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,15 +25,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3589efcaa4f7..e70fdfde1a80 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 33ae3e4ca09e..af271eaf4aa2 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.multi-thread.yml b/.multi-thread.yml index 22bede264ff3..883f890fd85a 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -16,7 +16,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 828a7234451d..6587f3ef99ed 100644 --- a/.stress.yml +++ b/.stress.yml @@ -20,7 +20,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-sgx + image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 8332565d25e0..7fe5cd9fbe19 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index 77abd366cb90..c330e00a35aa 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -178,7 +178,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:oe-0.19.3-snp", + default="ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp", ) parser.add_argument( "--aci-type", From 7d89f84c5cf901837b73a0bfe628b39728da3456 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 11 Aug 2023 18:02:42 +0100 Subject: [PATCH 053/135] [release/4.x] Cherry pick: Update CI image to 11-08-2023 (#5538) (#5540) --- .azure-pipelines-gh-pages.yml | 2 +- .azure-pipelines-templates/deploy_aci.yml | 4 ++-- .azure-pipelines.yml | 6 +++--- .azure_pipelines_snp.yml | 2 +- .daily.yml | 6 +++--- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci-checks.yml | 2 +- .multi-thread.yml | 2 +- .stress.yml | 2 +- docker/ccf_ci_built | 2 +- scripts/azure_deployment/arm_aci.py | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index 2e4b4571a89b..90f25ac62d38 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -11,7 +11,7 @@ jobs: variables: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true - container: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index e85993b7e1d3..9c4a71738d56 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -50,7 +50,7 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:10-08-2023-snp-clang15 + docker pull $ACR_REGISTRY/ccf/ci:11-08-2023-snp-clang15 docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image @@ -59,7 +59,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 - script: | set -ex diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index b8d501caedfd..26274d67220f 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,15 +29,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 72a7426539ea..dec19b9e6353 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -35,7 +35,7 @@ variables: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.daily.yml b/.daily.yml index 6ceb24c08cee..d4d6d963530d 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,15 +25,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e70fdfde1a80..1953bc51c0db 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index af271eaf4aa2..80a173729b32 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.multi-thread.yml b/.multi-thread.yml index 883f890fd85a..9e6491008f11 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -16,7 +16,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 6587f3ef99ed..2bbbda91c57b 100644 --- a/.stress.yml +++ b/.stress.yml @@ -20,7 +20,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:10-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 7fe5cd9fbe19..7ad03b38e5b7 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index c330e00a35aa..5c461ec5019c 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -178,7 +178,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:10-08-2023-snp", + default="ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp", ) parser.add_argument( "--aci-type", From bfea93e820d3b9bf23b1c9e0191b43c9d45e0dba Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 15 Aug 2023 13:37:26 +0100 Subject: [PATCH 054/135] [release/4.x] Cherry pick: Update CI image from 11-08 to 14-08 (#5544) (#5546) --- .azure-pipelines-gh-pages.yml | 2 +- .azure-pipelines-templates/deploy_aci.yml | 4 ++-- .azure-pipelines.yml | 6 +++--- .azure_pipelines_snp.yml | 2 +- .daily.yml | 6 +++--- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci-checks.yml | 2 +- .multi-thread.yml | 2 +- .stress.yml | 2 +- docker/ccf_ci_built | 2 +- scripts/azure_deployment/arm_aci.py | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index 90f25ac62d38..4088607a87c4 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -11,7 +11,7 @@ jobs: variables: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true - container: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 9c4a71738d56..c307fde94722 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -50,7 +50,7 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:11-08-2023-snp-clang15 + docker pull $ACR_REGISTRY/ccf/ci:14-08-2023-snp-clang15 docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image @@ -59,7 +59,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 - script: | set -ex diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 26274d67220f..0d10be7963df 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,15 +29,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index dec19b9e6353..dab28fa7b429 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -35,7 +35,7 @@ variables: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.daily.yml b/.daily.yml index d4d6d963530d..7a01147ad77b 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,15 +25,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1953bc51c0db..d126a7ff0c42 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 80a173729b32..669151e642a3 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.multi-thread.yml b/.multi-thread.yml index 9e6491008f11..466e5bb9c55d 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -16,7 +16,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 2bbbda91c57b..780d707d0069 100644 --- a/.stress.yml +++ b/.stress.yml @@ -20,7 +20,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:11-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 7ad03b38e5b7..29585ae4e5a2 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index 5c461ec5019c..d16e2d46d1a1 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -178,7 +178,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:11-08-2023-snp", + default="ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp", ) parser.add_argument( "--aci-type", From 2ccee47ce6f7a5c4c743499e3ab29ca2e3b81aec Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 17 Aug 2023 11:11:12 +0100 Subject: [PATCH 055/135] Do not have dot - release/4.x (#5556) --- Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index 91f70f6d5c93..6ee8294e1f1f 100644 --- a/Doxyfile +++ b/Doxyfile @@ -2399,7 +2399,7 @@ HIDE_UNDOC_RELATIONS = YES # set to NO # The default value is: NO. -HAVE_DOT = YES +HAVE_DOT = NO # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed # to run in parallel. When set to 0 doxygen will base this on the number of From 2d3b7a999ab2418ffdb939ca8a079f582199a813 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:29:59 +0100 Subject: [PATCH 056/135] [release/4.x] Cherry pick: Switch to a backup pool (#5512) (#5558) Co-authored-by: Amaury Chamayou --- .azure-pipelines-templates/daily-matrix.yml | 2 +- .azure-pipelines-templates/matrix.yml | 2 +- .azure-pipelines-templates/stress-matrix.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index f40d093cc2c8..d4e79b9195cb 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -5,7 +5,7 @@ parameters: pool: ado-virtual-ccf-sub SGX: container: sgx - pool: ado-sgx-ccf-sub + pool: ado-sgx-ccf-sub-backup SNPCC: container: snp pool: ado-virtual-ccf-sub diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index 391076c911e6..71fe0a34d0ab 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -11,7 +11,7 @@ parameters: pool: ado-virtual-ccf-sub SGX: container: sgx - pool: ado-sgx-ccf-sub + pool: ado-sgx-ccf-sub-backup SNPCC: container: snp pool: ado-virtual-ccf-sub diff --git a/.azure-pipelines-templates/stress-matrix.yml b/.azure-pipelines-templates/stress-matrix.yml index 21c239116f1f..19071e629dfa 100644 --- a/.azure-pipelines-templates/stress-matrix.yml +++ b/.azure-pipelines-templates/stress-matrix.yml @@ -4,7 +4,7 @@ jobs: target: SGX env: container: sgx - pool: ado-sgx-ccf-sub + pool: ado-sgx-ccf-sub-backup cmake_args: "-DCOMPILE_TARGET=sgx" suffix: "StressTest" artifact_name: "StressTest" From 50c71482cad632d5317e59a10a2cb6549ce80c71 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 17 Aug 2023 13:19:02 +0100 Subject: [PATCH 057/135] [release/4.x] Cherry pick: Remove graphviz from llvm task (#5551) (#5554) --- .azure-pipelines-gh-pages.yml | 2 +- .azure-pipelines-templates/deploy_aci.yml | 4 ++-- .azure-pipelines.yml | 6 +++--- .azure_pipelines_snp.yml | 2 +- .daily.yml | 6 +++--- .devcontainer/devcontainer.json | 2 +- .github/workflows/ci-checks.yml | 2 +- .multi-thread.yml | 2 +- .stress.yml | 2 +- docker/ccf_ci_built | 2 +- getting_started/setup_vm/roles/autoremove/install.yml | 11 +++++++++++ .../setup_vm/roles/ccf_build/vars/clang11.yml | 1 - .../setup_vm/roles/ccf_build/vars/clang15.yml | 1 - scripts/azure_deployment/arm_aci.py | 2 +- 14 files changed, 27 insertions(+), 18 deletions(-) create mode 100644 getting_started/setup_vm/roles/autoremove/install.yml diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index 4088607a87c4..b55ce678f9c0 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -11,7 +11,7 @@ jobs: variables: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true - container: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index c307fde94722..b657fbcd681d 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -50,7 +50,7 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:14-08-2023-snp-clang15 + docker pull $ACR_REGISTRY/ccf/ci:16-08-2023-1-snp-clang15 docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image @@ -59,7 +59,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-snp-clang15 - script: | set -ex diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 0d10be7963df..14961f3494e7 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,15 +29,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index dab28fa7b429..9010c1667b60 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -35,7 +35,7 @@ variables: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.daily.yml b/.daily.yml index 7a01147ad77b..665cb37370f7 100644 --- a/.daily.yml +++ b/.daily.yml @@ -25,15 +25,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d126a7ff0c42..e7c143f59e55 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 669151e642a3..cd3477df0035 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -9,7 +9,7 @@ on: jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.multi-thread.yml b/.multi-thread.yml index 466e5bb9c55d..7e90377ee79d 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -16,7 +16,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 780d707d0069..a0bdfce012b2 100644 --- a/.stress.yml +++ b/.stress.yml @@ -20,7 +20,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:14-08-2023-sgx + image: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 29585ae4e5a2..65a7d24d6ea5 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/getting_started/setup_vm/roles/autoremove/install.yml b/getting_started/setup_vm/roles/autoremove/install.yml new file mode 100644 index 000000000000..c439ff434dc1 --- /dev/null +++ b/getting_started/setup_vm/roles/autoremove/install.yml @@ -0,0 +1,11 @@ +- name: Remove graphviz debian package + apt: + name: graphviz + state: absent + become: yes + +- name: Remove any uncessary packages + apt: + name: "autoremove" + autoremove: yes + become: yes diff --git a/getting_started/setup_vm/roles/ccf_build/vars/clang11.yml b/getting_started/setup_vm/roles/ccf_build/vars/clang11.yml index 45d6f4887d57..306691cd32ed 100644 --- a/getting_started/setup_vm/roles/ccf_build/vars/clang11.yml +++ b/getting_started/setup_vm/roles/ccf_build/vars/clang11.yml @@ -28,7 +28,6 @@ debs: - libclang1-9 # required to build doxygen - libclang-cpp9 # required to build doxygen - pkg-config # required by v8 - - graphviz # required to run doxygen - unzip # required to unzip protoc install # Not installed on GitHub Actions environment because of conflicting package diff --git a/getting_started/setup_vm/roles/ccf_build/vars/clang15.yml b/getting_started/setup_vm/roles/ccf_build/vars/clang15.yml index de43678752d4..4f545cc04e3d 100644 --- a/getting_started/setup_vm/roles/ccf_build/vars/clang15.yml +++ b/getting_started/setup_vm/roles/ccf_build/vars/clang15.yml @@ -28,7 +28,6 @@ debs: - libclang1-9 # required to build doxygen - libclang-cpp9 # required to build doxygen - pkg-config # required by v8 - - graphviz # required to run doxygen - unzip # required to unzip protoc install # Not installed on GitHub Actions environment because of conflicting package diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index d16e2d46d1a1..582d3d11dc1c 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -178,7 +178,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:14-08-2023-snp", + default="ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-snp", ) parser.add_argument( "--aci-type", From 8fb28dce154f6a94545290caf378d35a1b47ef90 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 17 Aug 2023 13:24:23 +0100 Subject: [PATCH 058/135] Fix bad merge in 4.x daily (#5560) --- .azure-pipelines-templates/daily-matrix.yml | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index d4e79b9195cb..11ba1f4014d3 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -57,28 +57,9 @@ jobs: target: Virtual env: "${{ parameters.env.Virtual }}" fetch_quictls: debug -<<<<<<< HEAD - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.SAN.cmake_args }} ${{ parameters.build.QUICTLS.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" + cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.SAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "Instrumented" artifact_name: "Virtual_Instrumented" -======= - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.ASAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" - suffix: "ASAN" - artifact_name: "Virtual_ASAN" - ctest_filter: '-LE "benchmark|perf"' - ctest_timeout: "1600" - depends_on: configure - installExtendedTestingTools: true - - - template: common.yml - parameters: - target: Virtual - env: "${{ parameters.env.Virtual }}" - fetch_quictls: debug - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.TSAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" - suffix: "TSAN" - artifact_name: "Virtual_TSAN" ->>>>>>> ad7f64d81... Update CI jobs to 10-08-2023 base (#5532) ctest_filter: '-LE "benchmark|perf"' ctest_timeout: "1600" depends_on: configure From 2fcf7435b5abb7dc34af8387d38c0eef0104df7f Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:10:01 +0100 Subject: [PATCH 059/135] [release/4.x] Cherry pick: Use endpoint path in statistics (#5543) (#5553) --- include/ccf/endpoint_registry.h | 11 ++++--- src/endpoints/endpoint_registry.cpp | 17 ++++++----- src/node/rpc/frontend.h | 46 ++++++++++++++++------------- tests/e2e_logging.py | 26 +++++++++------- 4 files changed, 58 insertions(+), 42 deletions(-) diff --git a/include/ccf/endpoint_registry.h b/include/ccf/endpoint_registry.h index c276222beec9..ebf8ff14b26a 100644 --- a/include/ccf/endpoint_registry.h +++ b/include/ccf/endpoint_registry.h @@ -255,9 +255,12 @@ namespace ccf::endpoints void set_history(kv::TxHistory* h); - virtual void increment_metrics_calls(const ccf::RpcContext& rpc_ctx); - virtual void increment_metrics_errors(const ccf::RpcContext& rpc_ctx); - virtual void increment_metrics_failures(const ccf::RpcContext& rpc_ctx); - virtual void increment_metrics_retries(const ccf::RpcContext& rpc_ctx); + virtual void increment_metrics_calls(const EndpointDefinitionPtr& endpoint); + virtual void increment_metrics_errors( + const EndpointDefinitionPtr& endpoint); + virtual void increment_metrics_failures( + const EndpointDefinitionPtr& endpoint); + virtual void increment_metrics_retries( + const EndpointDefinitionPtr& endpoint); }; } diff --git a/src/endpoints/endpoint_registry.cpp b/src/endpoints/endpoint_registry.cpp index 965043a350bb..8a768523c89f 100644 --- a/src/endpoints/endpoint_registry.cpp +++ b/src/endpoints/endpoint_registry.cpp @@ -568,38 +568,39 @@ namespace ccf::endpoints history = h; } - void EndpointRegistry::increment_metrics_calls(const ccf::RpcContext& rpc_ctx) + void EndpointRegistry::increment_metrics_calls( + const endpoints::EndpointDefinitionPtr& endpoint) { std::lock_guard guard(metrics_lock); get_metrics_for_request( - rpc_ctx.get_method(), rpc_ctx.get_request_verb().c_str()) + endpoint->dispatch.uri_path, endpoint->dispatch.verb.c_str()) .calls++; } void EndpointRegistry::increment_metrics_errors( - const ccf::RpcContext& rpc_ctx) + const endpoints::EndpointDefinitionPtr& endpoint) { std::lock_guard guard(metrics_lock); get_metrics_for_request( - rpc_ctx.get_method(), rpc_ctx.get_request_verb().c_str()) + endpoint->dispatch.uri_path, endpoint->dispatch.verb.c_str()) .errors++; } void EndpointRegistry::increment_metrics_failures( - const ccf::RpcContext& rpc_ctx) + const endpoints::EndpointDefinitionPtr& endpoint) { std::lock_guard guard(metrics_lock); get_metrics_for_request( - rpc_ctx.get_method(), rpc_ctx.get_request_verb().c_str()) + endpoint->dispatch.uri_path, endpoint->dispatch.verb.c_str()) .failures++; } void EndpointRegistry::increment_metrics_retries( - const ccf::RpcContext& rpc_ctx) + const endpoints::EndpointDefinitionPtr& endpoint) { std::lock_guard guard(metrics_lock); get_metrics_for_request( - rpc_ctx.get_method(), rpc_ctx.get_request_verb().c_str()) + endpoint->dispatch.uri_path, endpoint->dispatch.verb.c_str()) .retries++; } } diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index 26503e7e4c70..9a1922ce0520 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -71,16 +71,18 @@ namespace ccf endpoints.set_history(history); } - void update_metrics(const std::shared_ptr& ctx) + void update_metrics( + const std::shared_ptr& ctx, + const endpoints::EndpointDefinitionPtr& endpoint) { int cat = ctx->get_response_status() / 100; switch (cat) { case 4: - endpoints.increment_metrics_errors(*ctx); + endpoints.increment_metrics_errors(endpoint); return; case 5: - endpoints.increment_metrics_failures(*ctx); + endpoints.increment_metrics_failures(endpoint); return; } } @@ -279,7 +281,7 @@ namespace ccf ccf::errors::InvalidAuthenticationInfo, "Invalid authentication credentials.", std::move(json_details)); - update_metrics(ctx); + update_metrics(ctx, endpoint); } return identity; @@ -319,7 +321,7 @@ namespace ccf HTTP_STATUS_NOT_IMPLEMENTED, ccf::errors::NotImplemented, "Request cannot be forwarded to primary on HTTP/2 interface."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -329,7 +331,7 @@ namespace ccf HTTP_STATUS_INTERNAL_SERVER_ERROR, ccf::errors::InternalError, "No consensus or forwarder to forward request."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -341,7 +343,7 @@ namespace ccf HTTP_STATUS_SERVICE_UNAVAILABLE, ccf::errors::RequestAlreadyForwarded, "RPC was already forwarded."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -359,7 +361,7 @@ namespace ccf HTTP_STATUS_SERVICE_UNAVAILABLE, ccf::errors::InternalError, "RPC could not be forwarded to unknown primary."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -373,7 +375,7 @@ namespace ccf HTTP_STATUS_SERVICE_UNAVAILABLE, ccf::errors::InternalError, "Unable to establish channel to forward to primary."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -393,6 +395,7 @@ namespace ccf { size_t attempts = 0; constexpr auto max_attempts = 30; + endpoints::EndpointDefinitionPtr endpoint = nullptr; while (attempts < max_attempts) { @@ -404,7 +407,6 @@ namespace ccf // If the endpoint has already been executed, the effects of its // execution should be dropped ctx->reset_response(); - endpoints.increment_metrics_retries(*ctx); } if (!is_open(*tx_p)) @@ -419,7 +421,7 @@ namespace ccf ++attempts; update_history(); - const auto endpoint = find_endpoint(ctx, *tx_p); + endpoint = find_endpoint(ctx, *tx_p); if (endpoint == nullptr) { return; @@ -429,7 +431,11 @@ namespace ccf // Only register calls to existing endpoints if (attempts == 1) { - endpoints.increment_metrics_calls(*ctx); + endpoints.increment_metrics_calls(endpoint); + } + else + { + endpoints.increment_metrics_retries(endpoint); } } @@ -503,7 +509,7 @@ namespace ccf if (!ctx->should_apply_writes()) { - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -580,7 +586,7 @@ namespace ccf history->try_emit_signature(); } - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } @@ -596,7 +602,7 @@ namespace ccf HTTP_STATUS_SERVICE_UNAVAILABLE, ccf::errors::TransactionReplicationFailed, "Transaction failed to replicate."); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } } @@ -613,7 +619,7 @@ namespace ccf { ctx->clear_response_headers(); ctx->set_error(std::move(e.error)); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } catch (const JsonParseError& e) @@ -621,7 +627,7 @@ namespace ccf ctx->clear_response_headers(); ctx->set_error( HTTP_STATUS_BAD_REQUEST, ccf::errors::InvalidInput, e.describe()); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } catch (const nlohmann::json::exception& e) @@ -629,7 +635,7 @@ namespace ccf ctx->clear_response_headers(); ctx->set_error( HTTP_STATUS_BAD_REQUEST, ccf::errors::InvalidInput, e.what()); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } catch (const kv::KvSerialiserException& e) @@ -648,7 +654,7 @@ namespace ccf HTTP_STATUS_INTERNAL_SERVER_ERROR, ccf::errors::InternalError, e.what()); - update_metrics(ctx); + update_metrics(ctx, endpoint); return; } } // end of while loop @@ -661,7 +667,7 @@ namespace ccf "Transaction continued to conflict after {} attempts. Retry " "later.", max_attempts)); - update_metrics(ctx); + update_metrics(ctx, endpoint); static constexpr size_t retry_after_seconds = 3; ctx->set_response_header(http::headers::RETRY_AFTER, retry_after_seconds); diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 8dd8d68e666c..f3fa0e63ef55 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -706,6 +706,17 @@ def test_custom_auth_safety(network, args): return network +def get_metrics(r, path, method, default=None): + try: + return next( + v + for v in r.body.json()["metrics"] + if v["path"] == path and v["method"] == method + ) + except StopIteration: + return default + + @reqs.description("Write non-JSON body") @reqs.supports_methods("/app/log/private/raw_text/{id}", "/app/log/private") @app.scoped_txs() @@ -718,6 +729,11 @@ def test_raw_text(network, args): r = network.txs.request(log_id, priv=True) assert msg in r.body.json()["msg"], r + primary, _ = network.find_primary() + with primary.client("user0") as c: + r = c.get("/app/api/metrics") + assert get_metrics(r, "log/private/raw_text/{id}", "POST")["calls"] > 0 + return network @@ -726,16 +742,6 @@ def test_raw_text(network, args): def test_metrics(network, args): primary, _ = network.find_primary() - def get_metrics(r, path, method, default=None): - try: - return next( - v - for v in r.body.json()["metrics"] - if v["path"] == path and v["method"] == method - ) - except StopIteration: - return default - calls = 0 errors = 0 with primary.client("user0") as c: From 5af191948349479ed2845e49f6c6b2f818f35599 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Fri, 18 Aug 2023 11:30:15 +0100 Subject: [PATCH 060/135] [release/4.x] Cherry pick: Do not compare types, use `isinstance()` (#5566) (#5567) --- tests/election.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/election.py b/tests/election.py index 97d3d6172b9b..506e66b9912b 100644 --- a/tests/election.py +++ b/tests/election.py @@ -32,7 +32,7 @@ def test_kill_primary_no_reqs(network, args): res = c.get("/app/commit?view_history=true") assert res.status_code == http.HTTPStatus.OK assert "view_history" in res.body.json() - assert type(res.body.json()["view_history"]) == list + assert isinstance(res.body.json()["view_history"], list) old_view_history = res.body.json()["view_history"] old_primary.stop() @@ -45,7 +45,7 @@ def test_kill_primary_no_reqs(network, args): res = c.get("/app/commit?view_history=true") assert res.status_code == http.HTTPStatus.OK assert "view_history" in res.body.json() - assert type(res.body.json()["view_history"]) == list + assert isinstance(res.body.json()["view_history"], list) new_view_history = res.body.json()["view_history"] # Check that the view history has been updated with a new term for the new primary # new view history should be longer than old view history but may be more than one ahead due to multiple rounds occurring. @@ -126,7 +126,7 @@ def test_commit_view_history(network, args): res = c.get("/app/commit?view_history=true") assert res.status_code == http.HTTPStatus.OK assert "view_history" in res.body.json() - assert type(res.body.json()["view_history"]) == list + assert isinstance(res.body.json()["view_history"], list) view_history = res.body.json()["view_history"] res = c.get("/node/network") From ddf8a02bd61823f8eb1cb2e6040f3aad4431d5d2 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 18 Aug 2023 11:31:29 +0100 Subject: [PATCH 061/135] [release/4.x] Cherry pick: Fix SNP CI authentication (#5552) (#5565) --- .azure-pipelines-templates/deploy_aci.yml | 110 +----------------- .azure-pipelines-templates/test_on_remote.yml | 1 - .azure_pipelines_snp.yml | 8 -- scripts/azure_deployment/arm_aci.py | 52 +-------- scripts/azure_deployment/arm_template.py | 6 +- tests/code_update.py | 41 ------- tests/infra/e2e_args.py | 6 - 7 files changed, 7 insertions(+), 217 deletions(-) diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index b657fbcd681d..34e09eae1e20 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -86,102 +86,6 @@ jobs: env: CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID) - - ${{ if not(eq(parameters.secondaries.count, 0)) }}: - - job: deploy_secondary_aci - displayName: "Deploy Secondary ACI" - pool: - vmImage: ubuntu-20.04 - dependsOn: - - generate_ssh_key - variables: - Codeql.SkipTaskAutoInjection: true - skipComponentGovernanceDetection: true - sshKey: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] - - steps: - - template: install_ssh_key.yml - parameters: - ssh_key: $(sshKey) - - - template: azure_cli.yml - parameters: - app_id: $(CCF_SNP_CI_APP_ID) - service_principal_password: $(CCF_SNP_CI_SERVICE_PRINCIPAL_PASSWORD) - tenant: $(CCF_SNP_CI_TENANT) - - # TODO: Fix this too - - script: | - set -ex - RETRIES=100 - HEAD_SHA=`git rev-parse HEAD` - if [ $(Build.SourceBranchName) == "merge" ]; then - # If this is the case, we're running in a PR, and the SHA we really want is in the - # commit message of the last commit (which merges it into target) - echo "Running in a PR, getting the SHA from the commit message" - LAST_COMMIT=(`git log -1 --pretty=%B`) - HEAD_SHA=${LAST_COMMIT[1]} - fi - echo "##vso[task.setvariable variable=gitSha;isOutput=true]$HEAD_SHA" - until [ $RETRIES -eq 0 ] || [[ `az acr repository show-tags -n ccfmsrc --repository ccf/ci` == *"$HEAD_SHA"* ]] - do - sleep 60 - echo "$(( RETRIES-- )) tries remaining" - done - name: wait_for_image - displayName: "Wait for CI Container Building" - - - script: | - set -ex - python3.8 -m venv ./scripts/azure_deployment/.env - source ./scripts/azure_deployment/.env/bin/activate - pip install -r ./scripts/azure_deployment/requirements.txt - python3.8 scripts/azure_deployment/arm_template.py deploy aci \ - --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) \ - --resource-group ccf-aci \ - --region northeurope \ - --aci-type dynamic-agent \ - --deployment-name ci-$(Build.BuildNumber)-secondaries \ - --aci-image ccfmsrc.azurecr.io/ccf/ci:pr-$(wait_for_image.gitSha) \ - --ports 22 \ - --count ${{ parameters.secondaries.count }} \ - --aci-setup-timeout 600 \ - --out ~/secondary_aci_ips - echo "##vso[task.setvariable variable=secondaryIpAddresses;isOutput=true]`base64 -w 0 ~/secondary_aci_ips`" - name: deploy_secondary_aci - displayName: "Deploy Secondary ACI" - env: - CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID) - - - ${{ if not(eq(parameters.secondaries.count, 0)) }}: - - job: connect_primary_with_secondaries - displayName: "Connect Primary ACI with Secondaries" - dependsOn: - - generate_ssh_key - - deploy_primary_aci - - deploy_secondary_aci - variables: - Codeql.SkipTaskAutoInjection: true - skipComponentGovernanceDetection: true - sshKey: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] - IpAddresses: $[ dependencies.deploy_primary_aci.outputs['deploy_primary_aci.ipAddresses'] ] - SecondaryIpAddresses: $[ dependencies.deploy_secondary_aci.outputs['deploy_secondary_aci.secondaryIpAddresses'] ] - steps: - - checkout: none - - - template: install_ssh_key.yml - parameters: - ssh_key: $(sshKey) - - - script: | - set -ex - echo $(SecondaryIpAddresses) | base64 -d > ~/secondary_ip_addresses - mapfile -t IP_ADDR_LIST <<< $(echo "$(IpAddresses)" | awk '{print $2}') - for IP_ADDR in "${IP_ADDR_LIST[@]}"; do - ssh agent@$IP_ADDR -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 'echo "Connected successfully"' - scp ~/secondary_ip_addresses agent@$IP_ADDR:${{ parameters.secondaries.path }} - done - displayName: "Connect Primary ACI with Secondaries" - - job: cleanup_aci displayName: "Cleanup ACI" pool: @@ -230,15 +134,5 @@ jobs: name: cleanup_primary_aci displayName: "Delete the primary ACIs and Azure Deployments" continueOnError: true - - - script: | - source ./scripts/azure_deployment/.env/bin/activate - if [[ ${{ parameters.secondaries.count }} != 0 ]]; then - python3.8 scripts/azure_deployment/arm_template.py remove aci \ - --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) \ - --resource-group ccf-aci \ - --aci-type dynamic-agent \ - --deployment-name ci-$(Build.BuildNumber)-secondaries - fi - name: cleanup_secondary_acis - displayName: "Delete the secondary ACIs and Azure Deployments" + env: + CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID) diff --git a/.azure-pipelines-templates/test_on_remote.yml b/.azure-pipelines-templates/test_on_remote.yml index 8e71773958f0..cf37f7503557 100644 --- a/.azure-pipelines-templates/test_on_remote.yml +++ b/.azure-pipelines-templates/test_on_remote.yml @@ -13,7 +13,6 @@ jobs: variables: runOn: ${{ parameters.run_on }} sshKey: ${{ parameters.ssh_key }} - secondaryAcisPath: ${{ parameters.secondary_acis_path }} Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 9010c1667b60..0e40402cc7d0 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -28,10 +28,6 @@ schedules: - main always: true -variables: - - name: secondaryAcisPath - value: "/home/agent/secondary_acis" - resources: containers: - container: virtual @@ -45,9 +41,6 @@ jobs: parameters: used_by: - test_snp - secondaries: - count: 0 # Disabled for now - path: ${{ variables.secondaryAcisPath }} - template: .azure-pipelines-templates/test_on_remote.yml parameters: @@ -58,4 +51,3 @@ jobs: - deploy_primary_aci run_on: $[ dependencies.deploy_primary_aci.outputs['deploy_primary_aci.ipAddresses'] ] ssh_key: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] - secondary_acis_path: ${{ variables.secondaryAcisPath }} diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index 582d3d11dc1c..9a1244fb6514 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -9,7 +9,7 @@ import base64 import tempfile -from azure.identity import DefaultAzureCredential +from azure.identity import AzureCliCredential from azure.mgmt.resource.resources.models import ( Deployment, DeploymentProperties, @@ -108,16 +108,6 @@ def make_dev_container_command(args): ] -def make_attestation_container_command(): - return ["app", "-socket-address", "/mnt/uds/sock"] - - -def make_dummy_business_logic_container_command(): - # Convenient way to to keep dummy business logic container up - # as it uses the same image as the attestation container - return ["app", "-socket-address", "/tmp/unused.sock"] - - def make_dev_container(id, name, image, command, ports, with_volume): t = { "name": f"{name}-{id}", @@ -136,42 +126,6 @@ def make_dev_container(id, name, image, command, ports, with_volume): return t -def make_attestation_container(name, image, command, with_volume): - t = { - "name": name, - "properties": { - "image": image, - "command": command, - "ports": [], - "environmentVariables": [], - "resources": {"requests": {"memoryInGB": 8, "cpu": 2}}, - }, - } - if with_volume: - t["properties"]["volumeMounts"] = [ - {"name": "udsemptydir", "mountPath": "/mnt/uds"}, - ] - return t - - -def make_dummy_business_logic_container(name, image, command, with_volume): - t = { - "name": name, - "properties": { - "image": image, - "command": command, - "ports": [], - "environmentVariables": [], - "resources": {"requests": {"memoryInGB": 8, "cpu": 2}}, - }, - } - if with_volume: - t["properties"]["volumeMounts"] = [ - {"name": "udsemptydir", "mountPath": "/mnt/uds"}, - ] - return t - - def parse_aci_args(parser: ArgumentParser) -> Namespace: # Generic options parser.add_argument( @@ -453,7 +407,7 @@ def make_aci_deployment(args: Namespace) -> Deployment: def remove_aci_deployment(args: Namespace, deployment: Deployment): container_client = ContainerInstanceManagementClient( - DefaultAzureCredential(), args.subscription_id + AzureCliCredential(), args.subscription_id ) for resource in deployment.properties.output_resources: @@ -476,7 +430,7 @@ def check_aci_deployment( """ container_client = ContainerInstanceManagementClient( - DefaultAzureCredential(), args.subscription_id + AzureCliCredential(), args.subscription_id ) for resource in deployment.properties.output_resources: diff --git a/scripts/azure_deployment/arm_template.py b/scripts/azure_deployment/arm_template.py index 08a41fc7dd3a..498a3f39339f 100644 --- a/scripts/azure_deployment/arm_template.py +++ b/scripts/azure_deployment/arm_template.py @@ -2,7 +2,7 @@ # Licensed under the Apache 2.0 License. import argparse -from azure.identity import DefaultAzureCredential +from azure.identity import AzureCliCredential from azure.mgmt.resource import ResourceManagementClient from arm_aci import ( check_aci_deployment, @@ -65,9 +65,7 @@ args, unknown_args = parser.parse_known_args() -resource_client = ResourceManagementClient( - DefaultAzureCredential(), args.subscription_id -) +resource_client = ResourceManagementClient(AzureCliCredential(), args.subscription_id) deployment_type_to_funcs = { "aci": ( diff --git a/tests/code_update.py b/tests/code_update.py index bfc882690b15..7e8bddd64c6b 100644 --- a/tests/code_update.py +++ b/tests/code_update.py @@ -8,7 +8,6 @@ import infra.utils import suite.test_requirements as reqs import os -import time from infra.checker import check_can_progress import infra.snp as snp import tempfile @@ -469,44 +468,6 @@ def test_proposal_invalidation(network, args): return network -@reqs.description( - "Test deploying secondary ACIs which will be used to test SNP code update" -) -@reqs.snp_only() -def test_snp_secondary_deployment(network, args): - LOG.info(f"Secondary ACI information expected at: {args.snp_secondary_acis_path}") - if args.snp_secondary_acis_path is None: - LOG.warning( - "Skipping test snp secondary deployment as no target secondary ACIs specified" - ) - return network - - timeout = 60 * 60 # 60 minutes - start_time = time.time() - end_time = start_time + timeout - - while time.time() < end_time and not os.path.exists(args.snp_secondary_acis_path): - LOG.info( - f"({time.time() - start_time}) Waiting for SNP secondary IP addresses file at: ({args.snp_secondary_acis_path}) to be created" - ) - time.sleep(10) - - if os.path.exists(args.snp_secondary_acis_path): - LOG.info("SNP secondary IP addresses file created") - with open(args.snp_secondary_acis_path, "r", encoding="utf-8") as f: - secondary_acis = [ - tuple(secondary_aci.split(" ")) - for secondary_aci in f.read().splitlines() - ] - for secondary_name, secondary_ip in secondary_acis: - LOG.info( - f'Secondary ACI with name "{secondary_name}" has IP: {secondary_ip}' - ) - - else: - LOG.error("SNP secondary IP addresses file not created before timeout") - - def run(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb @@ -531,8 +492,6 @@ def run(args): # Run again at the end to confirm current nodes are acceptable test_verify_quotes(network, args) - test_snp_secondary_deployment(network, args) - if __name__ == "__main__": args = infra.e2e_args.cli_args() diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index 2616e009ca19..9fc12e4bfcad 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -385,12 +385,6 @@ def cli_args(add=lambda x: None, parser=None, accept_unknown=False): action="append", default=[], ) - parser.add_argument( - "--snp-secondary-acis-path", - help="The location in which the details about secondary ACIs will be stored", - type=str, - default=os.getenv("SECONDARY_ACIS_PATH"), - ) parser.add_argument( "--forwarding-timeout-ms", help="Timeout for forwarded RPC calls (in milliseconds)", From 15785093b1a748064cc005b8416b115b691a7ede Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:31:16 +0100 Subject: [PATCH 062/135] [release/4.x] Cherry pick: Update merklecpp from `1.0.1` to `1.1.0` (#5572) (#5573) --- 3rdparty/exported/merklecpp/merklecpp.h | 42 ++++++++++--------------- cgmanifest.json | 2 +- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/3rdparty/exported/merklecpp/merklecpp.h b/3rdparty/exported/merklecpp/merklecpp.h index a77cb977d78b..6fff26dff924 100644 --- a/3rdparty/exported/merklecpp/merklecpp.h +++ b/3rdparty/exported/merklecpp/merklecpp.h @@ -17,6 +17,7 @@ #include #ifdef HAVE_OPENSSL +# include # include #endif @@ -1390,7 +1391,8 @@ namespace merkle if (index >= num_leaves()) throw std::runtime_error("leaf index out of bounds"); if (index - num_flushed >= leaf_nodes.size()) - return uninserted_leaf_nodes.at(index - num_flushed - leaf_nodes.size()) + return uninserted_leaf_nodes + .at(index - num_flushed - leaf_nodes.size()) ->hash; else return leaf_nodes.at(index - num_flushed)->hash; @@ -1622,7 +1624,8 @@ namespace merkle if (index >= num_leaves()) throw std::runtime_error("leaf index out of bounds"); if (index - num_flushed >= leaf_nodes.size()) - return uninserted_leaf_nodes.at(index - num_flushed - leaf_nodes.size()); + return uninserted_leaf_nodes.at( + index - num_flushed - leaf_nodes.size()); else return leaf_nodes.at(index - num_flushed); } @@ -1734,7 +1737,8 @@ namespace merkle MERKLECPP_TRACE({ std::string nodes; for (size_t i = 0; i < insertion_stack.size(); i++) - nodes += " " + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE); + nodes += + " " + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE); MERKLECPP_TOUT << " X " << (complete ? "complete" : "continue") << ":" << nodes << std::endl; }); @@ -1882,27 +1886,6 @@ namespace merkle // clang-format on #ifdef HAVE_OPENSSL - /// @brief OpenSSL's SHA256 compression function - /// @param l Left node hash - /// @param r Right node hash - /// @param out Output node hash - /// @note Some versions of OpenSSL may not provide SHA256_Transform. - static inline void sha256_compress_openssl( - const HashT<32>& l, const HashT<32>& r, HashT<32>& out) - { - unsigned char block[32 * 2]; - memcpy(&block[0], l.bytes, 32); - memcpy(&block[32], r.bytes, 32); - - SHA256_CTX ctx; - if (SHA256_Init(&ctx) != 1) - printf("SHA256_Init error"); - SHA256_Transform(&ctx, &block[0]); - - for (int i = 0; i < 8; i++) - ((uint32_t*)out.bytes)[i] = convert_endianness(((uint32_t*)ctx.h)[i]); - } - /// @brief OpenSSL SHA256 /// @param l Left node hash /// @param r Right node hash @@ -1916,7 +1899,14 @@ namespace merkle uint8_t block[32 * 2]; memcpy(&block[0], l.bytes, 32); memcpy(&block[32], r.bytes, 32); - SHA256(block, sizeof(block), out.bytes); + + const EVP_MD* md = EVP_sha256(); + int rc = + EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); + if (rc != 1) + { + throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + } } #endif @@ -1967,4 +1957,4 @@ namespace merkle /// @brief Default tree with default hash size and function typedef TreeT<32, sha256_compress> Tree; -}; +}; \ No newline at end of file diff --git a/cgmanifest.json b/cgmanifest.json index c111ba357483..5a05e8f81c2f 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -96,7 +96,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/microsoft/merklecpp", - "commitHash": "2f3853db4b043b2df00869043c34cc1608bdad87" + "commitHash": "1a8a3a000c16fee62bc55d2efdb10d2ebd2b0173" } } }, From 59088dd17ec821db5291675ad72d4e8f251f5290 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:00:48 +0100 Subject: [PATCH 063/135] [release/4.x] Cherry pick: JWT auth: cache verifiers (#5575) (#5576) --- .../ccf/endpoints/authentication/jwt_auth.h | 6 ++ src/endpoints/authentication/jwt_auth.cpp | 92 +++++++++++++------ src/http/http_jwt.h | 6 +- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/include/ccf/endpoints/authentication/jwt_auth.h b/include/ccf/endpoints/authentication/jwt_auth.h index d1ff334a8072..fd12b389fab4 100644 --- a/include/ccf/endpoints/authentication/jwt_auth.h +++ b/include/ccf/endpoints/authentication/jwt_auth.h @@ -17,14 +17,20 @@ namespace ccf nlohmann::json payload; }; + struct VerifiersCache; + class JwtAuthnPolicy : public AuthnPolicy { protected: static const OpenAPISecuritySchema security_schema; + std::unique_ptr verifiers; public: static constexpr auto SECURITY_SCHEME_NAME = "jwt"; + JwtAuthnPolicy(); + virtual ~JwtAuthnPolicy(); + std::unique_ptr authenticate( kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, diff --git a/src/endpoints/authentication/jwt_auth.cpp b/src/endpoints/authentication/jwt_auth.cpp index 3c70966d36d7..ef17ffb9b0bc 100644 --- a/src/endpoints/authentication/jwt_auth.cpp +++ b/src/endpoints/authentication/jwt_auth.cpp @@ -3,13 +3,47 @@ #include "ccf/endpoints/authentication/jwt_auth.h" +#include "ccf/pal/locking.h" #include "ccf/rpc_context.h" #include "ccf/service/tables/jwt.h" +#include "ds/lru.h" #include "enclave/enclave_time.h" #include "http/http_jwt.h" namespace ccf { + struct VerifiersCache + { + static constexpr size_t DEFAULT_MAX_VERIFIERS = 10; + + using DER = std::vector; + ccf::pal::Mutex verifiers_lock; + LRU verifiers; + + VerifiersCache(size_t max_verifiers = DEFAULT_MAX_VERIFIERS) : + verifiers(max_verifiers) + {} + + crypto::VerifierPtr get_verifier(const DER& der) + { + std::lock_guard guard(verifiers_lock); + + auto it = verifiers.find(der); + if (it == verifiers.end()) + { + it = verifiers.insert(der, crypto::make_unique_verifier(der)); + } + + return it->second; + } + }; + + JwtAuthnPolicy::JwtAuthnPolicy() : + verifiers(std::make_unique()) + {} + + JwtAuthnPolicy::~JwtAuthnPolicy() = default; + std::unique_ptr JwtAuthnPolicy::authenticate( kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, @@ -29,43 +63,47 @@ namespace ccf ccf::Tables::JWT_PUBLIC_SIGNING_KEY_ISSUER); const auto key_id = token.header_typed.kid; const auto token_key = keys->get(key_id); + if (!token_key.has_value()) { error_reason = "JWT signing key not found"; } - else if (!http::JwtVerifier::validate_token_signature( - token, token_key.value())) - { - error_reason = "JWT signature is invalid"; - } else { - // Check that the Not Before and Expiration Time claims are valid - const size_t time_now = - std::chrono::duration_cast( - ccf::get_enclave_time()) - .count(); - if (time_now < token.payload_typed.nbf) - { - error_reason = fmt::format( - "Current time {} is before token's Not Before (nbf) claim {}", - time_now, - token.payload_typed.nbf); - } - else if (time_now > token.payload_typed.exp) + auto verifier = verifiers->get_verifier(token_key.value()); + if (!http::JwtVerifier::validate_token_signature(token, verifier)) { - error_reason = fmt::format( - "Current time {} is after token's Expiration Time (exp) claim {}", - time_now, - token.payload_typed.exp); + error_reason = "JWT signature is invalid"; } else { - auto identity = std::make_unique(); - identity->key_issuer = key_issuers->get(key_id).value(); - identity->header = std::move(token.header); - identity->payload = std::move(token.payload); - return identity; + // Check that the Not Before and Expiration Time claims are valid + const size_t time_now = + std::chrono::duration_cast( + ccf::get_enclave_time()) + .count(); + if (time_now < token.payload_typed.nbf) + { + error_reason = fmt::format( + "Current time {} is before token's Not Before (nbf) claim {}", + time_now, + token.payload_typed.nbf); + } + else if (time_now > token.payload_typed.exp) + { + error_reason = fmt::format( + "Current time {} is after token's Expiration Time (exp) claim {}", + time_now, + token.payload_typed.exp); + } + else + { + auto identity = std::make_unique(); + identity->key_issuer = key_issuers->get(key_id).value(); + identity->header = std::move(token.header); + identity->payload = std::move(token.payload); + return identity; + } } } } diff --git a/src/http/http_jwt.h b/src/http/http_jwt.h index e300214105e1..64e731d3fa05 100644 --- a/src/http/http_jwt.h +++ b/src/http/http_jwt.h @@ -171,16 +171,14 @@ namespace http } static bool validate_token_signature( - const Token& token, std::vector cert_der) + const Token& token, const crypto::VerifierPtr& verifier) { - auto verifier = crypto::make_unique_verifier(cert_der); - bool valid = verifier->verify( + return verifier->verify( (uint8_t*)token.signed_content.data(), token.signed_content.size(), token.signature.data(), token.signature.size(), crypto::MDType::SHA256); - return valid; } }; } From f7d1bfd93d9df0883728fc7db86aad9d8921ed10 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:11:29 +0100 Subject: [PATCH 064/135] [release/4.x] Cherry pick: Install testing tools for release jobs (#5580) (#5585) --- .azure-pipelines-templates/matrix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index 71fe0a34d0ab..95a00b05e39b 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -122,7 +122,7 @@ jobs: artifact_name: "SGX_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure - installExtendedTestingTools: false + installExtendedTestingTools: true - template: common.yml parameters: @@ -134,7 +134,7 @@ jobs: artifact_name: "SNPCC_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure - installExtendedTestingTools: false + installExtendedTestingTools: true - template: common.yml parameters: @@ -146,7 +146,7 @@ jobs: artifact_name: "Virtual_Release" ctest_filter: "${{ parameters.test.release.ctest_args }}" depends_on: configure - installExtendedTestingTools: false + installExtendedTestingTools: true # Build that produces unsafe binaries for troubleshooting purposes - template: common.yml From 324bd9f2c7d3d6695400fcaf5457e566348006f9 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 23 Aug 2023 11:43:52 +0100 Subject: [PATCH 065/135] [release/4.x] Cherry pick: Fix schema test in SNP (#5581) (#5584) Co-authored-by: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Co-authored-by: Julien Maffre --- .snpcc_canary | 2 +- tests/e2e_operations.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.snpcc_canary b/.snpcc_canary index 34ebaf23d036..9b2e4859708e 100644 --- a/.snpcc_canary +++ b/.snpcc_canary @@ -1,4 +1,4 @@ ___ ___ ___ - (. =) Y (9 3) (* *) Y + (. =) Y (. 3) (* *) Y O / . | / /-xXx--//-----x=x--/-xXx--/---x---->xxxx diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index ce4433243062..d878e241cc6b 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -18,6 +18,8 @@ import json import subprocess import time +import http +import infra.snp as snp from loguru import logger as LOG @@ -372,9 +374,18 @@ def run_tls_san_checks(args): LOG.info("No config at all") assert not os.path.exists(os.path.join(start_node_path, "0.config.json")) LOG.info(f"Attempt to start node without a config under {start_node_path}") + config_timeout = 10 + env = {} + if args.enclave_platform == "snp": + env = snp.get_aci_env() + env["ASAN_OPTIONS"] = "alloc_dealloc_mismatch=0" + proc = subprocess.Popen( ["./cchost", "--config", "0.config.json", "--config-timeout", "10s"], cwd=start_node_path, + env=env, + stdout=open(os.path.join(start_node_path, "out"), "wb"), + stderr=open(os.path.join(start_node_path, "err"), "wb"), ) time.sleep(2) LOG.info("Copy a partial config") From 5ec29ab5f12a91d89dcaebfafb4a51c67924bba6 Mon Sep 17 00:00:00 2001 From: "CCF [bot]" <62645686+ccf-bot@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:15:53 +0100 Subject: [PATCH 066/135] [release/4.x] Cherry pick: Fix SNP reconfiguration test error (#5577) (#5586) --- .../pal/attestation_sev_snp_endorsements.h | 2 ++ scripts/azure_deployment/arm_template.py | 2 +- src/node/quote_endorsements_client.h | 27 +++++++++++-------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/include/ccf/pal/attestation_sev_snp_endorsements.h b/include/ccf/pal/attestation_sev_snp_endorsements.h index 33d62afa938c..453033d1ff7f 100644 --- a/include/ccf/pal/attestation_sev_snp_endorsements.h +++ b/include/ccf/pal/attestation_sev_snp_endorsements.h @@ -44,6 +44,8 @@ namespace ccf::pal::snp std::string uri; std::map params; bool response_is_der = false; + + bool operator==(const EndpointInfo&) const = default; }; using Server = std::list; diff --git a/scripts/azure_deployment/arm_template.py b/scripts/azure_deployment/arm_template.py index 498a3f39339f..c2308d46efde 100644 --- a/scripts/azure_deployment/arm_template.py +++ b/scripts/azure_deployment/arm_template.py @@ -90,7 +90,7 @@ def deploy(args, make_template) -> str: def remove(args, remove_deployment, deployment): try: - # Call deployement type specific removal + # Call deployment type specific removal remove_deployment( args, deployment, diff --git a/src/node/quote_endorsements_client.h b/src/node/quote_endorsements_client.h index cf7d8b7c994c..cb770406bab2 100644 --- a/src/node/quote_endorsements_client.h +++ b/src/node/quote_endorsements_client.h @@ -130,7 +130,7 @@ namespace ccf auto& server = servers.front(); LOG_FAIL_FMT( "Giving up retrying fetching attestation endorsements from " - "{} after {} attempts ", + "{} after {} attempts", server.front().host, max_server_retries_count); return; @@ -149,16 +149,23 @@ namespace ccf std::chrono::milliseconds(server_connection_timeout_s * 1000)); } - void handle_success_response(std::vector&& data, bool is_der) + void handle_success_response( + std::vector&& data, const EndpointInfo& response_endpoint) { - if (has_completed) + // We may receive a response to an in-flight request after having + // fetched all endorsements + auto& server = config.servers.front(); + if (server.empty()) + { + return; + } + auto endpoint = server.front(); + if (has_completed || response_endpoint != endpoint) { - // We may receive a response to an in-flight request after having - // fetched all endorsements return; } - if (is_der) + if (response_endpoint.response_is_der) { auto raw = crypto::cert_der_to_pem(data).raw(); endorsements_pem.insert(endorsements_pem.end(), raw.begin(), raw.end()); @@ -169,7 +176,6 @@ namespace ccf endorsements_pem.end(), data.begin(), data.end()); } - auto& server = config.servers.front(); server.pop_front(); if (server.empty()) { @@ -185,18 +191,17 @@ namespace ccf void fetch(const Server& server) { - auto& endpoint = server.front(); + auto endpoint = server.front(); auto c = create_unauthenticated_client(); c->connect( endpoint.host, endpoint.port, - [this, server]( + [this, server, endpoint]( http_status status, http::HeaderMap&& headers, std::vector&& data) { last_received_request_id++; - auto& endpoint = server.front(); if (status == HTTP_STATUS_OK) { @@ -205,7 +210,7 @@ namespace ccf "{} bytes", data.size()); - handle_success_response(std::move(data), endpoint.response_is_der); + handle_success_response(std::move(data), endpoint); return; } From d01a4adc0e807be69fe81a7f0af937a8991f290e Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 24 Aug 2023 10:20:00 +0100 Subject: [PATCH 067/135] [release/4.x] Cherry pick: Add app and gov read endpoints (#5574) (#5587) Co-authored-by: Julien Maffre <42961061+jumaffre@users.noreply.github.com> --- CHANGELOG.md | 6 +++ doc/schemas/node_openapi.json | 32 ++++++++++++- src/node/node_state.h | 24 +++++++++- src/node/rpc/node_frontend.h | 64 ++++++++++++++++++++++++- src/node/rpc/node_interface.h | 3 ++ src/node/rpc/node_operation.h | 15 ++++++ src/node/rpc/node_operation_interface.h | 4 ++ src/node/rpc/test/node_stub.h | 15 ++++++ tests/e2e_common_endpoints.py | 13 +++++ tests/e2e_operations.py | 25 ++++++++++ tests/reconfiguration.py | 6 +++ tests/recovery.py | 8 ++++ 12 files changed, 211 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea3726b575b2..1755bf9748f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [4.0.8] + +[4.0.8]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.8 + +- Add `/node/ready/app` and `/node/ready/gov` endpoints for the use of load balancers wanting to check if a node is ready to accept application or governance transactions. See [Operator RPC API](https://microsoft.github.io/CCF/main/operations/operator_rpc_api.html) for details. + ## [4.0.7] [4.0.7]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.7 diff --git a/doc/schemas/node_openapi.json b/doc/schemas/node_openapi.json index f8a981c9b857..d307bfe36791 100644 --- a/doc/schemas/node_openapi.json +++ b/doc/schemas/node_openapi.json @@ -908,7 +908,7 @@ "info": { "description": "This API provides public, uncredentialed access to service and node state.", "title": "CCF Public Node API", - "version": "4.2.0" + "version": "4.3.0" }, "openapi": "3.0.0", "paths": { @@ -1412,6 +1412,36 @@ } } }, + "/node/ready/app": { + "get": { + "responses": { + "204": { + "description": "Default response description" + }, + "default": { + "$ref": "#/components/responses/default" + } + }, + "x-ccf-forwarding": { + "$ref": "#/components/x-ccf-forwarding/never" + } + } + }, + "/node/ready/gov": { + "get": { + "responses": { + "204": { + "description": "Default response description" + }, + "default": { + "$ref": "#/components/responses/default" + } + }, + "x-ccf-forwarding": { + "$ref": "#/components/x-ccf-forwarding/never" + } + } + }, "/node/receipt": { "get": { "description": "A signed statement from the service over a transaction entry in the ledger", diff --git a/src/node/node_state.h b/src/node/node_state.h index c106e3703bee..05dfe3387d74 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -1682,6 +1682,14 @@ namespace ccf return sm.check(NodeStartupState::partOfPublicNetwork); } + bool is_accessible_to_members() const override + { + const auto val = sm.value(); + return val == NodeStartupState::partOfNetwork || + val == NodeStartupState::partOfPublicNetwork || + val == NodeStartupState::readingPrivateLedger; + } + ExtendedState state() override { std::lock_guard guard(lock); @@ -1858,11 +1866,23 @@ namespace ccf open_frontend(ActorsType::users, &network.identity->cert); } - bool is_member_frontend_open() + bool is_member_frontend_open_unsafe() { return find_frontend(ActorsType::members)->is_open(); } + bool is_member_frontend_open() override + { + std::lock_guard guard(lock); + return is_member_frontend_open_unsafe(); + } + + bool is_user_frontend_open() override + { + std::lock_guard guard(lock); + return find_frontend(ActorsType::users)->is_open(); + } + std::shared_ptr find_acme_challenge_frontend() { auto acme_challenge_opt = rpc_map->find(ActorsType::acme_challenge); @@ -2240,7 +2260,7 @@ namespace ccf accept_network_tls_connections(); - if (is_member_frontend_open()) + if (is_member_frontend_open_unsafe()) { // Also, automatically refresh self-signed node certificate, // using the same validity period as the endorsed certificate. diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 5a36da05cb62..e010dd08fca4 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -373,7 +373,7 @@ namespace ccf openapi_info.description = "This API provides public, uncredentialed access to service and node " "state."; - openapi_info.document_version = "4.2.0"; + openapi_info.document_version = "4.3.0"; } void init_handlers() override @@ -1647,6 +1647,68 @@ namespace ccf .set_forwarding_required(endpoints::ForwardingRequired::Never) .set_auto_schema() .install(); + + auto get_ready_app = + [this](const ccf::endpoints::ReadOnlyEndpointContext& ctx) { + auto node_configuration_subsystem = + this->context.get_subsystem(); + if (!node_configuration_subsystem) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "NodeConfigurationSubsystem is not available"); + return; + } + if ( + !node_configuration_subsystem->has_received_stop_notice() && + this->node_operation.is_part_of_network() && + this->node_operation.is_user_frontend_open()) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); + } + else + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_SERVICE_UNAVAILABLE); + } + return; + }; + make_read_only_endpoint( + "/ready/app", HTTP_GET, get_ready_app, no_auth_required) + .set_auto_schema() + .set_forwarding_required(endpoints::ForwardingRequired::Never) + .install(); + + auto get_ready_gov = + [this](const ccf::endpoints::ReadOnlyEndpointContext& ctx) { + auto node_configuration_subsystem = + this->context.get_subsystem(); + if (!node_configuration_subsystem) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "NodeConfigurationSubsystem is not available"); + return; + } + if ( + !node_configuration_subsystem->has_received_stop_notice() && + this->node_operation.is_accessible_to_members() && + this->node_operation.is_member_frontend_open()) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); + } + else + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_SERVICE_UNAVAILABLE); + } + return; + }; + make_read_only_endpoint( + "/ready/gov", HTTP_GET, get_ready_gov, no_auth_required) + .set_auto_schema() + .set_forwarding_required(endpoints::ForwardingRequired::Never) + .install(); } }; diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index 4d24237de4ba..02b036d7843f 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -61,6 +61,9 @@ namespace ccf virtual crypto::Pem get_network_cert() = 0; virtual void stop_notice() = 0; virtual bool has_received_stop_notice() = 0; + virtual bool is_member_frontend_open() = 0; + virtual bool is_user_frontend_open() = 0; + virtual bool is_accessible_to_members() const = 0; virtual void make_http_request( const http::URL& url, diff --git a/src/node/rpc/node_operation.h b/src/node/rpc/node_operation.h index 748f68b49039..65d6f62e5659 100644 --- a/src/node/rpc/node_operation.h +++ b/src/node/rpc/node_operation.h @@ -45,6 +45,21 @@ namespace ccf return impl.is_reading_private_ledger(); } + bool is_user_frontend_open() override + { + return impl.is_user_frontend_open(); + } + + bool is_member_frontend_open() override + { + return impl.is_member_frontend_open(); + } + + bool is_accessible_to_members() const override + { + return impl.is_accessible_to_members(); + } + bool can_replicate() override { return impl.can_replicate(); diff --git a/src/node/rpc/node_operation_interface.h b/src/node/rpc/node_operation_interface.h index b08caea70270..4b09c5efb424 100644 --- a/src/node/rpc/node_operation_interface.h +++ b/src/node/rpc/node_operation_interface.h @@ -36,6 +36,10 @@ namespace ccf virtual bool is_reading_public_ledger() const = 0; virtual bool is_reading_private_ledger() const = 0; + virtual bool is_user_frontend_open() = 0; + virtual bool is_member_frontend_open() = 0; + virtual bool is_accessible_to_members() const = 0; + virtual bool can_replicate() = 0; virtual kv::Version get_last_recovered_signed_idx() = 0; diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index d7fd35eb1f44..7e3aec7de8c8 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -47,6 +47,21 @@ namespace ccf return false; } + bool is_member_frontend_open() override + { + return true; + } + + bool is_user_frontend_open() override + { + return true; + } + + bool is_accessible_to_members() const override + { + return true; + } + bool can_replicate() override { return true; diff --git a/tests/e2e_common_endpoints.py b/tests/e2e_common_endpoints.py index 3611b0509150..bf0a350cce03 100644 --- a/tests/e2e_common_endpoints.py +++ b/tests/e2e_common_endpoints.py @@ -160,6 +160,18 @@ def test_memory(network, args): return network +@reqs.description("Frontend readiness") +def test_readiness(network, args): + primary, _ = network.find_primary() + with primary.client() as c: + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + + return network + + @reqs.description("Write/Read large messages on primary") def test_large_messages(network, args): primary, _ = network.find_primary() @@ -283,3 +295,4 @@ def run(args): test_node_ids(network, args) test_memory(network, args) test_large_messages(network, args) + test_readiness(network, args) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index d878e241cc6b..46fa9ee739c9 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -427,6 +427,30 @@ def run_configuration_file_checks(args): assert rc == 0, f"Failed to check configuration: {rc}" +def run_preopen_readiness_check(args): + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + args.perf_nodes, + pdb=args.pdb, + ) as network: + args.common_read_only_ledger_dir = None # Reset from previous test + network.start(args) + primary, _ = network.find_primary() + with primary.client() as c: + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value, r + network.open(args) + with primary.client() as c: + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + + def run_pid_file_check(args): with infra.network.network( args.nodes, @@ -467,3 +491,4 @@ def run(args): run_tls_san_checks(args) run_configuration_file_checks(args) run_pid_file_check(args) + run_preopen_readiness_check(args) diff --git a/tests/reconfiguration.py b/tests/reconfiguration.py index 9215c828819a..66f40308234c 100644 --- a/tests/reconfiguration.py +++ b/tests/reconfiguration.py @@ -164,6 +164,12 @@ def test_ignore_first_sigterm(network, args): new_node.sigterm() + with new_node.client() as c: + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value, r + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value, r + with new_node.client() as c: r = c.get("/node/state") assert r.body.json()["stop_notice"] is True, r diff --git a/tests/recovery.py b/tests/recovery.py index 250154536346..c22f09509535 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -126,6 +126,10 @@ def test_recover_service(network, args, from_snapshot=True): assert ( received_prev_ident == prev_ident ), f"Response doesn't match previous identity: {received_prev_ident} != {prev_ident}" + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value, r recovered_network.recover(args) @@ -138,6 +142,10 @@ def test_recover_service(network, args, from_snapshot=True): ).view == prev_view + 2 ) + r = c.get("/node/ready/gov") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r + r = c.get("/node/ready/app") + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r return recovered_network From 962e7c9cc7bc6bb5f0745beeb207e5caa73eb1fe Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Tue, 5 Sep 2023 09:33:57 +0100 Subject: [PATCH 068/135] [release/4.x] Cherry pick: Upgrade `nghttp` from `1.51.0` to `1.55.1` (#5604) (#5611) --- .daily_canary | 8 +- 3rdparty/exported/nghttp2/CMakeLists.txt | 1 + 3rdparty/exported/nghttp2/Makefile.am | 6 +- 3rdparty/exported/nghttp2/Makefile.in | 37 +- 3rdparty/exported/nghttp2/Makefile.msvc | 38 +- .../exported/nghttp2/includes/Makefile.in | 23 +- .../nghttp2/includes/nghttp2/nghttp2ver.h | 4 +- 3rdparty/exported/nghttp2/nghttp2_http.c | 716 +--------- 3rdparty/exported/nghttp2/nghttp2_http.h | 48 - 3rdparty/exported/nghttp2/nghttp2_map.c | 61 +- 3rdparty/exported/nghttp2/nghttp2_map.h | 8 +- 3rdparty/exported/nghttp2/nghttp2_session.c | 34 +- 3rdparty/exported/nghttp2/sfparse.c | 1146 +++++++++++++++++ 3rdparty/exported/nghttp2/sfparse.h | 409 ++++++ CHANGELOG.md | 1 + cgmanifest.json | 2 +- cmake/nghttp2.cmake | 1 + 17 files changed, 1674 insertions(+), 869 deletions(-) create mode 100644 3rdparty/exported/nghttp2/sfparse.c create mode 100644 3rdparty/exported/nghttp2/sfparse.h diff --git a/.daily_canary b/.daily_canary index 332142186300..04cf585147f0 100644 --- a/.daily_canary +++ b/.daily_canary @@ -1,4 +1,4 @@ - --- ___ ___ - (- -) (o =) | Y & +-- - ( V ) z x z O +---=---' -/--x-m- /--n-m---xXx--/--yY------ + -^- ___ ___ + (- -) (= =) | Y & +--? + ( V ) / . \ O +---=---' +/--x-m- /--n-n---xXx--/--yY------ diff --git a/3rdparty/exported/nghttp2/CMakeLists.txt b/3rdparty/exported/nghttp2/CMakeLists.txt index fb24fa186f11..fad04716ffaf 100644 --- a/3rdparty/exported/nghttp2/CMakeLists.txt +++ b/3rdparty/exported/nghttp2/CMakeLists.txt @@ -25,6 +25,7 @@ set(NGHTTP2_SOURCES nghttp2_rcbuf.c nghttp2_extpri.c nghttp2_debug.c + sfparse.c ) set(NGHTTP2_RES "") diff --git a/3rdparty/exported/nghttp2/Makefile.am b/3rdparty/exported/nghttp2/Makefile.am index 9a985bf76b28..cd928ae46550 100644 --- a/3rdparty/exported/nghttp2/Makefile.am +++ b/3rdparty/exported/nghttp2/Makefile.am @@ -51,7 +51,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \ nghttp2_http.c \ nghttp2_rcbuf.c \ nghttp2_extpri.c \ - nghttp2_debug.c + nghttp2_debug.c \ + sfparse.c HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_frame.h \ @@ -68,7 +69,8 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_http.h \ nghttp2_rcbuf.h \ nghttp2_extpri.h \ - nghttp2_debug.h + nghttp2_debug.h \ + sfparse.h libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS) libnghttp2_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined \ diff --git a/3rdparty/exported/nghttp2/Makefile.in b/3rdparty/exported/nghttp2/Makefile.in index 6b8722d2d9a9..24a6cc468ed7 100644 --- a/3rdparty/exported/nghttp2/Makefile.in +++ b/3rdparty/exported/nghttp2/Makefile.in @@ -107,13 +107,8 @@ host_triplet = @host@ target_triplet = @target@ subdir = lib ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_boost_asio.m4 \ - $(top_srcdir)/m4/ax_boost_base.m4 \ - $(top_srcdir)/m4/ax_boost_system.m4 \ - $(top_srcdir)/m4/ax_boost_thread.m4 \ - $(top_srcdir)/m4/ax_check_compile_flag.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_compile_flag.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_python_devel.m4 \ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/configure.ac @@ -162,7 +157,7 @@ am__objects_2 = nghttp2_pq.lo nghttp2_map.lo nghttp2_queue.lo \ nghttp2_hd_huffman.lo nghttp2_hd_huffman_data.lo \ nghttp2_version.lo nghttp2_priority_spec.lo nghttp2_option.lo \ nghttp2_callbacks.lo nghttp2_mem.lo nghttp2_http.lo \ - nghttp2_rcbuf.lo nghttp2_extpri.lo nghttp2_debug.lo + nghttp2_rcbuf.lo nghttp2_extpri.lo nghttp2_debug.lo sfparse.lo am_libnghttp2_la_OBJECTS = $(am__objects_1) $(am__objects_2) libnghttp2_la_OBJECTS = $(am_libnghttp2_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) @@ -201,7 +196,8 @@ am__depfiles_remade = ./$(DEPDIR)/nghttp2_buf.Plo \ ./$(DEPDIR)/nghttp2_priority_spec.Plo \ ./$(DEPDIR)/nghttp2_queue.Plo ./$(DEPDIR)/nghttp2_rcbuf.Plo \ ./$(DEPDIR)/nghttp2_session.Plo ./$(DEPDIR)/nghttp2_stream.Plo \ - ./$(DEPDIR)/nghttp2_submit.Plo ./$(DEPDIR)/nghttp2_version.Plo + ./$(DEPDIR)/nghttp2_submit.Plo ./$(DEPDIR)/nghttp2_version.Plo \ + ./$(DEPDIR)/sfparse.Plo am__mv = mv -f COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) @@ -300,11 +296,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -BOOST_ASIO_LIB = @BOOST_ASIO_LIB@ -BOOST_CPPFLAGS = @BOOST_CPPFLAGS@ -BOOST_LDFLAGS = @BOOST_LDFLAGS@ -BOOST_SYSTEM_LIB = @BOOST_SYSTEM_LIB@ -BOOST_THREAD_LIB = @BOOST_THREAD_LIB@ BPFCFLAGS = @BPFCFLAGS@ CC = @CC@ CCDEPMODE = @CCDEPMODE@ @@ -321,7 +312,6 @@ CXXCPP = @CXXCPP@ CXXDEPMODE = @CXXDEPMODE@ CXXFLAGS = @CXXFLAGS@ CYGPATH_W = @CYGPATH_W@ -CYTHON = @CYTHON@ DEFS = @DEFS@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ @@ -366,8 +356,8 @@ LIBNGHTTP3_LIBS = @LIBNGHTTP3_LIBS@ LIBNGTCP2_CFLAGS = @LIBNGTCP2_CFLAGS@ LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS = @LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS@ LIBNGTCP2_CRYPTO_BORINGSSL_LIBS = @LIBNGTCP2_CRYPTO_BORINGSSL_LIBS@ -LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS = @LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ -LIBNGTCP2_CRYPTO_OPENSSL_LIBS = @LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ +LIBNGTCP2_CRYPTO_QUICTLS_CFLAGS = @LIBNGTCP2_CRYPTO_QUICTLS_CFLAGS@ +LIBNGTCP2_CRYPTO_QUICTLS_LIBS = @LIBNGTCP2_CRYPTO_QUICTLS_LIBS@ LIBNGTCP2_LIBS = @LIBNGTCP2_LIBS@ LIBOBJS = @LIBOBJS@ LIBS = @LIBS@ @@ -406,15 +396,9 @@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ PYTHON = @PYTHON@ -PYTHON_CPPFLAGS = @PYTHON_CPPFLAGS@ PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ -PYTHON_EXTRA_LDFLAGS = @PYTHON_EXTRA_LDFLAGS@ -PYTHON_EXTRA_LIBS = @PYTHON_EXTRA_LIBS@ -PYTHON_LIBS = @PYTHON_LIBS@ PYTHON_PLATFORM = @PYTHON_PLATFORM@ -PYTHON_PLATFORM_SITE_PKG = @PYTHON_PLATFORM_SITE_PKG@ PYTHON_PREFIX = @PYTHON_PREFIX@ -PYTHON_SITE_PKG = @PYTHON_SITE_PKG@ PYTHON_VERSION = @PYTHON_VERSION@ RANLIB = @RANLIB@ SED = @SED@ @@ -526,7 +510,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \ nghttp2_http.c \ nghttp2_rcbuf.c \ nghttp2_extpri.c \ - nghttp2_debug.c + nghttp2_debug.c \ + sfparse.c HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_frame.h \ @@ -543,7 +528,8 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_http.h \ nghttp2_rcbuf.h \ nghttp2_extpri.h \ - nghttp2_debug.h + nghttp2_debug.h \ + sfparse.h libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS) libnghttp2_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined \ @@ -652,6 +638,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nghttp2_stream.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nghttp2_submit.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nghttp2_version.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sfparse.Plo@am__quote@ # am--include-marker $(am__depfiles_remade): @$(MKDIR_P) $(@D) @@ -934,6 +921,7 @@ distclean: distclean-recursive -rm -f ./$(DEPDIR)/nghttp2_stream.Plo -rm -f ./$(DEPDIR)/nghttp2_submit.Plo -rm -f ./$(DEPDIR)/nghttp2_version.Plo + -rm -f ./$(DEPDIR)/sfparse.Plo -rm -f Makefile distclean-am: clean-am distclean-compile distclean-generic \ distclean-tags @@ -1002,6 +990,7 @@ maintainer-clean: maintainer-clean-recursive -rm -f ./$(DEPDIR)/nghttp2_stream.Plo -rm -f ./$(DEPDIR)/nghttp2_submit.Plo -rm -f ./$(DEPDIR)/nghttp2_version.Plo + -rm -f ./$(DEPDIR)/sfparse.Plo -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic diff --git a/3rdparty/exported/nghttp2/Makefile.msvc b/3rdparty/exported/nghttp2/Makefile.msvc index f649c0bda4bc..611b39d0b1d9 100644 --- a/3rdparty/exported/nghttp2/Makefile.msvc +++ b/3rdparty/exported/nghttp2/Makefile.msvc @@ -6,15 +6,8 @@ # The MIT License apply. # -# -# Choose your weapons: -# Set 'USE_CYTHON=1' to build and install the 'nghttp2.pyd' Python extension. -# THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) -USE_CYTHON := 0 -#USE_CYTHON := 1 - _VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -e 's/-DEV//g' -e 's/], //g') _VERSION := $(subst ., ,$(_VERSION)) VER_MAJOR := $(word 1,$(_VERSION)) @@ -102,7 +95,7 @@ NGHTTP2_OBJ_D := $(addprefix $(OBJ_DIR)/d_, $(notdir $(NGHTTP2_SRC:.c=.obj))) clean_nghttp2_pyd_0 clean_nghttp2_pyd_1 -all: intro includes/nghttp2/nghttp2ver.h $(OBJ_DIR) $(TARGETS) build_nghttp2_pyd_$(USE_CYTHON) +all: intro includes/nghttp2/nghttp2ver.h $(OBJ_DIR) $(TARGETS) @echo 'Welcome to NgHTTP2 (release + debug).' @echo 'Do a "make -f Makefile.MSVC install" at own risk!' @@ -121,7 +114,7 @@ $(OBJ_DIR): install: includes/nghttp2/nghttp2.h includes/nghttp2/nghttp2ver.h \ $(TARGETS) \ - copy_headers_and_libs install_nghttp2_pyd_$(USE_CYTHON) + copy_headers_and_libs # # This MUST be done before using the 'install_nghttp2_pyd_1' rule. @@ -160,31 +153,6 @@ $(DLL_D): $(NGHTTP2_OBJ_D) $(OBJ_DIR)/d_nghttp2.res WIN_OBJDIR:=$(shell cygpath -w $(abspath $(OBJ_DIR))) WIN_OBJDIR:=$(subst \,/,$(WIN_OBJDIR)) -../python/setup.py: ../python/setup.py.in $(THIS_MAKEFILE) - cd ../python ; \ - echo '# $(GENERATED). DO NOT EDIT.' > setup.py ; \ - sed -e 's/@top_srcdir@/../' \ - -e 's%@top_builddir@%$(WIN_OBJDIR)%' \ - -e 's/@PACKAGE_VERSION@/$(VERSION)/' setup.py.in >> setup.py ; - -build_nghttp2_pyd_0: ; - -build_nghttp2_pyd_1: $(addprefix ../python/, setup.py nghttp2.pyx) - cd ../python ; \ - python setup.py build_ext -i -f bdist_wininst - -install_nghttp2_pyd_0: ; - -install_nghttp2_pyd_1: $(addprefix ../python/, setup.py nghttp2.pyx) - cd ../python ; \ - pip install . - -clean_nghttp2_pyd_0: ; - -clean_nghttp2_pyd_1: - cd ../python ; \ - rm -fR build dist - $(OBJ_DIR)/r_%.obj: %.c $(THIS_MAKEFILE) $(CC) $(CFLAGS_R) $(CFLAGS) -Fo$@ -c $< @echo @@ -262,7 +230,7 @@ clean: rm -f $(OBJ_DIR)/* includes/nghttp2/nghttp2ver.h @echo -vclean realclean: clean clean_nghttp2_pyd_$(USE_CYTHON) +vclean realclean: clean - rm -rf $(OBJ_DIR) - rm -f .depend.MSVC diff --git a/3rdparty/exported/nghttp2/includes/Makefile.in b/3rdparty/exported/nghttp2/includes/Makefile.in index 1943ca83222b..3de90d7bef3e 100644 --- a/3rdparty/exported/nghttp2/includes/Makefile.in +++ b/3rdparty/exported/nghttp2/includes/Makefile.in @@ -114,13 +114,8 @@ host_triplet = @host@ target_triplet = @target@ subdir = lib/includes ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_boost_asio.m4 \ - $(top_srcdir)/m4/ax_boost_base.m4 \ - $(top_srcdir)/m4/ax_boost_system.m4 \ - $(top_srcdir)/m4/ax_boost_thread.m4 \ - $(top_srcdir)/m4/ax_check_compile_flag.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_compile_flag.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_python_devel.m4 \ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/configure.ac @@ -208,11 +203,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -BOOST_ASIO_LIB = @BOOST_ASIO_LIB@ -BOOST_CPPFLAGS = @BOOST_CPPFLAGS@ -BOOST_LDFLAGS = @BOOST_LDFLAGS@ -BOOST_SYSTEM_LIB = @BOOST_SYSTEM_LIB@ -BOOST_THREAD_LIB = @BOOST_THREAD_LIB@ BPFCFLAGS = @BPFCFLAGS@ CC = @CC@ CCDEPMODE = @CCDEPMODE@ @@ -229,7 +219,6 @@ CXXCPP = @CXXCPP@ CXXDEPMODE = @CXXDEPMODE@ CXXFLAGS = @CXXFLAGS@ CYGPATH_W = @CYGPATH_W@ -CYTHON = @CYTHON@ DEFS = @DEFS@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ @@ -274,8 +263,8 @@ LIBNGHTTP3_LIBS = @LIBNGHTTP3_LIBS@ LIBNGTCP2_CFLAGS = @LIBNGTCP2_CFLAGS@ LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS = @LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS@ LIBNGTCP2_CRYPTO_BORINGSSL_LIBS = @LIBNGTCP2_CRYPTO_BORINGSSL_LIBS@ -LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS = @LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ -LIBNGTCP2_CRYPTO_OPENSSL_LIBS = @LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ +LIBNGTCP2_CRYPTO_QUICTLS_CFLAGS = @LIBNGTCP2_CRYPTO_QUICTLS_CFLAGS@ +LIBNGTCP2_CRYPTO_QUICTLS_LIBS = @LIBNGTCP2_CRYPTO_QUICTLS_LIBS@ LIBNGTCP2_LIBS = @LIBNGTCP2_LIBS@ LIBOBJS = @LIBOBJS@ LIBS = @LIBS@ @@ -314,15 +303,9 @@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ PYTHON = @PYTHON@ -PYTHON_CPPFLAGS = @PYTHON_CPPFLAGS@ PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ -PYTHON_EXTRA_LDFLAGS = @PYTHON_EXTRA_LDFLAGS@ -PYTHON_EXTRA_LIBS = @PYTHON_EXTRA_LIBS@ -PYTHON_LIBS = @PYTHON_LIBS@ PYTHON_PLATFORM = @PYTHON_PLATFORM@ -PYTHON_PLATFORM_SITE_PKG = @PYTHON_PLATFORM_SITE_PKG@ PYTHON_PREFIX = @PYTHON_PREFIX@ -PYTHON_SITE_PKG = @PYTHON_SITE_PKG@ PYTHON_VERSION = @PYTHON_VERSION@ RANLIB = @RANLIB@ SED = @SED@ diff --git a/3rdparty/exported/nghttp2/includes/nghttp2/nghttp2ver.h b/3rdparty/exported/nghttp2/includes/nghttp2/nghttp2ver.h index 6fe61752afb9..fba310c7888d 100644 --- a/3rdparty/exported/nghttp2/includes/nghttp2/nghttp2ver.h +++ b/3rdparty/exported/nghttp2/includes/nghttp2/nghttp2ver.h @@ -29,7 +29,7 @@ * @macro * Version number of the nghttp2 library release */ -#define NGHTTP2_VERSION "1.51.0" +#define NGHTTP2_VERSION "1.55.1" /** * @macro @@ -37,6 +37,6 @@ * release. This is a 24 bit number with 8 bits for major number, 8 bits * for minor and 8 bits for patch. Version 1.2.3 becomes 0x010203. */ -#define NGHTTP2_VERSION_NUM 0x013300 +#define NGHTTP2_VERSION_NUM 0x013701 #endif /* NGHTTP2VER_H */ diff --git a/3rdparty/exported/nghttp2/nghttp2_http.c b/3rdparty/exported/nghttp2/nghttp2_http.c index 83e5e6685f86..ecdeb21ddb69 100644 --- a/3rdparty/exported/nghttp2/nghttp2_http.c +++ b/3rdparty/exported/nghttp2/nghttp2_http.c @@ -31,6 +31,7 @@ #include "nghttp2_hd.h" #include "nghttp2_helper.h" #include "nghttp2_extpri.h" +#include "sfparse.h" static uint8_t downcase(uint8_t c) { return 'A' <= c && c <= 'Z' ? (uint8_t)(c - 'A' + 'a') : c; @@ -578,713 +579,52 @@ void nghttp2_http_record_request_method(nghttp2_stream *stream, } } -/* Generated by genchartbl.py */ -static const int SF_KEY_CHARS[] = { - 0 /* NUL */, 0 /* SOH */, 0 /* STX */, 0 /* ETX */, 0 /* EOT */, - 0 /* ENQ */, 0 /* ACK */, 0 /* BEL */, 0 /* BS */, 0 /* HT */, - 0 /* LF */, 0 /* VT */, 0 /* FF */, 0 /* CR */, 0 /* SO */, - 0 /* SI */, 0 /* DLE */, 0 /* DC1 */, 0 /* DC2 */, 0 /* DC3 */, - 0 /* DC4 */, 0 /* NAK */, 0 /* SYN */, 0 /* ETB */, 0 /* CAN */, - 0 /* EM */, 0 /* SUB */, 0 /* ESC */, 0 /* FS */, 0 /* GS */, - 0 /* RS */, 0 /* US */, 0 /* SPC */, 0 /* ! */, 0 /* " */, - 0 /* # */, 0 /* $ */, 0 /* % */, 0 /* & */, 0 /* ' */, - 0 /* ( */, 0 /* ) */, 1 /* * */, 0 /* + */, 0 /* , */, - 1 /* - */, 1 /* . */, 0 /* / */, 1 /* 0 */, 1 /* 1 */, - 1 /* 2 */, 1 /* 3 */, 1 /* 4 */, 1 /* 5 */, 1 /* 6 */, - 1 /* 7 */, 1 /* 8 */, 1 /* 9 */, 0 /* : */, 0 /* ; */, - 0 /* < */, 0 /* = */, 0 /* > */, 0 /* ? */, 0 /* @ */, - 0 /* A */, 0 /* B */, 0 /* C */, 0 /* D */, 0 /* E */, - 0 /* F */, 0 /* G */, 0 /* H */, 0 /* I */, 0 /* J */, - 0 /* K */, 0 /* L */, 0 /* M */, 0 /* N */, 0 /* O */, - 0 /* P */, 0 /* Q */, 0 /* R */, 0 /* S */, 0 /* T */, - 0 /* U */, 0 /* V */, 0 /* W */, 0 /* X */, 0 /* Y */, - 0 /* Z */, 0 /* [ */, 0 /* \ */, 0 /* ] */, 0 /* ^ */, - 1 /* _ */, 0 /* ` */, 1 /* a */, 1 /* b */, 1 /* c */, - 1 /* d */, 1 /* e */, 1 /* f */, 1 /* g */, 1 /* h */, - 1 /* i */, 1 /* j */, 1 /* k */, 1 /* l */, 1 /* m */, - 1 /* n */, 1 /* o */, 1 /* p */, 1 /* q */, 1 /* r */, - 1 /* s */, 1 /* t */, 1 /* u */, 1 /* v */, 1 /* w */, - 1 /* x */, 1 /* y */, 1 /* z */, 0 /* { */, 0 /* | */, - 0 /* } */, 0 /* ~ */, 0 /* DEL */, 0 /* 0x80 */, 0 /* 0x81 */, - 0 /* 0x82 */, 0 /* 0x83 */, 0 /* 0x84 */, 0 /* 0x85 */, 0 /* 0x86 */, - 0 /* 0x87 */, 0 /* 0x88 */, 0 /* 0x89 */, 0 /* 0x8a */, 0 /* 0x8b */, - 0 /* 0x8c */, 0 /* 0x8d */, 0 /* 0x8e */, 0 /* 0x8f */, 0 /* 0x90 */, - 0 /* 0x91 */, 0 /* 0x92 */, 0 /* 0x93 */, 0 /* 0x94 */, 0 /* 0x95 */, - 0 /* 0x96 */, 0 /* 0x97 */, 0 /* 0x98 */, 0 /* 0x99 */, 0 /* 0x9a */, - 0 /* 0x9b */, 0 /* 0x9c */, 0 /* 0x9d */, 0 /* 0x9e */, 0 /* 0x9f */, - 0 /* 0xa0 */, 0 /* 0xa1 */, 0 /* 0xa2 */, 0 /* 0xa3 */, 0 /* 0xa4 */, - 0 /* 0xa5 */, 0 /* 0xa6 */, 0 /* 0xa7 */, 0 /* 0xa8 */, 0 /* 0xa9 */, - 0 /* 0xaa */, 0 /* 0xab */, 0 /* 0xac */, 0 /* 0xad */, 0 /* 0xae */, - 0 /* 0xaf */, 0 /* 0xb0 */, 0 /* 0xb1 */, 0 /* 0xb2 */, 0 /* 0xb3 */, - 0 /* 0xb4 */, 0 /* 0xb5 */, 0 /* 0xb6 */, 0 /* 0xb7 */, 0 /* 0xb8 */, - 0 /* 0xb9 */, 0 /* 0xba */, 0 /* 0xbb */, 0 /* 0xbc */, 0 /* 0xbd */, - 0 /* 0xbe */, 0 /* 0xbf */, 0 /* 0xc0 */, 0 /* 0xc1 */, 0 /* 0xc2 */, - 0 /* 0xc3 */, 0 /* 0xc4 */, 0 /* 0xc5 */, 0 /* 0xc6 */, 0 /* 0xc7 */, - 0 /* 0xc8 */, 0 /* 0xc9 */, 0 /* 0xca */, 0 /* 0xcb */, 0 /* 0xcc */, - 0 /* 0xcd */, 0 /* 0xce */, 0 /* 0xcf */, 0 /* 0xd0 */, 0 /* 0xd1 */, - 0 /* 0xd2 */, 0 /* 0xd3 */, 0 /* 0xd4 */, 0 /* 0xd5 */, 0 /* 0xd6 */, - 0 /* 0xd7 */, 0 /* 0xd8 */, 0 /* 0xd9 */, 0 /* 0xda */, 0 /* 0xdb */, - 0 /* 0xdc */, 0 /* 0xdd */, 0 /* 0xde */, 0 /* 0xdf */, 0 /* 0xe0 */, - 0 /* 0xe1 */, 0 /* 0xe2 */, 0 /* 0xe3 */, 0 /* 0xe4 */, 0 /* 0xe5 */, - 0 /* 0xe6 */, 0 /* 0xe7 */, 0 /* 0xe8 */, 0 /* 0xe9 */, 0 /* 0xea */, - 0 /* 0xeb */, 0 /* 0xec */, 0 /* 0xed */, 0 /* 0xee */, 0 /* 0xef */, - 0 /* 0xf0 */, 0 /* 0xf1 */, 0 /* 0xf2 */, 0 /* 0xf3 */, 0 /* 0xf4 */, - 0 /* 0xf5 */, 0 /* 0xf6 */, 0 /* 0xf7 */, 0 /* 0xf8 */, 0 /* 0xf9 */, - 0 /* 0xfa */, 0 /* 0xfb */, 0 /* 0xfc */, 0 /* 0xfd */, 0 /* 0xfe */, - 0 /* 0xff */, -}; - -static ssize_t sf_parse_key(const uint8_t *begin, const uint8_t *end) { - const uint8_t *p = begin; - - if ((*p < 'a' || 'z' < *p) && *p != '*') { - return -1; - } - - for (; p != end && SF_KEY_CHARS[*p]; ++p) - ; - - return p - begin; -} - -static ssize_t sf_parse_integer_or_decimal(nghttp2_sf_value *dest, - const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - int sign = 1; - int64_t value = 0; - int type = NGHTTP2_SF_VALUE_TYPE_INTEGER; - size_t len = 0; - size_t fpos = 0; - size_t i; - - if (*p == '-') { - if (++p == end) { - return -1; - } - - sign = -1; - } - - if (*p < '0' || '9' < *p) { - return -1; - } - - for (; p != end; ++p) { - switch (*p) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - value *= 10; - value += *p - '0'; - - if (++len > 15) { - return -1; - } - - break; - case '.': - if (type != NGHTTP2_SF_VALUE_TYPE_INTEGER) { - goto fin; - } - - if (len > 12) { - return -1; - } - fpos = len; - type = NGHTTP2_SF_VALUE_TYPE_DECIMAL; - - break; - default: - goto fin; - }; - } - -fin: - switch (type) { - case NGHTTP2_SF_VALUE_TYPE_INTEGER: - if (dest) { - dest->type = (uint8_t)type; - dest->i = value * sign; - } - - return p - begin; - case NGHTTP2_SF_VALUE_TYPE_DECIMAL: - if (fpos == len || len - fpos > 3) { - return -1; - } - - if (dest) { - dest->type = (uint8_t)type; - dest->d = (double)value; - for (i = len - fpos; i > 0; --i) { - dest->d /= (double)10; - } - dest->d *= sign; - } - - return p - begin; - default: - assert(0); - abort(); - } -} - -/* Generated by genchartbl.py */ -static const int SF_DQUOTE_CHARS[] = { - 0 /* NUL */, 0 /* SOH */, 0 /* STX */, 0 /* ETX */, 0 /* EOT */, - 0 /* ENQ */, 0 /* ACK */, 0 /* BEL */, 0 /* BS */, 0 /* HT */, - 0 /* LF */, 0 /* VT */, 0 /* FF */, 0 /* CR */, 0 /* SO */, - 0 /* SI */, 0 /* DLE */, 0 /* DC1 */, 0 /* DC2 */, 0 /* DC3 */, - 0 /* DC4 */, 0 /* NAK */, 0 /* SYN */, 0 /* ETB */, 0 /* CAN */, - 0 /* EM */, 0 /* SUB */, 0 /* ESC */, 0 /* FS */, 0 /* GS */, - 0 /* RS */, 0 /* US */, 1 /* SPC */, 1 /* ! */, 0 /* " */, - 1 /* # */, 1 /* $ */, 1 /* % */, 1 /* & */, 1 /* ' */, - 1 /* ( */, 1 /* ) */, 1 /* * */, 1 /* + */, 1 /* , */, - 1 /* - */, 1 /* . */, 1 /* / */, 1 /* 0 */, 1 /* 1 */, - 1 /* 2 */, 1 /* 3 */, 1 /* 4 */, 1 /* 5 */, 1 /* 6 */, - 1 /* 7 */, 1 /* 8 */, 1 /* 9 */, 1 /* : */, 1 /* ; */, - 1 /* < */, 1 /* = */, 1 /* > */, 1 /* ? */, 1 /* @ */, - 1 /* A */, 1 /* B */, 1 /* C */, 1 /* D */, 1 /* E */, - 1 /* F */, 1 /* G */, 1 /* H */, 1 /* I */, 1 /* J */, - 1 /* K */, 1 /* L */, 1 /* M */, 1 /* N */, 1 /* O */, - 1 /* P */, 1 /* Q */, 1 /* R */, 1 /* S */, 1 /* T */, - 1 /* U */, 1 /* V */, 1 /* W */, 1 /* X */, 1 /* Y */, - 1 /* Z */, 1 /* [ */, 0 /* \ */, 1 /* ] */, 1 /* ^ */, - 1 /* _ */, 1 /* ` */, 1 /* a */, 1 /* b */, 1 /* c */, - 1 /* d */, 1 /* e */, 1 /* f */, 1 /* g */, 1 /* h */, - 1 /* i */, 1 /* j */, 1 /* k */, 1 /* l */, 1 /* m */, - 1 /* n */, 1 /* o */, 1 /* p */, 1 /* q */, 1 /* r */, - 1 /* s */, 1 /* t */, 1 /* u */, 1 /* v */, 1 /* w */, - 1 /* x */, 1 /* y */, 1 /* z */, 1 /* { */, 1 /* | */, - 1 /* } */, 1 /* ~ */, 0 /* DEL */, 0 /* 0x80 */, 0 /* 0x81 */, - 0 /* 0x82 */, 0 /* 0x83 */, 0 /* 0x84 */, 0 /* 0x85 */, 0 /* 0x86 */, - 0 /* 0x87 */, 0 /* 0x88 */, 0 /* 0x89 */, 0 /* 0x8a */, 0 /* 0x8b */, - 0 /* 0x8c */, 0 /* 0x8d */, 0 /* 0x8e */, 0 /* 0x8f */, 0 /* 0x90 */, - 0 /* 0x91 */, 0 /* 0x92 */, 0 /* 0x93 */, 0 /* 0x94 */, 0 /* 0x95 */, - 0 /* 0x96 */, 0 /* 0x97 */, 0 /* 0x98 */, 0 /* 0x99 */, 0 /* 0x9a */, - 0 /* 0x9b */, 0 /* 0x9c */, 0 /* 0x9d */, 0 /* 0x9e */, 0 /* 0x9f */, - 0 /* 0xa0 */, 0 /* 0xa1 */, 0 /* 0xa2 */, 0 /* 0xa3 */, 0 /* 0xa4 */, - 0 /* 0xa5 */, 0 /* 0xa6 */, 0 /* 0xa7 */, 0 /* 0xa8 */, 0 /* 0xa9 */, - 0 /* 0xaa */, 0 /* 0xab */, 0 /* 0xac */, 0 /* 0xad */, 0 /* 0xae */, - 0 /* 0xaf */, 0 /* 0xb0 */, 0 /* 0xb1 */, 0 /* 0xb2 */, 0 /* 0xb3 */, - 0 /* 0xb4 */, 0 /* 0xb5 */, 0 /* 0xb6 */, 0 /* 0xb7 */, 0 /* 0xb8 */, - 0 /* 0xb9 */, 0 /* 0xba */, 0 /* 0xbb */, 0 /* 0xbc */, 0 /* 0xbd */, - 0 /* 0xbe */, 0 /* 0xbf */, 0 /* 0xc0 */, 0 /* 0xc1 */, 0 /* 0xc2 */, - 0 /* 0xc3 */, 0 /* 0xc4 */, 0 /* 0xc5 */, 0 /* 0xc6 */, 0 /* 0xc7 */, - 0 /* 0xc8 */, 0 /* 0xc9 */, 0 /* 0xca */, 0 /* 0xcb */, 0 /* 0xcc */, - 0 /* 0xcd */, 0 /* 0xce */, 0 /* 0xcf */, 0 /* 0xd0 */, 0 /* 0xd1 */, - 0 /* 0xd2 */, 0 /* 0xd3 */, 0 /* 0xd4 */, 0 /* 0xd5 */, 0 /* 0xd6 */, - 0 /* 0xd7 */, 0 /* 0xd8 */, 0 /* 0xd9 */, 0 /* 0xda */, 0 /* 0xdb */, - 0 /* 0xdc */, 0 /* 0xdd */, 0 /* 0xde */, 0 /* 0xdf */, 0 /* 0xe0 */, - 0 /* 0xe1 */, 0 /* 0xe2 */, 0 /* 0xe3 */, 0 /* 0xe4 */, 0 /* 0xe5 */, - 0 /* 0xe6 */, 0 /* 0xe7 */, 0 /* 0xe8 */, 0 /* 0xe9 */, 0 /* 0xea */, - 0 /* 0xeb */, 0 /* 0xec */, 0 /* 0xed */, 0 /* 0xee */, 0 /* 0xef */, - 0 /* 0xf0 */, 0 /* 0xf1 */, 0 /* 0xf2 */, 0 /* 0xf3 */, 0 /* 0xf4 */, - 0 /* 0xf5 */, 0 /* 0xf6 */, 0 /* 0xf7 */, 0 /* 0xf8 */, 0 /* 0xf9 */, - 0 /* 0xfa */, 0 /* 0xfb */, 0 /* 0xfc */, 0 /* 0xfd */, 0 /* 0xfe */, - 0 /* 0xff */, -}; - -static ssize_t sf_parse_string(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - - if (*p++ != '"') { - return -1; - } - - for (; p != end; ++p) { - switch (*p) { - case '\\': - if (++p == end) { - return -1; - } - - switch (*p) { - case '"': - case '\\': - break; - default: - return -1; - } - - break; - case '"': - if (dest) { - dest->type = NGHTTP2_SF_VALUE_TYPE_STRING; - dest->s.base = begin + 1; - dest->s.len = (size_t)(p - dest->s.base); - } - - ++p; - - return p - begin; - default: - if (!SF_DQUOTE_CHARS[*p]) { - return -1; - } - } - } - - return -1; -} - -/* Generated by genchartbl.py */ -static const int SF_TOKEN_CHARS[] = { - 0 /* NUL */, 0 /* SOH */, 0 /* STX */, 0 /* ETX */, 0 /* EOT */, - 0 /* ENQ */, 0 /* ACK */, 0 /* BEL */, 0 /* BS */, 0 /* HT */, - 0 /* LF */, 0 /* VT */, 0 /* FF */, 0 /* CR */, 0 /* SO */, - 0 /* SI */, 0 /* DLE */, 0 /* DC1 */, 0 /* DC2 */, 0 /* DC3 */, - 0 /* DC4 */, 0 /* NAK */, 0 /* SYN */, 0 /* ETB */, 0 /* CAN */, - 0 /* EM */, 0 /* SUB */, 0 /* ESC */, 0 /* FS */, 0 /* GS */, - 0 /* RS */, 0 /* US */, 0 /* SPC */, 1 /* ! */, 0 /* " */, - 1 /* # */, 1 /* $ */, 1 /* % */, 1 /* & */, 1 /* ' */, - 0 /* ( */, 0 /* ) */, 1 /* * */, 1 /* + */, 0 /* , */, - 1 /* - */, 1 /* . */, 1 /* / */, 1 /* 0 */, 1 /* 1 */, - 1 /* 2 */, 1 /* 3 */, 1 /* 4 */, 1 /* 5 */, 1 /* 6 */, - 1 /* 7 */, 1 /* 8 */, 1 /* 9 */, 1 /* : */, 0 /* ; */, - 0 /* < */, 0 /* = */, 0 /* > */, 0 /* ? */, 0 /* @ */, - 1 /* A */, 1 /* B */, 1 /* C */, 1 /* D */, 1 /* E */, - 1 /* F */, 1 /* G */, 1 /* H */, 1 /* I */, 1 /* J */, - 1 /* K */, 1 /* L */, 1 /* M */, 1 /* N */, 1 /* O */, - 1 /* P */, 1 /* Q */, 1 /* R */, 1 /* S */, 1 /* T */, - 1 /* U */, 1 /* V */, 1 /* W */, 1 /* X */, 1 /* Y */, - 1 /* Z */, 0 /* [ */, 0 /* \ */, 0 /* ] */, 1 /* ^ */, - 1 /* _ */, 1 /* ` */, 1 /* a */, 1 /* b */, 1 /* c */, - 1 /* d */, 1 /* e */, 1 /* f */, 1 /* g */, 1 /* h */, - 1 /* i */, 1 /* j */, 1 /* k */, 1 /* l */, 1 /* m */, - 1 /* n */, 1 /* o */, 1 /* p */, 1 /* q */, 1 /* r */, - 1 /* s */, 1 /* t */, 1 /* u */, 1 /* v */, 1 /* w */, - 1 /* x */, 1 /* y */, 1 /* z */, 0 /* { */, 1 /* | */, - 0 /* } */, 1 /* ~ */, 0 /* DEL */, 0 /* 0x80 */, 0 /* 0x81 */, - 0 /* 0x82 */, 0 /* 0x83 */, 0 /* 0x84 */, 0 /* 0x85 */, 0 /* 0x86 */, - 0 /* 0x87 */, 0 /* 0x88 */, 0 /* 0x89 */, 0 /* 0x8a */, 0 /* 0x8b */, - 0 /* 0x8c */, 0 /* 0x8d */, 0 /* 0x8e */, 0 /* 0x8f */, 0 /* 0x90 */, - 0 /* 0x91 */, 0 /* 0x92 */, 0 /* 0x93 */, 0 /* 0x94 */, 0 /* 0x95 */, - 0 /* 0x96 */, 0 /* 0x97 */, 0 /* 0x98 */, 0 /* 0x99 */, 0 /* 0x9a */, - 0 /* 0x9b */, 0 /* 0x9c */, 0 /* 0x9d */, 0 /* 0x9e */, 0 /* 0x9f */, - 0 /* 0xa0 */, 0 /* 0xa1 */, 0 /* 0xa2 */, 0 /* 0xa3 */, 0 /* 0xa4 */, - 0 /* 0xa5 */, 0 /* 0xa6 */, 0 /* 0xa7 */, 0 /* 0xa8 */, 0 /* 0xa9 */, - 0 /* 0xaa */, 0 /* 0xab */, 0 /* 0xac */, 0 /* 0xad */, 0 /* 0xae */, - 0 /* 0xaf */, 0 /* 0xb0 */, 0 /* 0xb1 */, 0 /* 0xb2 */, 0 /* 0xb3 */, - 0 /* 0xb4 */, 0 /* 0xb5 */, 0 /* 0xb6 */, 0 /* 0xb7 */, 0 /* 0xb8 */, - 0 /* 0xb9 */, 0 /* 0xba */, 0 /* 0xbb */, 0 /* 0xbc */, 0 /* 0xbd */, - 0 /* 0xbe */, 0 /* 0xbf */, 0 /* 0xc0 */, 0 /* 0xc1 */, 0 /* 0xc2 */, - 0 /* 0xc3 */, 0 /* 0xc4 */, 0 /* 0xc5 */, 0 /* 0xc6 */, 0 /* 0xc7 */, - 0 /* 0xc8 */, 0 /* 0xc9 */, 0 /* 0xca */, 0 /* 0xcb */, 0 /* 0xcc */, - 0 /* 0xcd */, 0 /* 0xce */, 0 /* 0xcf */, 0 /* 0xd0 */, 0 /* 0xd1 */, - 0 /* 0xd2 */, 0 /* 0xd3 */, 0 /* 0xd4 */, 0 /* 0xd5 */, 0 /* 0xd6 */, - 0 /* 0xd7 */, 0 /* 0xd8 */, 0 /* 0xd9 */, 0 /* 0xda */, 0 /* 0xdb */, - 0 /* 0xdc */, 0 /* 0xdd */, 0 /* 0xde */, 0 /* 0xdf */, 0 /* 0xe0 */, - 0 /* 0xe1 */, 0 /* 0xe2 */, 0 /* 0xe3 */, 0 /* 0xe4 */, 0 /* 0xe5 */, - 0 /* 0xe6 */, 0 /* 0xe7 */, 0 /* 0xe8 */, 0 /* 0xe9 */, 0 /* 0xea */, - 0 /* 0xeb */, 0 /* 0xec */, 0 /* 0xed */, 0 /* 0xee */, 0 /* 0xef */, - 0 /* 0xf0 */, 0 /* 0xf1 */, 0 /* 0xf2 */, 0 /* 0xf3 */, 0 /* 0xf4 */, - 0 /* 0xf5 */, 0 /* 0xf6 */, 0 /* 0xf7 */, 0 /* 0xf8 */, 0 /* 0xf9 */, - 0 /* 0xfa */, 0 /* 0xfb */, 0 /* 0xfc */, 0 /* 0xfd */, 0 /* 0xfe */, - 0 /* 0xff */, -}; - -static ssize_t sf_parse_token(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - - if ((*p < 'A' || 'Z' < *p) && (*p < 'a' || 'z' < *p) && *p != '*') { - return -1; - } - - for (; p != end && SF_TOKEN_CHARS[*p]; ++p) - ; - - if (dest) { - dest->type = NGHTTP2_SF_VALUE_TYPE_TOKEN; - dest->s.base = begin; - dest->s.len = (size_t)(p - begin); - } - - return p - begin; -} - -/* Generated by genchartbl.py */ -static const int SF_BYTESEQ_CHARS[] = { - 0 /* NUL */, 0 /* SOH */, 0 /* STX */, 0 /* ETX */, 0 /* EOT */, - 0 /* ENQ */, 0 /* ACK */, 0 /* BEL */, 0 /* BS */, 0 /* HT */, - 0 /* LF */, 0 /* VT */, 0 /* FF */, 0 /* CR */, 0 /* SO */, - 0 /* SI */, 0 /* DLE */, 0 /* DC1 */, 0 /* DC2 */, 0 /* DC3 */, - 0 /* DC4 */, 0 /* NAK */, 0 /* SYN */, 0 /* ETB */, 0 /* CAN */, - 0 /* EM */, 0 /* SUB */, 0 /* ESC */, 0 /* FS */, 0 /* GS */, - 0 /* RS */, 0 /* US */, 0 /* SPC */, 0 /* ! */, 0 /* " */, - 0 /* # */, 0 /* $ */, 0 /* % */, 0 /* & */, 0 /* ' */, - 0 /* ( */, 0 /* ) */, 0 /* * */, 1 /* + */, 0 /* , */, - 0 /* - */, 0 /* . */, 1 /* / */, 1 /* 0 */, 1 /* 1 */, - 1 /* 2 */, 1 /* 3 */, 1 /* 4 */, 1 /* 5 */, 1 /* 6 */, - 1 /* 7 */, 1 /* 8 */, 1 /* 9 */, 0 /* : */, 0 /* ; */, - 0 /* < */, 1 /* = */, 0 /* > */, 0 /* ? */, 0 /* @ */, - 1 /* A */, 1 /* B */, 1 /* C */, 1 /* D */, 1 /* E */, - 1 /* F */, 1 /* G */, 1 /* H */, 1 /* I */, 1 /* J */, - 1 /* K */, 1 /* L */, 1 /* M */, 1 /* N */, 1 /* O */, - 1 /* P */, 1 /* Q */, 1 /* R */, 1 /* S */, 1 /* T */, - 1 /* U */, 1 /* V */, 1 /* W */, 1 /* X */, 1 /* Y */, - 1 /* Z */, 0 /* [ */, 0 /* \ */, 0 /* ] */, 0 /* ^ */, - 0 /* _ */, 0 /* ` */, 1 /* a */, 1 /* b */, 1 /* c */, - 1 /* d */, 1 /* e */, 1 /* f */, 1 /* g */, 1 /* h */, - 1 /* i */, 1 /* j */, 1 /* k */, 1 /* l */, 1 /* m */, - 1 /* n */, 1 /* o */, 1 /* p */, 1 /* q */, 1 /* r */, - 1 /* s */, 1 /* t */, 1 /* u */, 1 /* v */, 1 /* w */, - 1 /* x */, 1 /* y */, 1 /* z */, 0 /* { */, 0 /* | */, - 0 /* } */, 0 /* ~ */, 0 /* DEL */, 0 /* 0x80 */, 0 /* 0x81 */, - 0 /* 0x82 */, 0 /* 0x83 */, 0 /* 0x84 */, 0 /* 0x85 */, 0 /* 0x86 */, - 0 /* 0x87 */, 0 /* 0x88 */, 0 /* 0x89 */, 0 /* 0x8a */, 0 /* 0x8b */, - 0 /* 0x8c */, 0 /* 0x8d */, 0 /* 0x8e */, 0 /* 0x8f */, 0 /* 0x90 */, - 0 /* 0x91 */, 0 /* 0x92 */, 0 /* 0x93 */, 0 /* 0x94 */, 0 /* 0x95 */, - 0 /* 0x96 */, 0 /* 0x97 */, 0 /* 0x98 */, 0 /* 0x99 */, 0 /* 0x9a */, - 0 /* 0x9b */, 0 /* 0x9c */, 0 /* 0x9d */, 0 /* 0x9e */, 0 /* 0x9f */, - 0 /* 0xa0 */, 0 /* 0xa1 */, 0 /* 0xa2 */, 0 /* 0xa3 */, 0 /* 0xa4 */, - 0 /* 0xa5 */, 0 /* 0xa6 */, 0 /* 0xa7 */, 0 /* 0xa8 */, 0 /* 0xa9 */, - 0 /* 0xaa */, 0 /* 0xab */, 0 /* 0xac */, 0 /* 0xad */, 0 /* 0xae */, - 0 /* 0xaf */, 0 /* 0xb0 */, 0 /* 0xb1 */, 0 /* 0xb2 */, 0 /* 0xb3 */, - 0 /* 0xb4 */, 0 /* 0xb5 */, 0 /* 0xb6 */, 0 /* 0xb7 */, 0 /* 0xb8 */, - 0 /* 0xb9 */, 0 /* 0xba */, 0 /* 0xbb */, 0 /* 0xbc */, 0 /* 0xbd */, - 0 /* 0xbe */, 0 /* 0xbf */, 0 /* 0xc0 */, 0 /* 0xc1 */, 0 /* 0xc2 */, - 0 /* 0xc3 */, 0 /* 0xc4 */, 0 /* 0xc5 */, 0 /* 0xc6 */, 0 /* 0xc7 */, - 0 /* 0xc8 */, 0 /* 0xc9 */, 0 /* 0xca */, 0 /* 0xcb */, 0 /* 0xcc */, - 0 /* 0xcd */, 0 /* 0xce */, 0 /* 0xcf */, 0 /* 0xd0 */, 0 /* 0xd1 */, - 0 /* 0xd2 */, 0 /* 0xd3 */, 0 /* 0xd4 */, 0 /* 0xd5 */, 0 /* 0xd6 */, - 0 /* 0xd7 */, 0 /* 0xd8 */, 0 /* 0xd9 */, 0 /* 0xda */, 0 /* 0xdb */, - 0 /* 0xdc */, 0 /* 0xdd */, 0 /* 0xde */, 0 /* 0xdf */, 0 /* 0xe0 */, - 0 /* 0xe1 */, 0 /* 0xe2 */, 0 /* 0xe3 */, 0 /* 0xe4 */, 0 /* 0xe5 */, - 0 /* 0xe6 */, 0 /* 0xe7 */, 0 /* 0xe8 */, 0 /* 0xe9 */, 0 /* 0xea */, - 0 /* 0xeb */, 0 /* 0xec */, 0 /* 0xed */, 0 /* 0xee */, 0 /* 0xef */, - 0 /* 0xf0 */, 0 /* 0xf1 */, 0 /* 0xf2 */, 0 /* 0xf3 */, 0 /* 0xf4 */, - 0 /* 0xf5 */, 0 /* 0xf6 */, 0 /* 0xf7 */, 0 /* 0xf8 */, 0 /* 0xf9 */, - 0 /* 0xfa */, 0 /* 0xfb */, 0 /* 0xfc */, 0 /* 0xfd */, 0 /* 0xfe */, - 0 /* 0xff */, -}; - -static ssize_t sf_parse_byteseq(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - - if (*p++ != ':') { - return -1; - } - - for (; p != end; ++p) { - switch (*p) { - case ':': - if (dest) { - dest->type = NGHTTP2_SF_VALUE_TYPE_BYTESEQ; - dest->s.base = begin + 1; - dest->s.len = (size_t)(p - dest->s.base); - } - - ++p; - - return p - begin; - default: - if (!SF_BYTESEQ_CHARS[*p]) { - return -1; - } - } - } - - return -1; -} - -static ssize_t sf_parse_boolean(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - int b; - - if (*p++ != '?') { - return -1; - } - - if (p == end) { - return -1; - } - - switch (*p++) { - case '0': - b = 0; - break; - case '1': - b = 1; - break; - default: - return -1; - } - - if (dest) { - dest->type = NGHTTP2_SF_VALUE_TYPE_BOOLEAN; - dest->b = b; - } - - return p - begin; -} - -static ssize_t sf_parse_bare_item(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - switch (*begin) { - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return sf_parse_integer_or_decimal(dest, begin, end); - case '"': - return sf_parse_string(dest, begin, end); - case '*': - return sf_parse_token(dest, begin, end); - case ':': - return sf_parse_byteseq(dest, begin, end); - case '?': - return sf_parse_boolean(dest, begin, end); - default: - if (('A' <= *begin && *begin <= 'Z') || ('a' <= *begin && *begin <= 'z')) { - return sf_parse_token(dest, begin, end); - } - return -1; - } -} - -#define sf_discard_sp_end_err(BEGIN, END, ERR) \ - for (;; ++(BEGIN)) { \ - if ((BEGIN) == (END)) { \ - return (ERR); \ - } \ - if (*(BEGIN) != ' ') { \ - break; \ - } \ - } - -static ssize_t sf_parse_params(const uint8_t *begin, const uint8_t *end) { - const uint8_t *p = begin; - ssize_t slen; - - for (; p != end && *p == ';';) { - ++p; - - sf_discard_sp_end_err(p, end, -1); - - slen = sf_parse_key(p, end); - if (slen < 0) { - return -1; - } - - p += slen; - - if (p == end || *p != '=') { - /* Boolean true */ - } else if (++p == end) { - return -1; - } else { - slen = sf_parse_bare_item(NULL, p, end); - if (slen < 0) { - return -1; - } - - p += slen; - } - } - - return p - begin; -} - -static ssize_t sf_parse_item(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - ssize_t slen; - - slen = sf_parse_bare_item(dest, p, end); - if (slen < 0) { - return -1; - } - - p += slen; - - slen = sf_parse_params(p, end); - if (slen < 0) { - return -1; - } - - p += slen; - - return p - begin; -} - -ssize_t nghttp2_sf_parse_item(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - return sf_parse_item(dest, begin, end); -} - -static ssize_t sf_parse_inner_list(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end) { - const uint8_t *p = begin; - ssize_t slen; - - if (*p++ != '(') { - return -1; - } - - for (;;) { - sf_discard_sp_end_err(p, end, -1); - - if (*p == ')') { - ++p; - - slen = sf_parse_params(p, end); - if (slen < 0) { - return -1; - } - - p += slen; - - if (dest) { - dest->type = NGHTTP2_SF_VALUE_TYPE_INNER_LIST; - } - - return p - begin; - } - - slen = sf_parse_item(NULL, p, end); - if (slen < 0) { - return -1; - } - - p += slen; - - if (p == end || (*p != ' ' && *p != ')')) { - return -1; - } - } -} - -ssize_t nghttp2_sf_parse_inner_list(nghttp2_sf_value *dest, - const uint8_t *begin, const uint8_t *end) { - return sf_parse_inner_list(dest, begin, end); -} - -static ssize_t sf_parse_item_or_inner_list(nghttp2_sf_value *dest, - const uint8_t *begin, - const uint8_t *end) { - if (*begin == '(') { - return sf_parse_inner_list(dest, begin, end); - } - - return sf_parse_item(dest, begin, end); -} - -#define sf_discard_ows(BEGIN, END) \ - for (;; ++(BEGIN)) { \ - if ((BEGIN) == (END)) { \ - goto fin; \ - } \ - if (*(BEGIN) != ' ' && *(BEGIN) != '\t') { \ - break; \ - } \ - } - -#define sf_discard_ows_end_err(BEGIN, END, ERR) \ - for (;; ++(BEGIN)) { \ - if ((BEGIN) == (END)) { \ - return (ERR); \ - } \ - if (*(BEGIN) != ' ' && *(BEGIN) != '\t') { \ - break; \ - } \ - } - int nghttp2_http_parse_priority(nghttp2_extpri *dest, const uint8_t *value, size_t valuelen) { - const uint8_t *p = value, *end = value + valuelen; - ssize_t slen; - nghttp2_sf_value val; nghttp2_extpri pri = *dest; - const uint8_t *key; - size_t keylen; + sf_parser sfp; + sf_vec key; + sf_value val; + int rv; + + sf_parser_init(&sfp, value, valuelen); - for (; p != end && *p == ' '; ++p) - ; + for (;;) { + rv = sf_parser_dict(&sfp, &key, &val); + if (rv != 0) { + if (rv == SF_ERR_EOF) { + break; + } - for (; p != end;) { - slen = sf_parse_key(p, end); - if (slen < 0) { return NGHTTP2_ERR_INVALID_ARGUMENT; } - key = p; - keylen = (size_t)slen; - - p += slen; - - if (p == end || *p != '=') { - /* Boolean true */ - val.type = NGHTTP2_SF_VALUE_TYPE_BOOLEAN; - val.b = 1; + if (key.len != 1) { + continue; + } - slen = sf_parse_params(p, end); - if (slen < 0) { + switch (key.base[0]) { + case 'i': + if (val.type != SF_TYPE_BOOLEAN) { return NGHTTP2_ERR_INVALID_ARGUMENT; } - } else if (++p == end) { - return NGHTTP2_ERR_INVALID_ARGUMENT; - } else { - slen = sf_parse_item_or_inner_list(&val, p, end); - if (slen < 0) { - return NGHTTP2_ERR_INVALID_ARGUMENT; - } - } - - p += slen; - if (keylen == 1) { - switch (key[0]) { - case 'i': - if (val.type != NGHTTP2_SF_VALUE_TYPE_BOOLEAN) { - return NGHTTP2_ERR_INVALID_ARGUMENT; - } - - pri.inc = val.b; + pri.inc = val.boolean; - break; - case 'u': - if (val.type != NGHTTP2_SF_VALUE_TYPE_INTEGER || - val.i < NGHTTP2_EXTPRI_URGENCY_HIGH || - NGHTTP2_EXTPRI_URGENCY_LOW < val.i) { - return NGHTTP2_ERR_INVALID_ARGUMENT; - } - - pri.urgency = (uint32_t)val.i; - - break; + break; + case 'u': + if (val.type != SF_TYPE_INTEGER || + val.integer < NGHTTP2_EXTPRI_URGENCY_HIGH || + NGHTTP2_EXTPRI_URGENCY_LOW < val.integer) { + return NGHTTP2_ERR_INVALID_ARGUMENT; } - } - sf_discard_ows(p, end); + pri.urgency = (uint32_t)val.integer; - if (*p++ != ',') { - return NGHTTP2_ERR_INVALID_ARGUMENT; + break; } - - sf_discard_ows_end_err(p, end, NGHTTP2_ERR_INVALID_ARGUMENT); } -fin: *dest = pri; return 0; diff --git a/3rdparty/exported/nghttp2/nghttp2_http.h b/3rdparty/exported/nghttp2/nghttp2_http.h index 0c3a78eeefab..d9992fe69083 100644 --- a/3rdparty/exported/nghttp2/nghttp2_http.h +++ b/3rdparty/exported/nghttp2/nghttp2_http.h @@ -94,54 +94,6 @@ int nghttp2_http_on_data_chunk(nghttp2_stream *stream, size_t n); void nghttp2_http_record_request_method(nghttp2_stream *stream, nghttp2_frame *frame); -/* - * RFC 8941 Structured Field Values. - */ -typedef enum nghttp2_sf_value_type { - NGHTTP2_SF_VALUE_TYPE_BOOLEAN, - NGHTTP2_SF_VALUE_TYPE_INTEGER, - NGHTTP2_SF_VALUE_TYPE_DECIMAL, - NGHTTP2_SF_VALUE_TYPE_STRING, - NGHTTP2_SF_VALUE_TYPE_TOKEN, - NGHTTP2_SF_VALUE_TYPE_BYTESEQ, - NGHTTP2_SF_VALUE_TYPE_INNER_LIST, -} nghttp2_sf_value_type; - -/* - * nghttp2_sf_value stores Structured Field Values item. For Inner - * List, only type is set to NGHTTP2_SF_VALUE_TYPE_INNER_LIST. - */ -typedef struct nghttp2_sf_value { - uint8_t type; - union { - int b; - int64_t i; - double d; - struct { - const uint8_t *base; - size_t len; - } s; - }; -} nghttp2_sf_value; - -/* - * nghttp2_sf_parse_item parses the input sequence [|begin|, |end|) - * and stores the parsed an Item in |dest|. It returns the number of - * bytes consumed if it succeeds, or -1. This function is declared - * here for unit tests. - */ -ssize_t nghttp2_sf_parse_item(nghttp2_sf_value *dest, const uint8_t *begin, - const uint8_t *end); - -/* - * nghttp2_sf_parse_inner_list parses the input sequence [|begin|, |end|) - * and stores the parsed an Inner List in |dest|. It returns the number of - * bytes consumed if it succeeds, or -1. This function is declared - * here for unit tests. - */ -ssize_t nghttp2_sf_parse_inner_list(nghttp2_sf_value *dest, - const uint8_t *begin, const uint8_t *end); - int nghttp2_http_parse_priority(nghttp2_extpri *dest, const uint8_t *value, size_t valuelen); diff --git a/3rdparty/exported/nghttp2/nghttp2_map.c b/3rdparty/exported/nghttp2/nghttp2_map.c index e5db168ca2bc..5f63fc2bb87e 100644 --- a/3rdparty/exported/nghttp2/nghttp2_map.c +++ b/3rdparty/exported/nghttp2/nghttp2_map.c @@ -31,21 +31,14 @@ #include "nghttp2_helper.h" -#define NGHTTP2_INITIAL_TABLE_LENBITS 8 +#define NGHTTP2_INITIAL_TABLE_LENBITS 4 -int nghttp2_map_init(nghttp2_map *map, nghttp2_mem *mem) { +void nghttp2_map_init(nghttp2_map *map, nghttp2_mem *mem) { map->mem = mem; - map->tablelen = 1 << NGHTTP2_INITIAL_TABLE_LENBITS; - map->tablelenbits = NGHTTP2_INITIAL_TABLE_LENBITS; - map->table = - nghttp2_mem_calloc(mem, map->tablelen, sizeof(nghttp2_map_bucket)); - if (map->table == NULL) { - return NGHTTP2_ERR_NOMEM; - } - + map->tablelen = 0; + map->tablelenbits = 0; + map->table = NULL; map->size = 0; - - return 0; } void nghttp2_map_free(nghttp2_map *map) { @@ -78,6 +71,10 @@ int nghttp2_map_each(nghttp2_map *map, int (*func)(void *data, void *ptr), uint32_t i; nghttp2_map_bucket *bkt; + if (map->size == 0) { + return 0; + } + for (i = 0; i < map->tablelen; ++i) { bkt = &map->table[i]; @@ -223,9 +220,17 @@ int nghttp2_map_insert(nghttp2_map *map, nghttp2_map_key_type key, void *data) { /* Load factor is 0.75 */ if ((map->size + 1) * 4 > map->tablelen * 3) { - rv = map_resize(map, map->tablelen * 2, map->tablelenbits + 1); - if (rv != 0) { - return rv; + if (map->tablelen) { + rv = map_resize(map, map->tablelen * 2, map->tablelenbits + 1); + if (rv != 0) { + return rv; + } + } else { + rv = map_resize(map, 1 << NGHTTP2_INITIAL_TABLE_LENBITS, + NGHTTP2_INITIAL_TABLE_LENBITS); + if (rv != 0) { + return rv; + } } } @@ -239,11 +244,18 @@ int nghttp2_map_insert(nghttp2_map *map, nghttp2_map_key_type key, void *data) { } void *nghttp2_map_find(nghttp2_map *map, nghttp2_map_key_type key) { - uint32_t h = hash(key); - size_t idx = h2idx(h, map->tablelenbits); + uint32_t h; + size_t idx; nghttp2_map_bucket *bkt; size_t d = 0; + if (map->size == 0) { + return NULL; + } + + h = hash(key); + idx = h2idx(h, map->tablelenbits); + for (;;) { bkt = &map->table[idx]; @@ -262,11 +274,18 @@ void *nghttp2_map_find(nghttp2_map *map, nghttp2_map_key_type key) { } int nghttp2_map_remove(nghttp2_map *map, nghttp2_map_key_type key) { - uint32_t h = hash(key); - size_t idx = h2idx(h, map->tablelenbits), didx; + uint32_t h; + size_t idx, didx; nghttp2_map_bucket *bkt; size_t d = 0; + if (map->size == 0) { + return NGHTTP2_ERR_INVALID_ARGUMENT; + } + + h = hash(key); + idx = h2idx(h, map->tablelenbits); + for (;;) { bkt = &map->table[idx]; @@ -306,6 +325,10 @@ int nghttp2_map_remove(nghttp2_map *map, nghttp2_map_key_type key) { } void nghttp2_map_clear(nghttp2_map *map) { + if (map->tablelen == 0) { + return; + } + memset(map->table, 0, sizeof(*map->table) * map->tablelen); map->size = 0; } diff --git a/3rdparty/exported/nghttp2/nghttp2_map.h b/3rdparty/exported/nghttp2/nghttp2_map.h index 1419a09a35b1..d90245aab74c 100644 --- a/3rdparty/exported/nghttp2/nghttp2_map.h +++ b/3rdparty/exported/nghttp2/nghttp2_map.h @@ -54,14 +54,8 @@ typedef struct nghttp2_map { /* * Initializes the map |map|. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGHTTP2_ERR_NOMEM - * Out of memory */ -int nghttp2_map_init(nghttp2_map *map, nghttp2_mem *mem); +void nghttp2_map_init(nghttp2_map *map, nghttp2_mem *mem); /* * Deallocates any resources allocated for |map|. The stored entries diff --git a/3rdparty/exported/nghttp2/nghttp2_session.c b/3rdparty/exported/nghttp2/nghttp2_session.c index 93f3f07cf782..71858a39e07d 100644 --- a/3rdparty/exported/nghttp2/nghttp2_session.c +++ b/3rdparty/exported/nghttp2/nghttp2_session.c @@ -584,10 +584,6 @@ static int session_new(nghttp2_session **session_ptr, if (rv != 0) { goto fail_hd_inflater; } - rv = nghttp2_map_init(&(*session_ptr)->streams, mem); - if (rv != 0) { - goto fail_map; - } nbuffer = ((*session_ptr)->max_send_header_block_length + NGHTTP2_FRAMEBUF_CHUNKLEN - 1) / @@ -605,6 +601,8 @@ static int session_new(nghttp2_session **session_ptr, goto fail_aob_framebuf; } + nghttp2_map_init(&(*session_ptr)->streams, mem); + active_outbound_item_reset(&(*session_ptr)->aob, mem); (*session_ptr)->callbacks = *callbacks; @@ -637,8 +635,6 @@ static int session_new(nghttp2_session **session_ptr, return 0; fail_aob_framebuf: - nghttp2_map_free(&(*session_ptr)->streams); -fail_map: nghttp2_hd_inflate_free(&(*session_ptr)->hd_inflater); fail_hd_inflater: nghttp2_hd_deflate_free(&(*session_ptr)->hd_deflater); @@ -3300,6 +3296,7 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session, if (rv < 0) { int32_t opened_stream_id = 0; uint32_t error_code = NGHTTP2_INTERNAL_ERROR; + int rv2 = 0; DEBUGF("send: frame preparation failed with %s\n", nghttp2_strerror(rv)); @@ -3342,19 +3339,18 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session, } if (opened_stream_id) { /* careful not to override rv */ - int rv2; rv2 = nghttp2_session_close_stream(session, opened_stream_id, error_code); - - if (nghttp2_is_fatal(rv2)) { - return rv2; - } } nghttp2_outbound_item_free(item, mem); nghttp2_mem_free(mem, item); active_outbound_item_reset(aob, mem); + if (nghttp2_is_fatal(rv2)) { + return rv2; + } + if (rv == NGHTTP2_ERR_HEADER_COMP) { /* If header compression error occurred, should terminiate connection. */ @@ -5931,7 +5927,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, in += readlen; if (nghttp2_buf_mark_avail(&iframe->sbuf)) { - return in - first; + return (ssize_t)(in - first); } if (iframe->sbuf.pos[3] != NGHTTP2_SETTINGS || @@ -5968,7 +5964,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, in += readlen; if (nghttp2_buf_mark_avail(&iframe->sbuf)) { - return in - first; + return (ssize_t)(in - first); } nghttp2_frame_unpack_frame_hd(&iframe->frame.hd, iframe->sbuf.pos); @@ -6468,7 +6464,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, iframe->payloadleft, nghttp2_buf_mark_avail(&iframe->sbuf)); if (nghttp2_buf_mark_avail(&iframe->sbuf)) { - return in - first; + return (ssize_t)(in - first); } switch (iframe->frame.hd.type) { @@ -6772,7 +6768,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, in += hd_proclen; iframe->payloadleft -= hd_proclen; - return in - first; + return (ssize_t)(in - first); } if (rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { @@ -6963,7 +6959,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, in += readlen; if (nghttp2_buf_mark_avail(&iframe->sbuf)) { - return in - first; + return (ssize_t)(in - first); } nghttp2_frame_unpack_frame_hd(&cont_hd, iframe->sbuf.pos); @@ -7021,7 +7017,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, iframe->payloadleft, nghttp2_buf_mark_avail(&iframe->sbuf)); if (nghttp2_buf_mark_avail(&iframe->sbuf)) { - return in - first; + return (ssize_t)(in - first); } /* Pad Length field is subject to flow control */ @@ -7171,7 +7167,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, session, iframe->frame.hd.flags, iframe->frame.hd.stream_id, in - readlen, (size_t)data_readlen, session->user_data); if (rv == NGHTTP2_ERR_PAUSE) { - return in - first; + return (ssize_t)(in - first); } if (nghttp2_is_fatal(rv)) { @@ -7351,7 +7347,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, assert(in == last); - return in - first; + return (ssize_t)(in - first); } int nghttp2_session_recv(nghttp2_session *session) { diff --git a/3rdparty/exported/nghttp2/sfparse.c b/3rdparty/exported/nghttp2/sfparse.c new file mode 100644 index 000000000000..efa2850c9d66 --- /dev/null +++ b/3rdparty/exported/nghttp2/sfparse.c @@ -0,0 +1,1146 @@ +/* + * sfparse + * + * Copyright (c) 2023 sfparse contributors + * Copyright (c) 2019 nghttp3 contributors + * Copyright (c) 2015 nghttp2 contributors + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "sfparse.h" + +#include +#include +#include + +#define SF_STATE_DICT 0x08u +#define SF_STATE_LIST 0x10u +#define SF_STATE_ITEM 0x18u + +#define SF_STATE_INNER_LIST 0x04u + +#define SF_STATE_BEFORE 0x00u +#define SF_STATE_BEFORE_PARAMS 0x01u +#define SF_STATE_PARAMS 0x02u +#define SF_STATE_AFTER 0x03u + +#define SF_STATE_OP_MASK 0x03u + +#define SF_SET_STATE_AFTER(NAME) (SF_STATE_##NAME | SF_STATE_AFTER) +#define SF_SET_STATE_BEFORE_PARAMS(NAME) \ + (SF_STATE_##NAME | SF_STATE_BEFORE_PARAMS) +#define SF_SET_STATE_INNER_LIST_BEFORE(NAME) \ + (SF_STATE_##NAME | SF_STATE_INNER_LIST | SF_STATE_BEFORE) + +#define SF_STATE_DICT_AFTER SF_SET_STATE_AFTER(DICT) +#define SF_STATE_DICT_BEFORE_PARAMS SF_SET_STATE_BEFORE_PARAMS(DICT) +#define SF_STATE_DICT_INNER_LIST_BEFORE SF_SET_STATE_INNER_LIST_BEFORE(DICT) + +#define SF_STATE_LIST_AFTER SF_SET_STATE_AFTER(LIST) +#define SF_STATE_LIST_BEFORE_PARAMS SF_SET_STATE_BEFORE_PARAMS(LIST) +#define SF_STATE_LIST_INNER_LIST_BEFORE SF_SET_STATE_INNER_LIST_BEFORE(LIST) + +#define SF_STATE_ITEM_AFTER SF_SET_STATE_AFTER(ITEM) +#define SF_STATE_ITEM_BEFORE_PARAMS SF_SET_STATE_BEFORE_PARAMS(ITEM) +#define SF_STATE_ITEM_INNER_LIST_BEFORE SF_SET_STATE_INNER_LIST_BEFORE(ITEM) + +#define SF_STATE_INITIAL 0x00u + +#define DIGIT_CASES \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9' + +#define LCALPHA_CASES \ + case 'a': \ + case 'b': \ + case 'c': \ + case 'd': \ + case 'e': \ + case 'f': \ + case 'g': \ + case 'h': \ + case 'i': \ + case 'j': \ + case 'k': \ + case 'l': \ + case 'm': \ + case 'n': \ + case 'o': \ + case 'p': \ + case 'q': \ + case 'r': \ + case 's': \ + case 't': \ + case 'u': \ + case 'v': \ + case 'w': \ + case 'x': \ + case 'y': \ + case 'z' + +#define UCALPHA_CASES \ + case 'A': \ + case 'B': \ + case 'C': \ + case 'D': \ + case 'E': \ + case 'F': \ + case 'G': \ + case 'H': \ + case 'I': \ + case 'J': \ + case 'K': \ + case 'L': \ + case 'M': \ + case 'N': \ + case 'O': \ + case 'P': \ + case 'Q': \ + case 'R': \ + case 'S': \ + case 'T': \ + case 'U': \ + case 'V': \ + case 'W': \ + case 'X': \ + case 'Y': \ + case 'Z' + +#define ALPHA_CASES \ + UCALPHA_CASES: \ + LCALPHA_CASES + +#define X20_21_CASES \ + case ' ': \ + case '!' + +#define X23_5B_CASES \ + case '#': \ + case '$': \ + case '%': \ + case '&': \ + case '\'': \ + case '(': \ + case ')': \ + case '*': \ + case '+': \ + case ',': \ + case '-': \ + case '.': \ + case '/': \ + DIGIT_CASES: \ + case ':': \ + case ';': \ + case '<': \ + case '=': \ + case '>': \ + case '?': \ + case '@': \ + UCALPHA_CASES: \ + case '[' + +#define X5D_7E_CASES \ + case ']': \ + case '^': \ + case '_': \ + case '`': \ + LCALPHA_CASES: \ + case '{': \ + case '|': \ + case '}': \ + case '~' + +static int is_ws(uint8_t c) { + switch (c) { + case ' ': + case '\t': + return 1; + default: + return 0; + } +} + +static int parser_eof(sf_parser *sfp) { return sfp->pos == sfp->end; } + +static void parser_discard_ows(sf_parser *sfp) { + for (; !parser_eof(sfp) && is_ws(*sfp->pos); ++sfp->pos) + ; +} + +static void parser_discard_sp(sf_parser *sfp) { + for (; !parser_eof(sfp) && *sfp->pos == ' '; ++sfp->pos) + ; +} + +static void parser_set_op_state(sf_parser *sfp, uint32_t op) { + sfp->state &= ~SF_STATE_OP_MASK; + sfp->state |= op; +} + +static void parser_unset_inner_list_state(sf_parser *sfp) { + sfp->state &= ~SF_STATE_INNER_LIST; +} + +static int parser_key(sf_parser *sfp, sf_vec *dest) { + const uint8_t *base; + + switch (*sfp->pos) { + case '*': + LCALPHA_CASES: + break; + default: + return SF_ERR_PARSE_ERROR; + } + + base = sfp->pos++; + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + case '_': + case '-': + case '.': + case '*': + DIGIT_CASES: + LCALPHA_CASES: + continue; + } + + break; + } + + if (dest) { + dest->base = (uint8_t *)base; + dest->len = (size_t)(sfp->pos - dest->base); + } + + return 0; +} + +static int parser_number(sf_parser *sfp, sf_value *dest) { + int sign = 1; + int64_t value = 0; + size_t len = 0; + size_t fpos = 0; + + if (*sfp->pos == '-') { + ++sfp->pos; + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + sign = -1; + } + + assert(!parser_eof(sfp)); + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + DIGIT_CASES: + if (++len > 15) { + return SF_ERR_PARSE_ERROR; + } + + value *= 10; + value += *sfp->pos - '0'; + + continue; + } + + break; + } + + if (len == 0) { + return SF_ERR_PARSE_ERROR; + } + + if (parser_eof(sfp) || *sfp->pos != '.') { + if (dest) { + dest->type = SF_TYPE_INTEGER; + dest->flags = SF_VALUE_FLAG_NONE; + dest->integer = value * sign; + } + + return 0; + } + + /* decimal */ + + if (len > 12) { + return SF_ERR_PARSE_ERROR; + } + + fpos = len; + + ++sfp->pos; + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + DIGIT_CASES: + if (++len > 15) { + return SF_ERR_PARSE_ERROR; + } + + value *= 10; + value += *sfp->pos - '0'; + + continue; + } + + break; + } + + if (fpos == len || len - fpos > 3) { + return SF_ERR_PARSE_ERROR; + } + + if (dest) { + dest->type = SF_TYPE_DECIMAL; + dest->flags = SF_VALUE_FLAG_NONE; + dest->decimal.numer = value * sign; + + switch (len - fpos) { + case 1: + dest->decimal.denom = 10; + + break; + case 2: + dest->decimal.denom = 100; + + break; + case 3: + dest->decimal.denom = 1000; + + break; + } + } + + return 0; +} + +static int parser_date(sf_parser *sfp, sf_value *dest) { + int rv; + sf_value val; + + /* The first byte has already been validated by the caller. */ + assert('@' == *sfp->pos); + + ++sfp->pos; + + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + rv = parser_number(sfp, &val); + if (rv != 0) { + return rv; + } + + if (val.type != SF_TYPE_INTEGER) { + return SF_ERR_PARSE_ERROR; + } + + if (dest) { + *dest = val; + dest->type = SF_TYPE_DATE; + } + + return 0; +} + +static int parser_string(sf_parser *sfp, sf_value *dest) { + const uint8_t *base; + uint32_t flags = SF_VALUE_FLAG_NONE; + + /* The first byte has already been validated by the caller. */ + assert('"' == *sfp->pos); + + base = ++sfp->pos; + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + X20_21_CASES: + X23_5B_CASES: + X5D_7E_CASES: + break; + case '\\': + ++sfp->pos; + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + switch (*sfp->pos) { + case '"': + case '\\': + flags = SF_VALUE_FLAG_ESCAPED_STRING; + + break; + default: + return SF_ERR_PARSE_ERROR; + } + + break; + case '"': + if (dest) { + dest->type = SF_TYPE_STRING; + dest->flags = flags; + dest->vec.len = (size_t)(sfp->pos - base); + dest->vec.base = dest->vec.len == 0 ? NULL : (uint8_t *)base; + } + + ++sfp->pos; + + return 0; + default: + return SF_ERR_PARSE_ERROR; + } + } + + return SF_ERR_PARSE_ERROR; +} + +static int parser_token(sf_parser *sfp, sf_value *dest) { + const uint8_t *base; + + /* The first byte has already been validated by the caller. */ + base = sfp->pos++; + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + case '!': + case '#': + case '$': + case '%': + case '&': + case '\'': + case '*': + case '+': + case '-': + case '.': + case '^': + case '_': + case '`': + case '|': + case '~': + case ':': + case '/': + DIGIT_CASES: + ALPHA_CASES: + continue; + } + + break; + } + + if (dest) { + dest->type = SF_TYPE_TOKEN; + dest->flags = SF_VALUE_FLAG_NONE; + dest->vec.base = (uint8_t *)base; + dest->vec.len = (size_t)(sfp->pos - base); + } + + return 0; +} + +static int parser_byteseq(sf_parser *sfp, sf_value *dest) { + const uint8_t *base; + + /* The first byte has already been validated by the caller. */ + assert(':' == *sfp->pos); + + base = ++sfp->pos; + + for (; !parser_eof(sfp); ++sfp->pos) { + switch (*sfp->pos) { + case '+': + case '/': + DIGIT_CASES: + ALPHA_CASES: + continue; + case '=': + switch ((sfp->pos - base) & 0x3) { + case 0: + case 1: + return SF_ERR_PARSE_ERROR; + case 2: + switch (*(sfp->pos - 1)) { + case 'A': + case 'Q': + case 'g': + case 'w': + break; + default: + return SF_ERR_PARSE_ERROR; + } + + ++sfp->pos; + + if (parser_eof(sfp) || *sfp->pos != '=') { + return SF_ERR_PARSE_ERROR; + } + + break; + case 3: + switch (*(sfp->pos - 1)) { + case 'A': + case 'E': + case 'I': + case 'M': + case 'Q': + case 'U': + case 'Y': + case 'c': + case 'g': + case 'k': + case 'o': + case 's': + case 'w': + case '0': + case '4': + case '8': + break; + default: + return SF_ERR_PARSE_ERROR; + } + + break; + } + + ++sfp->pos; + + if (parser_eof(sfp) || *sfp->pos != ':') { + return SF_ERR_PARSE_ERROR; + } + + goto fin; + case ':': + if ((sfp->pos - base) & 0x3) { + return SF_ERR_PARSE_ERROR; + } + + goto fin; + default: + return SF_ERR_PARSE_ERROR; + } + } + + return SF_ERR_PARSE_ERROR; + +fin: + if (dest) { + dest->type = SF_TYPE_BYTESEQ; + dest->flags = SF_VALUE_FLAG_NONE; + dest->vec.len = (size_t)(sfp->pos - base); + dest->vec.base = dest->vec.len == 0 ? NULL : (uint8_t *)base; + } + + ++sfp->pos; + + return 0; +} + +static int parser_boolean(sf_parser *sfp, sf_value *dest) { + int b; + + /* The first byte has already been validated by the caller. */ + assert('?' == *sfp->pos); + + ++sfp->pos; + + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + switch (*sfp->pos) { + case '0': + b = 0; + + break; + case '1': + b = 1; + + break; + default: + return SF_ERR_PARSE_ERROR; + } + + ++sfp->pos; + + if (dest) { + dest->type = SF_TYPE_BOOLEAN; + dest->flags = SF_VALUE_FLAG_NONE; + dest->boolean = b; + } + + return 0; +} + +static int parser_bare_item(sf_parser *sfp, sf_value *dest) { + switch (*sfp->pos) { + case '"': + return parser_string(sfp, dest); + case '-': + DIGIT_CASES: + return parser_number(sfp, dest); + case '@': + return parser_date(sfp, dest); + case ':': + return parser_byteseq(sfp, dest); + case '?': + return parser_boolean(sfp, dest); + case '*': + ALPHA_CASES: + return parser_token(sfp, dest); + default: + return SF_ERR_PARSE_ERROR; + } +} + +static int parser_skip_inner_list(sf_parser *sfp); + +int sf_parser_param(sf_parser *sfp, sf_vec *dest_key, sf_value *dest_value) { + int rv; + + switch (sfp->state & SF_STATE_OP_MASK) { + case SF_STATE_BEFORE: + rv = parser_skip_inner_list(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_BEFORE_PARAMS: + parser_set_op_state(sfp, SF_STATE_PARAMS); + + break; + case SF_STATE_PARAMS: + break; + default: + assert(0); + abort(); + } + + if (parser_eof(sfp) || *sfp->pos != ';') { + parser_set_op_state(sfp, SF_STATE_AFTER); + + return SF_ERR_EOF; + } + + ++sfp->pos; + + parser_discard_sp(sfp); + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + rv = parser_key(sfp, dest_key); + if (rv != 0) { + return rv; + } + + if (parser_eof(sfp) || *sfp->pos != '=') { + if (dest_value) { + dest_value->type = SF_TYPE_BOOLEAN; + dest_value->flags = SF_VALUE_FLAG_NONE; + dest_value->boolean = 1; + } + + return 0; + } + + ++sfp->pos; + + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + return parser_bare_item(sfp, dest_value); +} + +static int parser_skip_params(sf_parser *sfp) { + int rv; + + for (;;) { + rv = sf_parser_param(sfp, NULL, NULL); + switch (rv) { + case 0: + break; + case SF_ERR_EOF: + return 0; + case SF_ERR_PARSE_ERROR: + return rv; + default: + assert(0); + abort(); + } + } +} + +int sf_parser_inner_list(sf_parser *sfp, sf_value *dest) { + int rv; + + switch (sfp->state & SF_STATE_OP_MASK) { + case SF_STATE_BEFORE: + parser_discard_sp(sfp); + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + break; + case SF_STATE_BEFORE_PARAMS: + rv = parser_skip_params(sfp); + if (rv != 0) { + return rv; + } + + /* Technically, we are entering SF_STATE_AFTER, but we will set + another state without reading the state. */ + /* parser_set_op_state(sfp, SF_STATE_AFTER); */ + + /* fall through */ + case SF_STATE_AFTER: + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + switch (*sfp->pos) { + case ' ': + parser_discard_sp(sfp); + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + break; + case ')': + break; + default: + return SF_ERR_PARSE_ERROR; + } + + break; + default: + assert(0); + abort(); + } + + if (*sfp->pos == ')') { + ++sfp->pos; + + parser_unset_inner_list_state(sfp); + parser_set_op_state(sfp, SF_STATE_BEFORE_PARAMS); + + return SF_ERR_EOF; + } + + rv = parser_bare_item(sfp, dest); + if (rv != 0) { + return rv; + } + + parser_set_op_state(sfp, SF_STATE_BEFORE_PARAMS); + + return 0; +} + +static int parser_skip_inner_list(sf_parser *sfp) { + int rv; + + for (;;) { + rv = sf_parser_inner_list(sfp, NULL); + switch (rv) { + case 0: + break; + case SF_ERR_EOF: + return 0; + case SF_ERR_PARSE_ERROR: + return rv; + default: + assert(0); + abort(); + } + } +} + +static int parser_next_key_or_item(sf_parser *sfp) { + parser_discard_ows(sfp); + + if (parser_eof(sfp)) { + return SF_ERR_EOF; + } + + if (*sfp->pos != ',') { + return SF_ERR_PARSE_ERROR; + } + + ++sfp->pos; + + parser_discard_ows(sfp); + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + return 0; +} + +static int parser_dict_value(sf_parser *sfp, sf_value *dest) { + int rv; + + if (parser_eof(sfp) || *(sfp->pos) != '=') { + /* Boolean true */ + if (dest) { + dest->type = SF_TYPE_BOOLEAN; + dest->flags = SF_VALUE_FLAG_NONE; + dest->boolean = 1; + } + + sfp->state = SF_STATE_DICT_BEFORE_PARAMS; + + return 0; + } + + ++sfp->pos; + + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + if (*sfp->pos == '(') { + if (dest) { + dest->type = SF_TYPE_INNER_LIST; + dest->flags = SF_VALUE_FLAG_NONE; + } + + ++sfp->pos; + + sfp->state = SF_STATE_DICT_INNER_LIST_BEFORE; + + return 0; + } + + rv = parser_bare_item(sfp, dest); + if (rv != 0) { + return rv; + } + + sfp->state = SF_STATE_DICT_BEFORE_PARAMS; + + return 0; +} + +int sf_parser_dict(sf_parser *sfp, sf_vec *dest_key, sf_value *dest_value) { + int rv; + + switch (sfp->state) { + case SF_STATE_DICT_INNER_LIST_BEFORE: + rv = parser_skip_inner_list(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_DICT_BEFORE_PARAMS: + rv = parser_skip_params(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_DICT_AFTER: + rv = parser_next_key_or_item(sfp); + if (rv != 0) { + return rv; + } + + break; + case SF_STATE_INITIAL: + parser_discard_sp(sfp); + + if (parser_eof(sfp)) { + return SF_ERR_EOF; + } + + break; + default: + assert(0); + abort(); + } + + rv = parser_key(sfp, dest_key); + if (rv != 0) { + return rv; + } + + return parser_dict_value(sfp, dest_value); +} + +int sf_parser_list(sf_parser *sfp, sf_value *dest) { + int rv; + + switch (sfp->state) { + case SF_STATE_LIST_INNER_LIST_BEFORE: + rv = parser_skip_inner_list(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_LIST_BEFORE_PARAMS: + rv = parser_skip_params(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_LIST_AFTER: + rv = parser_next_key_or_item(sfp); + if (rv != 0) { + return rv; + } + + break; + case SF_STATE_INITIAL: + parser_discard_sp(sfp); + + if (parser_eof(sfp)) { + return SF_ERR_EOF; + } + + break; + default: + assert(0); + abort(); + } + + if (*sfp->pos == '(') { + if (dest) { + dest->type = SF_TYPE_INNER_LIST; + dest->flags = SF_VALUE_FLAG_NONE; + } + + ++sfp->pos; + + sfp->state = SF_STATE_LIST_INNER_LIST_BEFORE; + + return 0; + } + + rv = parser_bare_item(sfp, dest); + if (rv != 0) { + return rv; + } + + sfp->state = SF_STATE_LIST_BEFORE_PARAMS; + + return 0; +} + +int sf_parser_item(sf_parser *sfp, sf_value *dest) { + int rv; + + switch (sfp->state) { + case SF_STATE_INITIAL: + parser_discard_sp(sfp); + + if (parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + break; + case SF_STATE_ITEM_INNER_LIST_BEFORE: + rv = parser_skip_inner_list(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_ITEM_BEFORE_PARAMS: + rv = parser_skip_params(sfp); + if (rv != 0) { + return rv; + } + + /* fall through */ + case SF_STATE_ITEM_AFTER: + parser_discard_sp(sfp); + + if (!parser_eof(sfp)) { + return SF_ERR_PARSE_ERROR; + } + + return SF_ERR_EOF; + default: + assert(0); + abort(); + } + + if (*sfp->pos == '(') { + if (dest) { + dest->type = SF_TYPE_INNER_LIST; + dest->flags = SF_VALUE_FLAG_NONE; + } + + ++sfp->pos; + + sfp->state = SF_STATE_ITEM_INNER_LIST_BEFORE; + + return 0; + } + + rv = parser_bare_item(sfp, dest); + if (rv != 0) { + return rv; + } + + sfp->state = SF_STATE_ITEM_BEFORE_PARAMS; + + return 0; +} + +void sf_parser_init(sf_parser *sfp, const uint8_t *data, size_t datalen) { + if (datalen == 0) { + sfp->pos = sfp->end = NULL; + } else { + sfp->pos = data; + sfp->end = data + datalen; + } + + sfp->state = SF_STATE_INITIAL; +} + +void sf_unescape(sf_vec *dest, const sf_vec *src) { + const uint8_t *p, *q; + uint8_t *o; + size_t len, slen; + + if (src->len == 0) { + *dest = *src; + + return; + } + + o = dest->base; + p = src->base; + len = src->len; + + for (;;) { + q = memchr(p, '\\', len); + if (q == NULL) { + if (len == src->len) { + *dest = *src; + + return; + } + + memcpy(o, p, len); + o += len; + + break; + } + + slen = (size_t)(q - p); + memcpy(o, p, slen); + o += slen; + + p = q + 1; + *o++ = *p++; + len -= slen + 2; + } + + dest->len = (size_t)(o - dest->base); +} + +void sf_base64decode(sf_vec *dest, const sf_vec *src) { + static const int index_tbl[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1}; + uint8_t *o; + const uint8_t *p, *end; + uint32_t n; + size_t i; + int idx; + + assert((src->len & 0x3) == 0); + + if (src->len == 0) { + *dest = *src; + + return; + } + + o = dest->base; + p = src->base; + end = src->base + src->len; + + for (; p != end;) { + n = 0; + + for (i = 1; i <= 4; ++i, ++p) { + idx = index_tbl[*p]; + + if (idx == -1) { + assert(i > 2); + + if (i == 3) { + assert(*p == '=' && *(p + 1) == '=' && p + 2 == end); + + *o++ = (uint8_t)(n >> 16); + + goto fin; + } + + assert(*p == '=' && p + 1 == end); + + *o++ = (uint8_t)(n >> 16); + *o++ = (n >> 8) & 0xffu; + + goto fin; + } + + n += (uint32_t)(idx << (24 - i * 6)); + } + + *o++ = (uint8_t)(n >> 16); + *o++ = (n >> 8) & 0xffu; + *o++ = n & 0xffu; + } + +fin: + dest->len = (size_t)(o - dest->base); +} diff --git a/3rdparty/exported/nghttp2/sfparse.h b/3rdparty/exported/nghttp2/sfparse.h new file mode 100644 index 000000000000..1474db1429ac --- /dev/null +++ b/3rdparty/exported/nghttp2/sfparse.h @@ -0,0 +1,409 @@ +/* + * sfparse + * + * Copyright (c) 2023 sfparse contributors + * Copyright (c) 2019 nghttp3 contributors + * Copyright (c) 2015 nghttp2 contributors + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SFPARSE_H +#define SFPARSE_H + +/* Define WIN32 when build target is Win32 API (borrowed from + libcurl) */ +#if (defined(_WIN32) || defined(__WIN32__)) && !defined(WIN32) +# define WIN32 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_MSC_VER) && (_MSC_VER < 1800) +/* MSVC < 2013 does not have inttypes.h because it is not C99 + compliant. See compiler macros and version number in + https://sourceforge.net/p/predef/wiki/Compilers/ */ +# include +#else /* !defined(_MSC_VER) || (_MSC_VER >= 1800) */ +# include +#endif /* !defined(_MSC_VER) || (_MSC_VER >= 1800) */ +#include +#include + +/** + * @enum + * + * :type:`sf_type` defines value type. + */ +typedef enum sf_type { + /** + * :enum:`SF_TYPE_BOOLEAN` indicates boolean type. + */ + SF_TYPE_BOOLEAN, + /** + * :enum:`SF_TYPE_INTEGER` indicates integer type. + */ + SF_TYPE_INTEGER, + /** + * :enum:`SF_TYPE_DECIMAL` indicates decimal type. + */ + SF_TYPE_DECIMAL, + /** + * :enum:`SF_TYPE_STRING` indicates string type. + */ + SF_TYPE_STRING, + /** + * :enum:`SF_TYPE_TOKEN` indicates token type. + */ + SF_TYPE_TOKEN, + /** + * :enum:`SF_TYPE_BYTESEQ` indicates byte sequence type. + */ + SF_TYPE_BYTESEQ, + /** + * :enum:`SF_TYPE_INNER_LIST` indicates inner list type. + */ + SF_TYPE_INNER_LIST, + /** + * :enum:`SF_TYPE_DATE` indicates date type. + */ + SF_TYPE_DATE +} sf_type; + +/** + * @macro + * + * :macro:`SF_ERR_PARSE_ERROR` indicates fatal parse error has + * occurred, and it is not possible to continue the processing. + */ +#define SF_ERR_PARSE_ERROR -1 + +/** + * @macro + * + * :macro:`SF_ERR_EOF` indicates that there is nothing left to read. + * The context of this error varies depending on the function that + * returns this error code. + */ +#define SF_ERR_EOF -2 + +/** + * @struct + * + * :type:`sf_vec` stores sequence of bytes. + */ +typedef struct sf_vec { + /** + * :member:`base` points to the beginning of the sequence of bytes. + */ + uint8_t *base; + /** + * :member:`len` is the number of bytes contained in this sequence. + */ + size_t len; +} sf_vec; + +/** + * @macro + * + * :macro:`SF_VALUE_FLAG_NONE` indicates no flag set. + */ +#define SF_VALUE_FLAG_NONE 0x0u + +/** + * @macro + * + * :macro:`SF_VALUE_FLAG_ESCAPED_STRING` indicates that a string + * contains escaped character(s). + */ +#define SF_VALUE_FLAG_ESCAPED_STRING 0x1u + +/** + * @struct + * + * :type:`sf_decimal` contains decimal value. + */ +typedef struct sf_decimal { + /** + * :member:`numer` contains numerator of the decimal value. + */ + int64_t numer; + /** + * :member:`denom` contains denominator of the decimal value. + */ + int64_t denom; +} sf_decimal; + +/** + * @struct + * + * :type:`sf_value` stores a Structured Field item. For Inner List, + * only type is set to :enum:`sf_type.SF_TYPE_INNER_LIST`. In order + * to read the items contained in an inner list, call + * `sf_parser_inner_list`. + */ +typedef struct sf_value { + /** + * :member:`type` is the type of the value contained in this + * particular object. + */ + sf_type type; + /** + * :member:`flags` is bitwise OR of one or more of + * :macro:`SF_VALUE_FLAG_* `. + */ + uint32_t flags; + /** + * @anonunion_start + * + * @sf_value_value + */ + union { + /** + * :member:`boolean` contains boolean value if :member:`type` == + * :enum:`sf_type.SF_TYPE_BOOLEAN`. 1 indicates true, and 0 + * indicates false. + */ + int boolean; + /** + * :member:`integer` contains integer value if :member:`type` is + * either :enum:`sf_type.SF_TYPE_INTEGER` or + * :enum:`sf_type.SF_TYPE_DATE`. + */ + int64_t integer; + /** + * :member:`decimal` contains decimal value if :member:`type` == + * :enum:`sf_type.SF_TYPE_DECIMAL`. + */ + sf_decimal decimal; + /** + * :member:`vec` contains sequence of bytes if :member:`type` is + * either :enum:`sf_type.SF_TYPE_STRING`, + * :enum:`sf_type.SF_TYPE_TOKEN`, or + * :enum:`sf_type.SF_TYPE_BYTESEQ`. + * + * For :enum:`sf_type.SF_TYPE_STRING`, this field contains one or + * more escaped characters if :member:`flags` has + * :macro:`SF_VALUE_FLAG_ESCAPED_STRING` set. To unescape the + * string, use `sf_unescape`. + * + * For :enum:`sf_type.SF_TYPE_BYTESEQ`, this field contains base64 + * encoded string. To decode this byte string, use + * `sf_base64decode`. + * + * If :member:`vec.len ` == 0, :member:`vec.base + * ` is guaranteed to be NULL. + */ + sf_vec vec; + /** + * @anonunion_end + */ + }; +} sf_value; + +/** + * @struct + * + * :type:`sf_parser` is the Structured Field Values parser. Use + * `sf_parser_init` to initialize it. + */ +typedef struct sf_parser { + /* all fields are private */ + const uint8_t *pos; + const uint8_t *end; + uint32_t state; +} sf_parser; + +/** + * @function + * + * `sf_parser_init` initializes |sfp| with the given buffer pointed by + * |data| of length |datalen|. + */ +void sf_parser_init(sf_parser *sfp, const uint8_t *data, size_t datalen); + +/** + * @function + * + * `sf_parser_param` reads a parameter. If this function returns 0, + * it stores parameter key and value in |dest_key| and |dest_value| + * respectively, if they are not NULL. + * + * This function does no effort to find duplicated keys. Same key may + * be reported more than once. + * + * Caller should keep calling this function until it returns negative + * error code. If it returns :macro:`SF_ERR_EOF`, all parameters have + * read, and caller can continue to read rest of the values. If it + * returns :macro:`SF_ERR_PARSE_ERROR`, it encountered fatal error + * while parsing field value. + */ +int sf_parser_param(sf_parser *sfp, sf_vec *dest_key, sf_value *dest_value); + +/** + * @function + * + * `sf_parser_dict` reads the next dictionary key and value pair. If + * this function returns 0, it stores the key and value in |dest_key| + * and |dest_value| respectively, if they are not NULL. + * + * Caller can optionally read parameters attached to the pair by + * calling `sf_parser_param`. + * + * This function does no effort to find duplicated keys. Same key may + * be reported more than once. + * + * Caller should keep calling this function until it returns negative + * error code. If it returns :macro:`SF_ERR_EOF`, all key and value + * pairs have been read, and there is nothing left to read. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :macro:`SF_ERR_EOF` + * All values in the dictionary have read. + * :macro:`SF_ERR_PARSE_ERROR` + * It encountered fatal error while parsing field value. + */ +int sf_parser_dict(sf_parser *sfp, sf_vec *dest_key, sf_value *dest_value); + +/** + * @function + * + * `sf_parser_list` reads the next list item. If this function + * returns 0, it stores the item in |dest| if it is not NULL. + * + * Caller can optionally read parameters attached to the item by + * calling `sf_parser_param`. + * + * Caller should keep calling this function until it returns negative + * error code. If it returns :macro:`SF_ERR_EOF`, all values in the + * list have been read, and there is nothing left to read. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :macro:`SF_ERR_EOF` + * All values in the list have read. + * :macro:`SF_ERR_PARSE_ERROR` + * It encountered fatal error while parsing field value. + */ +int sf_parser_list(sf_parser *sfp, sf_value *dest); + +/** + * @function + * + * `sf_parser_item` reads a single item. If this function returns 0, + * it stores the item in |dest| if it is not NULL. + * + * This function is only used for the field value that consists of a + * single item. + * + * Caller can optionally read parameters attached to the item by + * calling `sf_parser_param`. + * + * Caller should call this function again to make sure that there is + * nothing left to read. If this 2nd function call returns + * :macro:`SF_ERR_EOF`, all data have been processed successfully. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :macro:`SF_ERR_EOF` + * There is nothing left to read. + * :macro:`SF_ERR_PARSE_ERROR` + * It encountered fatal error while parsing field value. + */ +int sf_parser_item(sf_parser *sfp, sf_value *dest); + +/** + * @function + * + * `sf_parser_inner_list` reads the next inner list item. If this + * function returns 0, it stores the item in |dest| if it is not NULL. + * + * Caller can optionally read parameters attached to the item by + * calling `sf_parser_param`. + * + * Caller should keep calling this function until it returns negative + * error code. If it returns :macro:`SF_ERR_EOF`, all values in this + * inner list have been read, and caller can optionally read + * parameters attached to this inner list by calling + * `sf_parser_param`. Then caller can continue to read rest of the + * values. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :macro:`SF_ERR_EOF` + * All values in the inner list have read. + * :macro:`SF_ERR_PARSE_ERROR` + * It encountered fatal error while parsing field value. + */ +int sf_parser_inner_list(sf_parser *sfp, sf_value *dest); + +/** + * @function + * + * `sf_unescape` copies |src| to |dest| by removing escapes (``\``). + * |src| should be the pointer to :member:`sf_value.vec` of type + * :enum:`sf_type.SF_TYPE_STRING` produced by either `sf_parser_dict`, + * `sf_parser_list`, `sf_parser_inner_list`, `sf_parser_item`, or + * `sf_parser_param`, otherwise the behavior is undefined. + * + * :member:`dest->base ` must point to the buffer that + * has sufficient space to store the unescaped string. + * + * If there is no escape character in |src|, |*src| is assigned to + * |*dest|. This includes the case that :member:`src->len + * ` == 0. + * + * This function sets the length of unescaped string to + * :member:`dest->len `. + */ +void sf_unescape(sf_vec *dest, const sf_vec *src); + +/** + * @function + * + * `sf_base64decode` decodes Base64 encoded string |src| and writes + * the result into |dest|. |src| should be the pointer to + * :member:`sf_value.vec` of type :enum:`sf_type.SF_TYPE_BYTESEQ` + * produced by either `sf_parser_dict`, `sf_parser_list`, + * `sf_parser_inner_list`, `sf_parser_item`, or `sf_parser_param`, + * otherwise the behavior is undefined. + * + * :member:`dest->base ` must point to the buffer that + * has sufficient space to store the decoded byte string. + * + * If :member:`src->len ` == 0, |*src| is assigned to + * |*dest|. + * + * This function sets the length of decoded byte string to + * :member:`dest->len `. + */ +void sf_base64decode(sf_vec *dest, const sf_vec *src); + +#ifdef __cplusplus +} +#endif + +#endif /* SFPARSE_H */ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1755bf9748f5..b639662dc544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.8]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.8 - Add `/node/ready/app` and `/node/ready/gov` endpoints for the use of load balancers wanting to check if a node is ready to accept application or governance transactions. See [Operator RPC API](https://microsoft.github.io/CCF/main/operations/operator_rpc_api.html) for details. +- Upgrade `nghttp2` from `1.51.0` to `1.55.1`. ## [4.0.7] diff --git a/cgmanifest.json b/cgmanifest.json index 5a05e8f81c2f..2ad06131f2f0 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -114,7 +114,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/nghttp2/nghttp2", - "commitHash": "00399695cb3ea9715162b6a15c0d7b185ac384d2" + "commitHash": "781057e15626bf403c8739c25166ad123aa17ff3" } } }, diff --git a/cmake/nghttp2.cmake b/cmake/nghttp2.cmake index b1ee5ba93ac8..f353e4110fce 100644 --- a/cmake/nghttp2.cmake +++ b/cmake/nghttp2.cmake @@ -30,6 +30,7 @@ set(NGHTTP2_SRCS ${NGHTTP2_PREFIX}/nghttp2_stream.c ${NGHTTP2_PREFIX}/nghttp2_submit.c ${NGHTTP2_PREFIX}/nghttp2_version.c + ${NGHTTP2_PREFIX}/sfparse.c ) if(COMPILE_TARGET STREQUAL "sgx") From 2833690a25819e82c188331c06d446c1e6c79104 Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:23:37 +0100 Subject: [PATCH 069/135] [release/4.x] Cherry pick: Update `QCBOR` from `1.1` to `1.2` (#5608) (#5612) Co-authored-by: Eddy Ashton --- 3rdparty/exported/QCBOR/CMakeLists.txt | 86 +- 3rdparty/exported/QCBOR/Makefile | 11 +- 3rdparty/exported/QCBOR/README.md | 148 +- 3rdparty/exported/QCBOR/example.c | 2 +- 3rdparty/exported/QCBOR/qcbor/UsefulBuf.h | 27 +- 3rdparty/exported/QCBOR/qcbor/qcbor_common.h | 4 + 3rdparty/exported/QCBOR/qcbor/qcbor_decode.h | 19 + .../QCBOR/qcbor/qcbor_spiffy_decode.h | 37 +- 3rdparty/exported/QCBOR/src/UsefulBuf.c | 28 +- 3rdparty/exported/QCBOR/src/qcbor_decode.c | 695 +++++--- 3rdparty/exported/QCBOR/src/qcbor_encode.c | 9 +- 3rdparty/exported/QCBOR/test/CMakeLists.txt | 71 + .../exported/QCBOR/test/UsefulBuf_Tests.c | 6 + 3rdparty/exported/QCBOR/test/float_tests.c | 10 +- .../QCBOR/test/not_well_formed_cbor.h | 8 +- .../exported/QCBOR/test/qcbor_decode_tests.c | 1407 +++++++++++------ .../exported/QCBOR/test/qcbor_encode_tests.c | 57 +- 3rdparty/exported/QCBOR/test/run_tests.c | 9 +- 3rdparty/exported/QCBOR/ub-example.c | 2 +- CHANGELOG.md | 1 + cgmanifest.json | 2 +- 21 files changed, 1872 insertions(+), 767 deletions(-) create mode 100644 3rdparty/exported/QCBOR/test/CMakeLists.txt diff --git a/3rdparty/exported/QCBOR/CMakeLists.txt b/3rdparty/exported/QCBOR/CMakeLists.txt index f0b67b95531f..486946ce9e65 100644 --- a/3rdparty/exported/QCBOR/CMakeLists.txt +++ b/3rdparty/exported/QCBOR/CMakeLists.txt @@ -1,19 +1,77 @@ -cmake_minimum_required(VERSION 3.10.2) +#------------------------------------------------------------------------------- +# Copyright (c) 2022-2023, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# See BSD-3-Clause license in README.md +#------------------------------------------------------------------------------- + +cmake_minimum_required(VERSION 3.15) + project(qcbor - DESCRIPTION "QCBOR" - LANGUAGES C - VERSION 1.0.0) + DESCRIPTION "QCBOR" + LANGUAGES C + VERSION 1.1.0 +) + +set(BUILD_QCBOR_TEST "OFF" CACHE STRING "Build QCBOR test suite [OFF, LIB, APP]") +set(BUILD_QCBOR_WARN OFF CACHE BOOL "Compile with the warning flags used in the QCBOR release process") +# BUILD_SHARED_LIBS is a built-in global CMake flag +# The shared library is not made by default because of platform +# variability For example MacOS and Linux behave differently and some +# IoT OS's don't support them at all. +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries instead of static ones") + +# Configuration: +# Floating-point support (see README.md for more information) +set(QCBOR_OPT_DISABLE_FLOAT_HW_USE OFF CACHE BOOL "Eliminate dependency on FP hardware and FP instructions") +set(QCBOR_OPT_DISABLE_FLOAT_PREFERRED OFF CACHE BOOL "Eliminate support for half-precision and CBOR preferred serialization") +set(QCBOR_OPT_DISABLE_FLOAT_ALL OFF CACHE BOOL "Eliminate floating-point support completely") + +if (BUILD_QCBOR_WARN) + # Compile options applying to all targets in current directory and below + add_compile_options(-Wall -Wextra -Wpedantic -Wshadow -Wconversion -Wcast-qual) +endif() + +add_library(qcbor) + +target_sources(qcbor + PRIVATE + src/ieee754.c + src/qcbor_decode.c + src/qcbor_encode.c + src/qcbor_err_to_str.c + src/UsefulBuf.c +) + +target_include_directories(qcbor + PUBLIC + inc + PRIVATE + src +) -set(CMAKE_C_FLAGS "-pedantic -Wall -O3 -ffunction-sections") +target_compile_definitions(qcbor + PRIVATE + $<$:QCBOR_DISABLE_FLOAT_HW_USE> + $<$:QCBOR_DISABLE_PREFERRED_FLOAT> + $<$:USEFULBUF_DISABLE_ALL_FLOAT> +) -set(SOURCE - src/ieee754.c - src/qcbor_decode.c - src/qcbor_encode.c - src/qcbor_err_to_str.c - src/UsefulBuf.c -) +if (BUILD_SHARED_LIBS) + target_compile_options(qcbor PRIVATE -Os -fPIC) +endif() -add_library(qcbor ${SOURCE}) +# The math library is needed for floating-point support. +# To avoid need for it #define QCBOR_DISABLE_FLOAT_HW_USE +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Using GCC + target_link_libraries(qcbor + PRIVATE + $<$>:m> + ) +endif() -target_include_directories(qcbor PUBLIC inc) +if (NOT BUILD_QCBOR_TEST STREQUAL "OFF") + add_subdirectory(test) +endif() diff --git a/3rdparty/exported/QCBOR/Makefile b/3rdparty/exported/QCBOR/Makefile index a4fe2b5f65a2..d5d359b7121c 100644 --- a/3rdparty/exported/QCBOR/Makefile +++ b/3rdparty/exported/QCBOR/Makefile @@ -19,8 +19,8 @@ LIBS=-lm # The $(CMD_LINE) variable allows passing in extra flags. This is # used on the stringent build script that is in # https://github.com/laurencelundblade/qdv. This script is used -# before pushes to master (though not yet through and automated build -# process) +# before pushes to master (though not yet through an automated build +# process). See "warn:" below. CFLAGS=$(CMD_LINE) -I inc -I test -Os -fPIC @@ -30,7 +30,7 @@ TEST_OBJ=test/UsefulBuf_Tests.o test/qcbor_encode_tests.o \ test/qcbor_decode_tests.o test/run_tests.o \ test/float_tests.o test/half_to_double_from_rfc7049.o example.o ub-example.o -.PHONY: all so install uninstall clean +.PHONY: all so install uninstall clean warn all: qcbortest libqcbor.a @@ -42,6 +42,11 @@ qcbortest: libqcbor.a $(TEST_OBJ) cmd_line_main.o libqcbor.a: $(QCBOR_OBJ) ar -r $@ $^ +# run "make warn" as a handy way to compile with the warning flags +# used in the QCBOR release process. See CFLAGS above. +warn: + make CMD_LINE="-Wall -Wextra -Wpedantic -Wshadow -Wconversion -Wcast-qual" + # The shared library is not made by default because of platform # variability For example MacOS and Linux behave differently and some diff --git a/3rdparty/exported/QCBOR/README.md b/3rdparty/exported/QCBOR/README.md index d26c56c6f2f7..673e940e1469 100644 --- a/3rdparty/exported/QCBOR/README.md +++ b/3rdparty/exported/QCBOR/README.md @@ -12,7 +12,7 @@ Replaced by RFC 8949. ## QCBOR Characteristics -**Implemented in C with minimal dependency** – Dependent only +**Implemented in C with minimal dependency** – Dependent only on C99, , , and making it highly portable. and are used too, but their use can disabled. No #ifdefs or compiler options need to be set for @@ -34,7 +34,7 @@ Replaced by RFC 8949. encoded CBOR and encode/decode contexts so caller has full control of memory usage making it good for embedded implementations that have to run in small fixed memory. - + **Easy decoding of maps** – The "spiffy decode" functions allow fetching map items directly by label. Detection of duplicate map items is automatically performed. This makes decoding of complex @@ -93,7 +93,7 @@ implementations as seen in the following example. QCBOREncode_AddInt64ToMap(&EncodeCtx, "Horsepower", pE->uHorsePower); QCBOREncode_CloseMap(&EncodeCtx); uErr = QCBOREncode_Finish(&EncodeCtx, &EncodedEngine); - + /* Decode */ QCBORDecode_Init(&DecodeCtx, EncodedEngine, QCBOR_DECODE_MODE_NORMAL); QCBORDecode_EnterMap(&DecodeCtx); @@ -172,7 +172,7 @@ The current version is v1.1, a small feature addition and bug fix release over QCBOR 1.0. Code has been stable for over a year. The last major change was in -fall of 2020. +fall of 2020. QCBOR was originally developed by Qualcomm. It was [open sourced through CAF](https://source.codeaurora.org/quic/QCBOR/QCBOR/) with a @@ -181,7 +181,8 @@ permissive Linux license, September 2018 (thanks Qualcomm!). ## Building There is a simple makefile for the UNIX style command line binary that -compiles everything to run the tests. +compiles everything to run the tests. CMake is also available, please read +the "Building with CMake" section for more information. These eleven files, the contents of the src and inc directories, make up the entire implementation. @@ -213,14 +214,55 @@ RunTests() to invoke them all. While this code will run fine without configuration, there are several C pre processor macros that can be #defined in order to: - * use a more efficient implementation + * use a more efficient implementation * to reduce code size * to improve performance (a little) * remove features to reduce code size -See the comment sections on "Configuration" in inc/UsefulBuf.h and +See the comment sections on "Configuration" in inc/UsefulBuf.h and the pre processor defines that start with QCBOR_DISABLE_XXX. +### Building with CMake + +CMake can also be used to build QCBOR and the test application. Having the root +`CMakeLists.txt` file, QCBOR can be easily integrated with your project's +existing CMake environment. The result of the build process is a static library, +to build a shared library instead you must add the +`-DBUILD_SHARED_LIBS=ON` option at the CMake configuration step. +The tests can be built into a simple command line application to run them as it +was mentioned before; or it can be built as a library to be integrated with your +development environment. +The `BUILD_QCBOR_TEST` CMake option can be used for building the tests, it can +have three values: `APP`, `LIB` or `OFF` (default, test are not included in the +build). + +Building the QCBOR library: + +```bash +cd +# Configuring the project and generating a native build system +cmake -S . -B +# Building the project +cmake --build +``` + +Building and running the QCBOR test app: +```bash +cd +# Configuring the project and generating a native build system +cmake -S . -B -DBUILD_QCBOR_TEST=APP +# Building the project +cmake --build +# Running the test app +./test/qcbortest +``` + +To enable all the compiler warnings that are used in the QCBOR release process +you can use the `BUILD_QCBOR_WARN` option at the CMake configuration step: +```bash +cmake -S . -B -DBUILD_QCBOR_WARN=ON +``` + ### Floating Point Support & Configuration By default, all QCBOR floating-point features are enabled: @@ -238,7 +280,7 @@ used to reduce object code size and dependency. See discussion in qcbor_encode.h for other details. -### #define QCBOR_DISABLE_FLOAT_HW_USE +#### #define QCBOR_DISABLE_FLOAT_HW_USE This removes dependency on: @@ -278,7 +320,7 @@ This saves only a small amount of object code. The primary purpose for defining this is to remove dependency on floating point hardware and libraries. -#### #define QCBOR_DISABLE_PREFERRED_FLOAT +#### #define QCBOR_DISABLE_PREFERRED_FLOAT This eliminates support for half-precision and CBOR preferred serialization by disabling @@ -287,9 +329,9 @@ half-precision floating-point. With this defined, single and double-precision floating-point numbers can still be encoded and decoded. Conversion -of floating-point to and from integers, big numbers and +of floating-point to and from integers, big numbers and such is also supported. Floating-point dates are still -supported. +supported. The primary reason to define this is to save object code. Roughly 900 bytes are saved, though about half of this @@ -311,49 +353,61 @@ it is usually possible to give options to the compiler to avoid all floating-point hardware and instructions, to use software and replacement libraries instead. These are usually bigger and slower, but these options may still be useful -in getting QCBOR to run in some environments in +in getting QCBOR to run in some environments in combination with `QCBOR_DISABLE_FLOAT_HW_USE`. -In particular, `-mfloat-abi=soft`, disables use of +In particular, `-mfloat-abi=soft`, disables use of hardware instructions for the float and double - types in C for some architectures. + types in C for some architectures. + +#### CMake options + +If you are using CMake, it can also be used to configure the floating-point +support. These options can be enabled by adding them to the CMake configuration +step and setting their value to 'ON' (True). The following table shows the +available options and the associated #defines. + | CMake option | #define | + |-----------------------------------|-------------------------------| + | QCBOR_OPT_DISABLE_FLOAT_HW_USE | QCBOR_DISABLE_FLOAT_HW_USE | + | QCBOR_OPT_DISABLE_FLOAT_PREFERRED | QCBOR_DISABLE_PREFERRED_FLOAT | + | QCBOR_OPT_DISABLE_FLOAT_ALL | USEFULBUF_DISABLE_ALL_FLOAT | ## Code Size These are approximate sizes on a 64-bit x86 CPU with the -Os optimization. - | | smallest | largest | + | | smallest | largest | |---------------|----------|---------| - | encode only | 850 | 2100 | - | decode only | 2000 | 13300 | - | combined | 2850 | 15500 | - + | encode only | 900 | 2100 | + | decode only | 1550 | 13300 | + | combined | 2450 | 15500 | + From the table above, one can see that the amount of code pulled in from the QCBOR library varies a lot, ranging from 1KB to 15KB. The main factor is in this is the number of QCBOR functions called and which ones they are. QCBOR is constructed with less internal interdependency so only code necessary for the called functions is brought in. - + Encoding is simpler and smaller. An encode-only implementation may bring in only 1KB of code. - + Encoding of floating-point brings in a little more code as does encoding of tagged types and encoding of bstr wrapping. - + Basic decoding using QCBORDecode_GetNext() brings in 3KB. - + Use of the supplied MemPool by calling QCBORDecode_SetMemPool() to setup to decode indefinite-length strings adds 0.5KB. - + Basic use of spiffy decode to brings in about 3KB. Using more spiffy decode functions, such as those for tagged types bstr wrapping brings in more code. - + Finally, use of all of the integer conversion functions will bring in about 5KB, though you can use the simpler ones like QCBORDecode_GetInt64() without bringing in very much code. - + In addition to using fewer QCBOR functions, the following are some ways to make the code smaller. @@ -366,22 +420,23 @@ These are approximate sizes on a 64-bit x86 CPU with the -Os optimization. If QCBOR is installed as a shared library, then of course only one copy of the code is in memory no matter how many applications use it. - + ### Disabling Features Here's the list of all features that can be disabled to save object code. The amount saved is an approximation. - | #define | Saves | - | ----------------------------------------| ------| - | QCBOR_DISABLE_ENCODE_USAGE_GUARDS | 150 | - | QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS | 400 | - | QCBOR_DISABLE_INDEFINITE_LENGTH_ARRAYS | 200 | - | QCBOR_DISABLE_UNCOMMON_TAGS | 100 | - | QCBOR_DISABLE_EXP_AND_MANTISSA | 400 | - | QCBOR_DISABLE_PREFERRED_FLOAT | 900 | - | QCBOR_DISABLE_FLOAT_HW_USE | 50 | - | USEFULBUF_DISABLE_ALL_FLOAT | 950 | + | #define | Saves | + | ----------------------------------------| ------| + | QCBOR_DISABLE_ENCODE_USAGE_GUARDS | 150 | + | QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS | 400 | + | QCBOR_DISABLE_INDEFINITE_LENGTH_ARRAYS | 200 | + | QCBOR_DISABLE_UNCOMMON_TAGS | 100 | + | QCBOR_DISABLE_EXP_AND_MANTISSA | 400 | + | QCBOR_DISABLE_PREFERRED_FLOAT | 900 | + | QCBOR_DISABLE_FLOAT_HW_USE | 50 | + | QCBOR_DISABLE_TAGS | 400 | + | USEFULBUF_DISABLE_ALL_FLOAT | 950 | QCBOR_DISABLE_ENCODE_USAGE_GUARDS affects encoding only. It doesn't disable any encoding features, just some error checking. Disable it @@ -409,20 +464,25 @@ QCBOR_DISABLE_INDEFINITE_LENGTH_ARRAYS which will result in an error when an indefinite-length map or array arrives for decoding. QCBOR_DISABLE_UNCOMMON_TAGS disables the decoding of explicit tags for -base 64, regex, UUID and MIME data. This just disabled the automatic +base 64, regex, UUID and MIME data. This just disables the automatic recognition of these from a major type 6 tag. QCBOR_DISABLE_EXP_AND_MANTISSA disables the decoding of decimal fractions and big floats. +QCBOR_DISABLE_TAGS disables all decoding of CBOR tags. If the input has +a single tag, the error is unrecoverable so it is suitable only for protocols that +have no tags. "Borrowed" tag content formats (e.g. an epoch-based date +without the tag number), can still be processed. + See the discussion above on floating-point. ### Size of spiffy decode - + When creating a decode implementation, there is a choice of whether or not to use spiffy decode features or to just use QCBORDecode_GetNext(). - + The implementation using spiffy decode will be simpler resulting in the calling code being smaller, but the amount of code brought in from the QCBOR library will be larger. Basic use of spiffy decode @@ -430,7 +490,7 @@ See the discussion above on floating-point. concern, then it is probably better to use spiffy decode because it is less work, there is less complexity and less testing to worry about. - + If code size is a concern, then use of QCBORDecode_GetNext() will probably result in smaller overall code size for simpler CBOR protocols. However, if the CBOR protocol is complex then use of @@ -440,13 +500,13 @@ See the discussion above on floating-point. because the general purpose spiffy decode map processor is the one used for all the maps. - + ## Other Software Using QCBOR * [t_cose](https://github.com/laurencelundblade/t_cose) implements enough of [COSE, RFC 8152](https://tools.ietf.org/html/rfc8152) to support [CBOR Web Token (CWT)](https://tools.ietf.org/html/rfc8392) and -[Entity Attestation Token (EAT)](https://tools.ietf.org/html/draft-ietf-rats-eat-06). +[Entity Attestation Token (EAT)](https://tools.ietf.org/html/draft-ietf-rats-eat-06). Specifically it supports signing and verification of the COSE_Sign1 message. * [ctoken](https://github.com/laurencelundblade/ctoken) is an implementation of @@ -504,4 +564,4 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ### Copyright for this README Copyright (c) 2018-2021, Laurence Lundblade. All rights reserved. -Copyright (c) 2021, Arm Limited. All rights reserved. +Copyright (c) 2021-2023, Arm Limited. All rights reserved. diff --git a/3rdparty/exported/QCBOR/example.c b/3rdparty/exported/QCBOR/example.c index 80b21494d554..d580c78f0056 100644 --- a/3rdparty/exported/QCBOR/example.c +++ b/3rdparty/exported/QCBOR/example.c @@ -307,7 +307,7 @@ EngineDecodeErrors DecodeEngineSpiffy(UsefulBufC EncodedEngine, CarEngine *pE) } -int32_t RunQCborExample() +int32_t RunQCborExample(void) { CarEngine InitialEngine; CarEngine DecodedEngine; diff --git a/3rdparty/exported/QCBOR/qcbor/UsefulBuf.h b/3rdparty/exported/QCBOR/qcbor/UsefulBuf.h index 8a101fd473cf..aa245070299a 100644 --- a/3rdparty/exported/QCBOR/qcbor/UsefulBuf.h +++ b/3rdparty/exported/QCBOR/qcbor/UsefulBuf.h @@ -42,6 +42,7 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. when who what, where, why -------- ---- -------------------------------------------------- + 19/12/2022 llundblade Document that adding empty data is allowed. 4/11/2022 llundblade Add GetOutPlace and Advance to UsefulOutBuf. 9/21/2021 llundbla Clarify UsefulOutBuf size calculation mode 8/8/2021 dthaler/llundbla Work with C++ without compiler extensions @@ -673,15 +674,8 @@ static inline UsefulBuf UsefulBufC_Unconst(const UsefulBufC UBC) { UsefulBuf UB; - // See UsefulBuf_Unconst() implementation for comment on pragmas -#ifndef _MSC_VER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - UB.ptr = (void *)UBC.ptr; -#ifndef _MSC_VER -#pragma GCC diagnostic pop -#endif + // See UsefulBuf_Unconst() implementation for comment + UB.ptr = (void *)(uintptr_t)UBC.ptr; UB.len = UBC.len; @@ -954,6 +948,8 @@ static inline int UsefulOutBuf_AtStart(UsefulOutBuf *pUOutBuf); * Overlapping buffers are OK. @c NewData can point to data in the * output buffer. * + * NewData.len may be 0 in which case nothing will be inserted. + * * If an error occurs, an error state is set in the @ref * UsefulOutBuf. No error is returned. All subsequent attempts to add * data will do nothing. @@ -1759,16 +1755,9 @@ static inline UsefulBuf UsefulBuf_Unconst(const UsefulBufC UBC) UsefulBuf UB; /* -Wcast-qual is a good warning flag to use in general. This is - * the one place in UsefulBuf where it needs to be quieted. Since - * clang supports GCC pragmas, this works for clang too. */ -#ifndef _MSC_VER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - UB.ptr = (void *)UBC.ptr; -#ifndef _MSC_VER -#pragma GCC diagnostic pop -#endif + * the one place in UsefulBuf where it needs to be quieted. + */ + UB.ptr = (void *)(uintptr_t)UBC.ptr; UB.len = UBC.len; diff --git a/3rdparty/exported/QCBOR/qcbor/qcbor_common.h b/3rdparty/exported/QCBOR/qcbor/qcbor_common.h index e9f9a7c97b5c..127537d85322 100644 --- a/3rdparty/exported/QCBOR/qcbor/qcbor_common.h +++ b/3rdparty/exported/QCBOR/qcbor/qcbor_common.h @@ -468,6 +468,10 @@ typedef enum { indefinite length map or array in the input CBOR. */ QCBOR_ERR_INDEF_LEN_ARRAYS_DISABLED = 50, + /** All decoding of tags (major type 6) has been disabled and a tag + occurred in the decode input. */ + QCBOR_ERR_TAGS_DISABLED = 51, + #define QCBOR_END_OF_UNRECOVERABLE_DECODE_ERRORS 59 /** More than @ref QCBOR_MAX_TAGS_PER_ITEM tags encountered for a diff --git a/3rdparty/exported/QCBOR/qcbor/qcbor_decode.h b/3rdparty/exported/QCBOR/qcbor/qcbor_decode.h index 65eff4ca8d23..bf30e6ded32d 100644 --- a/3rdparty/exported/QCBOR/qcbor/qcbor_decode.h +++ b/3rdparty/exported/QCBOR/qcbor/qcbor_decode.h @@ -162,6 +162,23 @@ extern "C" { * item being sought, in which case the unrecoverable error will be * returned. Unrecoverable errors are those indicated by * QCBORDecode_IsUnrecoverableError(). + * + * @anchor Disabilng-Tag-Decoding + * # Disabilng Tag Decoding + * + * If QCBOR_DISABLE_TAGS is defined, all code for decoding tags will + * be omitted reducing the core decoder, QCBORDecode_VGetNext(), by + * about 400 bytes. If a tag number is encountered in the decoder + * input the unrecoverable error @ref QCBOR_ERR_TAGS_DISABLED will be + * returned. No input with tags can be decoded. + * + * Decode functions like QCBORDecode_GetEpochDate() and + * QCBORDecode_GetDecimalFraction() that can decode the tag content + * even if the tag number is absent are still available. Typically + * they won't be linked in because of dead stripping. The + * @c uTagRequirement parameter has no effect, but if it is + * @ref QCBOR_TAG_REQUIREMENT_TAG, @ref QCBOR_ERR_TAGS_DISABLED + * will be set. */ /** @@ -481,6 +498,7 @@ typedef struct _QCBORItem { uint64_t uint64; } label; +#ifndef QCBOR_DISABLE_TAGS /** * The tags numbers for which the item is the tag content. Tags * nest, so index 0 in the array is the tag on the data item @@ -502,6 +520,7 @@ typedef struct _QCBORItem { * having to reference this array. Also see @ref Tags-Overview. */ uint16_t uTags[QCBOR_MAX_TAGS_PER_ITEM]; +#endif } QCBORItem; diff --git a/3rdparty/exported/QCBOR/qcbor/qcbor_spiffy_decode.h b/3rdparty/exported/QCBOR/qcbor/qcbor_spiffy_decode.h index 91037822bcb9..0faddc386307 100644 --- a/3rdparty/exported/QCBOR/qcbor/qcbor_spiffy_decode.h +++ b/3rdparty/exported/QCBOR/qcbor/qcbor_spiffy_decode.h @@ -88,7 +88,7 @@ extern "C" { ## Tag Usage Data types beyond the basic CBOR types of numbers, strings, maps and - arrays are called tags. The main registry of these new types is in in + arrays are called tags. The main registry of these new types is in the IANA CBOR tags registry. These new types may be simple such a number that is to be interpreted as a date, or of moderate complexity such as defining a decimal fraction that is an array containing a @@ -296,6 +296,12 @@ static void QCBORDecode_GetInt64ConvertInMapSZ(QCBORDecodeContext *pCtx, See also QCBORDecode_GetInt64ConvertAll() which does some of these conversions, but links in much less object code. See also QCBORDecode_GetUInt64ConvertAll(). + + This relies on CBOR tags to identify big numbers, decimal fractions + and big floats. It will not attempt to decode non-tag CBOR that might + be one of these. (If QCBOR_DISABLE_TAGS is set, this is effectively + the same as QCBORDecode_GetInt64Convert() because all the additional + number types this decodes are tags). */ void QCBORDecode_GetInt64ConvertAll(QCBORDecodeContext *pCtx, uint32_t uConvertTypes, @@ -1210,6 +1216,10 @@ void QCBORDecode_GetBignumInMapSZ(QCBORDecodeContext *pCtx, See also @ref CBOR_TAG_DECIMAL_FRACTION, QCBOREncode_AddDecimalFraction(), @ref QCBOR_TYPE_DECIMAL_FRACTION and QCBORDecode_GetDecimalFractionBig(). + + If QCBOR_DISABLE_TAGS is set, the only input this will decode is + an array of two integers. It will set an error if the the array is preceded + by by a tag number or if the mantissa is a big number. */ void QCBORDecode_GetDecimalFraction(QCBORDecodeContext *pCtx, uint8_t uTagRequirement, @@ -1297,6 +1307,10 @@ void QCBORDecode_GetDecimalFractionBigInMapSZ(QCBORDecodeContext *pCtx, mantissa * ( 2 ** exponent ) + If the mantissa is a tag that is a positive or negative big number, + this will attempt to fit it into the int64_t that @c pnMantissa is + and set an overflow error if it doesn't fit. + See also QCBORDecode_GetInt64ConvertAll(), QCBORDecode_GetUInt64ConvertAll() and QCBORDecode_GetDoubleConvertAll() which can convert big floats. @@ -1979,21 +1993,28 @@ QCBORDecode_GetDoubleInMapSZ(QCBORDecodeContext *pMe, -// Semi private (this may change in the future) #define QCBOR_TAGSPEC_NUM_TYPES 4 -/* Improvement: Carefully understand what compilers do with this, -particularly initialization and see if it can be optimized so -there is less code and maybe so it can be smaller. */ +/* Semi-private data structure (which might change). + * + * See CheckTagRequirement() which uses this to check the type of a + * item to be decoded as a tag or tag content. + * + * Improvement: Carefully understand what compilers do with this, + * particularly initialization and see if it can be optimized so there + * is less code and maybe so it can be smaller. + */ typedef struct { /* One of QCBOR_TAGSPEC_MATCH_xxx */ uint8_t uTagRequirement; - /* The tagged type translated into QCBOR_TYPE_XXX. Used to match explicit - tagging */ + /* The tagged type translated into QCBOR_TYPE_XXX. Used to match + * explicit tagging */ uint8_t uTaggedTypes[QCBOR_TAGSPEC_NUM_TYPES]; - /* The types of the content, which are used to match implicit tagging */ + /* The types of the content, which are used to match implicit + * tagging */ uint8_t uAllowedContentTypes[QCBOR_TAGSPEC_NUM_TYPES]; } TagSpecification; + // Semi private void QCBORDecode_GetTaggedStringInternal(QCBORDecodeContext *pMe, TagSpecification TagSpec, diff --git a/3rdparty/exported/QCBOR/src/UsefulBuf.c b/3rdparty/exported/QCBOR/src/UsefulBuf.c index f5149a5a661a..b36e5d00d36d 100644 --- a/3rdparty/exported/QCBOR/src/UsefulBuf.c +++ b/3rdparty/exported/QCBOR/src/UsefulBuf.c @@ -41,7 +41,8 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. when who what, where, why -------- ---- --------------------------------------------------- - 4/11/2022 llundblade Add GetOutPlace and Advance to UsefulOutBuf + 19/12/2022 llundblade Don't pass NULL to memmove when adding empty data. + 4/11/2022 llundblade Add GetOutPlace and Advance to UsefulOutBuf 3/6/2021 mcr/llundblade Fix warnings related to --Wcast-qual 01/28/2020 llundblade Refine integer signedness to quiet static analysis. 01/08/2020 llundblade Documentation corrections & improved code formatting. @@ -254,22 +255,23 @@ void UsefulOutBuf_InsertUsefulBuf(UsefulOutBuf *pMe, UsefulBufC NewData, size_t } /* 3. Slide existing data to the right */ - uint8_t *pSourceOfMove = ((uint8_t *)pMe->UB.ptr) + uInsertionPos; // PtrMath #1 - size_t uNumBytesToMove = pMe->data_len - uInsertionPos; // PtrMath #2 - uint8_t *pDestinationOfMove = pSourceOfMove + NewData.len; // PtrMath #3 + if (!UsefulOutBuf_IsBufferNULL(pMe)) { + uint8_t *pSourceOfMove = ((uint8_t *)pMe->UB.ptr) + uInsertionPos; // PtrMath #1 + size_t uNumBytesToMove = pMe->data_len - uInsertionPos; // PtrMath #2 + uint8_t *pDestinationOfMove = pSourceOfMove + NewData.len; // PtrMath #3 - if(uNumBytesToMove && pMe->UB.ptr) { // To know memmove won't go off end of destination, see PtrMath #4 // Use memove because it handles overlapping buffers memmove(pDestinationOfMove, pSourceOfMove, uNumBytesToMove); - } - /* 4. Put the new data in */ - uint8_t *pInsertionPoint = ((uint8_t *)pMe->UB.ptr) + uInsertionPos; // PtrMath #5 - if(pMe->UB.ptr) { - // To know memmove won't go off end of destination, see PtrMath #6 - memmove(pInsertionPoint, NewData.ptr, NewData.len); + /* 4. Put the new data in */ + uint8_t *pInsertionPoint = pSourceOfMove; + // To know memmove won't go off end of destination, see PtrMath #5 + if(NewData.ptr != NULL) { + memmove(pInsertionPoint, NewData.ptr, NewData.len); + } } + pMe->data_len += NewData.len; } @@ -295,9 +297,7 @@ void UsefulOutBuf_InsertUsefulBuf(UsefulOutBuf *pMe, UsefulBufC NewData, size_t Check #3 allows Check #2 to be refactored as NewData.Len > (me->size - uInsertionPos) This algebraically rearranges to me->size > uInsertionPos + NewData.len - PtrMath #5 is exactly the same as PtrMath #1 - - PtrMath #6 will never wrap under because + PtrMath #5 will never wrap under because Calculation for extent of memove is uRoomInDestination = me->UB.len - uInsertionPos; Check #1 makes sure me->data_len is less than me->size Check #3 makes sure uInsertionPos is less than me->data_len diff --git a/3rdparty/exported/QCBOR/src/qcbor_decode.c b/3rdparty/exported/QCBOR/src/qcbor_decode.c index 46c4f126ad3c..8a547eef0a66 100644 --- a/3rdparty/exported/QCBOR/src/qcbor_decode.c +++ b/3rdparty/exported/QCBOR/src/qcbor_decode.c @@ -501,18 +501,11 @@ DecodeNesting_GetPreviousBoundedEnd(const QCBORDecodeNesting *pMe) static inline void StringAllocator_Free(const QCBORInternalAllocator *pMe, const void *pMem) { - /* These pragmas allow the "-Wcast-qual" warnings flag to be set for - * gcc and clang. This is the one place where the const needs to be - * cast away so const can be use in the rest of the code. + /* This cast to uintptr_t suppresses the "-Wcast-qual" warnings. + * This is the one place where the const needs to be cast away so const can + * be use in the rest of the code. */ -#ifndef _MSC_VER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - (pMe->pfAllocator)(pMe->pAllocateCxt, (void *)pMem, 0); -#ifndef _MSC_VER -#pragma GCC diagnostic pop -#endif + (pMe->pfAllocator)(pMe->pAllocateCxt, (void *)(uintptr_t)pMem, 0); } // StringAllocator_Reallocate called with pMem NULL is @@ -523,14 +516,7 @@ StringAllocator_Reallocate(const QCBORInternalAllocator *pMe, size_t uSize) { /* See comment in StringAllocator_Free() */ -#ifndef _MSC_VER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - return (pMe->pfAllocator)(pMe->pAllocateCxt, (void *)pMem, uSize); -#ifndef _MSC_VER -#pragma GCC diagnostic pop -#endif + return (pMe->pfAllocator)(pMe->pAllocateCxt, (void *)(uintptr_t)pMem, uSize); } static inline UsefulBuf @@ -543,16 +529,9 @@ static inline void StringAllocator_Destruct(const QCBORInternalAllocator *pMe) { /* See comment in StringAllocator_Free() */ -#ifndef _MSC_VER -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif if(pMe->pfAllocator) { (pMe->pfAllocator)(pMe->pAllocateCxt, NULL, 0); } -#ifndef _MSC_VER -#pragma GCC diagnostic pop -#endif } #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ @@ -1163,12 +1142,16 @@ DecodeAtomicDataItem(UsefulInputBuf *pUInBuf, break; case CBOR_MAJOR_TYPE_TAG: /* Major type 6, tag numbers */ +#ifndef QCBOR_DISABLE_TAGS if(nAdditionalInfo == LEN_IS_INDEFINITE) { uReturn = QCBOR_ERR_BAD_INT; } else { pDecodedItem->val.uTagV = uArgument; pDecodedItem->uDataType = QCBOR_TYPE_TAG; } +#else /* QCBOR_DISABLE_TAGS */ + uReturn = QCBOR_ERR_TAGS_DISABLED; +#endif /* QCBOR_DISABLE_TAGS */ break; case CBOR_MAJOR_TYPE_SIMPLE: @@ -1347,6 +1330,7 @@ QCBORDecode_GetNextFullString(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) } +#ifndef QCBOR_DISABLE_TAGS /** * @brief This converts a tag number to a shorter mapped value for storage. * @@ -1421,6 +1405,7 @@ UnMapTagNumber(const QCBORDecodeContext *pMe, uint16_t uMappedTagNumber) return pMe->auMappedTags[uIndex]; } } +#endif /* QCBOR_DISABLE_TAGS */ /** @@ -1450,6 +1435,7 @@ UnMapTagNumber(const QCBORDecodeContext *pMe, uint16_t uMappedTagNumber) static QCBORError QCBORDecode_GetNextTagNumber(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) { +#ifndef QCBOR_DISABLE_TAGS /* Accummulate the tags from multiple items here and then copy them * into the last item, the non-tag item. */ @@ -1508,8 +1494,13 @@ QCBORDecode_GetNextTagNumber(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) Done: return uReturn; -} +#else /* QCBOR_DISABLE_TAGS */ + + return QCBORDecode_GetNextFullString(pMe, pDecodedItem); + +#endif /* QCBOR_DISABLE_TAGS */ +} /** * @brief Combine a map entry label and value into one item (decode layer 3). @@ -1900,6 +1891,7 @@ QCBORDecode_GetNextMapOrArray(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) } +#ifndef QCBOR_DISABLE_TAGS /** * @brief Shift 0th tag out of the tag list. * @@ -1916,6 +1908,7 @@ static inline void ShiftTags(QCBORItem *pDecodedItem) pDecodedItem->uTags[QCBOR_MAX_TAGS_PER_ITEM-1] = CBOR_TAG_INVALID16; } +#endif /* QCBOR_DISABLE_TAGS */ /** * @brief Convert different epoch date formats in to the QCBOR epoch date format @@ -2074,6 +2067,18 @@ static QCBORError DecodeDaysEpoch(QCBORItem *pDecodedItem) #ifndef QCBOR_DISABLE_EXP_AND_MANTISSA + +/* Forward declaration is necessary for + * QCBORDecode_MantissaAndExponent(). to be able to decode bignum + * tags in the mantissa. If the mantissa is a decimal fraction or big + * float in error, this will result in a recurive call to + * QCBORDecode_MantissaAndExponent(), but the recursion will unwined + * correctly and the correct error is returned. + */ +static QCBORError +QCBORDecode_GetNextTagContent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem); + + /** * @brief Decode decimal fractions and big floats. * @@ -2086,14 +2091,19 @@ static QCBORError DecodeDaysEpoch(QCBORItem *pDecodedItem) * @returns Decoding errors from getting primitive data items or * \ref QCBOR_ERR_BAD_EXP_AND_MANTISSA. * - * When called pDecodedItem must be the array that is tagged as a big - * float or decimal fraction, the array that has the two members, the + * When called pDecodedItem must be the array with two members, the * exponent and mantissa. * * This will fetch and decode the exponent and mantissa and put the * result back into pDecodedItem. + * + * This does no checking or processing of tag numbers. That is to be + * done by the code that calls this. + * + * This stuffs the type of the mantissa into pDecodedItem with the expectation + * the caller will process it. */ -static inline QCBORError +static QCBORError QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) { QCBORError uReturn; @@ -2105,16 +2115,12 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem } /* A check for pDecodedItem->val.uCount == 2 would work for - * definite-length arrays, but not for indefnite. Instead remember + * definite-length arrays, but not for indefinite. Instead remember * the nesting level the two integers must be at, which is one * deeper than that of the array. */ const int nNestLevel = pDecodedItem->uNestingLevel + 1; - /* --- Which is it, decimal fraction or a bigfloat? --- */ - const bool bIsTaggedDecimalFraction = QCBORDecode_IsTagged(pMe, pDecodedItem, CBOR_TAG_DECIMAL_FRACTION); - pDecodedItem->uDataType = bIsTaggedDecimalFraction ? QCBOR_TYPE_DECIMAL_FRACTION : QCBOR_TYPE_BIGFLOAT; - /* --- Get the exponent --- */ QCBORItem exponentItem; uReturn = QCBORDecode_GetNextMapOrArray(pMe, &exponentItem); @@ -2142,7 +2148,7 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem /* --- Get the mantissa --- */ QCBORItem mantissaItem; - uReturn = QCBORDecode_GetNextWithTags(pMe, &mantissaItem, NULL); + uReturn = QCBORDecode_GetNextTagContent(pMe, &mantissaItem); if(uReturn != QCBOR_SUCCESS) { goto Done; } @@ -2151,6 +2157,9 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem uReturn = QCBOR_ERR_BAD_EXP_AND_MANTISSA; goto Done; } + /* Stuff the mantissa data type into the item to send it up to the + * the next level. */ + pDecodedItem->uDataType = mantissaItem.uDataType; if(mantissaItem.uDataType == QCBOR_TYPE_INT64) { /* Data arriving as an unsigned int < INT64_MAX has been * converted to QCBOR_TYPE_INT64 and thus handled here. This is @@ -2159,14 +2168,16 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem * and thus an error that will get handled in an else below. */ pDecodedItem->val.expAndMantissa.Mantissa.nInt = mantissaItem.val.int64; +#ifndef QCBOR_DISABLE_TAGS + /* With tags fully disabled a big number mantissa will error out + * in the call to QCBORDecode_GetNextWithTags() because it has + * a tag number. + */ } else if(mantissaItem.uDataType == QCBOR_TYPE_POSBIGNUM || mantissaItem.uDataType == QCBOR_TYPE_NEGBIGNUM) { /* Got a good big num mantissa */ pDecodedItem->val.expAndMantissa.Mantissa.bigNum = mantissaItem.val.bigNum; - /* Depends on numbering of QCBOR_TYPE_XXX */ - pDecodedItem->uDataType = (uint8_t)(pDecodedItem->uDataType + - mantissaItem.uDataType - QCBOR_TYPE_POSBIGNUM + - 1); +#endif /* QCBOR_DISABLE_TAGS */ } else { /* Wrong type of mantissa or a QCBOR_TYPE_UINT64 > INT64_MAX */ uReturn = QCBOR_ERR_BAD_EXP_AND_MANTISSA; @@ -2176,6 +2187,7 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem /* --- Check that array only has the two numbers --- */ if(mantissaItem.uNextNestLevel == nNestLevel) { /* Extra items in the decimal fraction / big float */ + /* Improvement: this should probably be an unrecoverable error. */ uReturn = QCBOR_ERR_BAD_EXP_AND_MANTISSA; goto Done; } @@ -2187,6 +2199,8 @@ QCBORDecode_MantissaAndExponent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem #endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ +#ifndef QCBOR_DISABLE_TAGS + #ifndef QCBOR_DISABLE_UNCOMMON_TAGS /** * @brief Decode the MIME type tag @@ -2215,7 +2229,6 @@ static inline QCBORError DecodeMIME(QCBORItem *pDecodedItem) } #endif /* QCBOR_DISABLE_UNCOMMON_TAGS */ - /** * Table of CBOR tags whose content is either a text string or a byte * string. The table maps the CBOR tag to the QCBOR type. The high-bit @@ -2303,6 +2316,31 @@ ProcessTaggedString(uint16_t uTag, QCBORItem *pDecodedItem) pDecodedItem->uDataType = (uint8_t)(uQCBORType & QCBOR_TYPE_MASK); return QCBOR_SUCCESS; } +#endif /* QCBOR_DISABLE_TAGS */ + + +#ifndef QCBOR_CONFIG_DISABLE_EXP_AND_MANTISSA +/* + * This returns the QCBOR_TYPE for a mantissa and exponent. + +Called in one context where there is always a tag + + Called in another context where there might be a tag or the caller might say what they are expecting. + + 6 possible outputs + */ +static inline uint8_t +MantissaExponentDataType(const uint16_t uTagToProcess, const QCBORItem *pDecodedItem) +{ + uint8_t uBase = uTagToProcess == CBOR_TAG_DECIMAL_FRACTION ? + QCBOR_TYPE_DECIMAL_FRACTION : + QCBOR_TYPE_BIGFLOAT; + if(pDecodedItem->uDataType != QCBOR_TYPE_INT64) { + uBase = (uint8_t)(uBase + pDecodedItem->uDataType - QCBOR_TYPE_POSBIGNUM + 1); + } + return uBase; +} +#endif /* QCBOR_CONFIG_DISABLE_EXP_AND_MANTISSA */ /** @@ -2328,6 +2366,7 @@ QCBORDecode_GetNextTagContent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) goto Done; } +#ifndef QCBOR_DISABLE_TAGS /* When there are no tag numbers for the item, this exits first * thing and effectively does nothing. * @@ -2357,6 +2396,9 @@ QCBORDecode_GetNextTagContent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) } else if(uTagToProcess == CBOR_TAG_DECIMAL_FRACTION || uTagToProcess == CBOR_TAG_BIGFLOAT) { uReturn = QCBORDecode_MantissaAndExponent(pMe, pDecodedItem); + /* --- Which is it, decimal fraction or a bigfloat? --- */ + pDecodedItem->uDataType = MantissaExponentDataType(uTagToProcess, pDecodedItem); + #endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ #ifndef QCBOR_DISABLE_UNCOMMON_TAGS } else if(uTagToProcess == CBOR_TAG_MIME || @@ -2388,6 +2430,7 @@ QCBORDecode_GetNextTagContent(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem) */ ShiftTags(pDecodedItem); } +#endif /* QCBOR_DISABLE_TAGS */ Done: return uReturn; @@ -2463,6 +2506,8 @@ QCBORDecode_GetNextWithTags(QCBORDecodeContext *pMe, QCBORItem *pDecodedItem, QCBORTagListOut *pTags) { +#ifndef QCBOR_DISABLE_TAGS + QCBORError uReturn; uReturn = QCBORDecode_GetNext(pMe, pDecodedItem); @@ -2486,6 +2531,13 @@ QCBORDecode_GetNextWithTags(QCBORDecodeContext *pMe, } return QCBOR_SUCCESS; + +#else /* QCBOR_DISABLE_TAGS */ + (void)pMe; + (void)pDecodedItem; + (void)pTags; + return QCBOR_ERR_TAGS_DISABLED; +#endif /* QCBOR_DISABLE_TAGS */ } @@ -2496,6 +2548,7 @@ bool QCBORDecode_IsTagged(QCBORDecodeContext *pMe, const QCBORItem *pItem, uint64_t uTag) { +#ifndef QCBOR_DISABLE_TAGS for(unsigned uTagIndex = 0; uTagIndex < QCBOR_MAX_TAGS_PER_ITEM; uTagIndex++) { if(pItem->uTags[uTagIndex] == CBOR_TAG_INVALID16) { break; @@ -2504,6 +2557,11 @@ bool QCBORDecode_IsTagged(QCBORDecodeContext *pMe, return true; } } +#else /* QCBOR_TAGS_DISABLED */ + (void)pMe; + (void)pItem; + (void)uTag; +#endif /* QCBOR_TAGS_DISABLED */ return false; } @@ -2564,6 +2622,7 @@ uint64_t QCBORDecode_GetNthTag(QCBORDecodeContext *pMe, const QCBORItem *pItem, uint32_t uIndex) { +#ifndef QCBOR_DISABLE_TAGS if(pItem->uDataType == QCBOR_TYPE_NONE) { return CBOR_TAG_INVALID64; } @@ -2572,6 +2631,13 @@ uint64_t QCBORDecode_GetNthTag(QCBORDecodeContext *pMe, } else { return UnMapTagNumber(pMe, pItem->uTags[uIndex]); } +#else /* QCBOR_DISABLE_TAGS */ + (void)pMe; + (void)pItem; + (void)uIndex; + + return CBOR_TAG_INVALID64; +#endif /* QCBOR_DISABLE_TAGS */ } @@ -2581,6 +2647,8 @@ uint64_t QCBORDecode_GetNthTag(QCBORDecodeContext *pMe, uint64_t QCBORDecode_GetNthTagOfLast(const QCBORDecodeContext *pMe, uint32_t uIndex) { +#ifndef QCBOR_DISABLE_TAGS + if(pMe->uLastError != QCBOR_SUCCESS) { return CBOR_TAG_INVALID64; } @@ -2589,6 +2657,12 @@ uint64_t QCBORDecode_GetNthTagOfLast(const QCBORDecodeContext *pMe, } else { return UnMapTagNumber(pMe, pMe->uLastTags[uIndex]); } +#else /* QCBOR_DISABLE_TAGS */ + (void)pMe; + (void)uIndex; + + return CBOR_TAG_INVALID64; +#endif /* QCBOR_DISABLE_TAGS */ } @@ -2784,9 +2858,15 @@ QCBORError QCBORDecode_SetMemPool(QCBORDecodeContext *pMe, -static inline void CopyTags(QCBORDecodeContext *pMe, const QCBORItem *pItem) +static inline void +CopyTags(QCBORDecodeContext *pMe, const QCBORItem *pItem) { +#ifndef QCBOR_DISABLE_TAGS memcpy(pMe->uLastTags, pItem->uTags, sizeof(pItem->uTags)); +#else + (void)pMe; + (void)pItem; +#endif } @@ -3224,7 +3304,6 @@ void QCBORDecode_GetItemInMapSZ(QCBORDecodeContext *pMe, } - static QCBORError CheckTypeList(int uDataType, const uint8_t puTypeList[QCBOR_TAGSPEC_NUM_TYPES]) { @@ -3238,30 +3317,47 @@ CheckTypeList(int uDataType, const uint8_t puTypeList[QCBOR_TAGSPEC_NUM_TYPES]) /** - @param[in] TagSpec Specification for matching tags. - @param[in] pItem The item to check. - - @retval QCBOR_SUCCESS \c uDataType is allowed by @c TagSpec - @retval QCBOR_ERR_UNEXPECTED_TYPE \c uDataType is not allowed by @c TagSpec - - The data type must be one of the QCBOR_TYPEs, not the IETF CBOR Registered - tag value. + * Match a tag/type specification against the type of the item. + * + * @param[in] TagSpec Specification for matching tags. + * @param[in] pItem The item to check. + * + * @retval QCBOR_SUCCESS \c uDataType is allowed by @c TagSpec + * @retval QCBOR_ERR_UNEXPECTED_TYPE \c uDataType is not allowed by @c TagSpec + * + * This checks the item data type of untagged items as well as of + * tagged items against a specification to see if decoding should + * proceed. + * + * This relies on the automatic tag decoding done by QCBOR that turns + * tag numbers into particular QCBOR_TYPEs so there is no actual + * comparsion of tag numbers, just of QCBOR_TYPEs. + * + * This checks the data item type as possibly representing the tag + * number or as the tag content type. + * + * If QCBOR_DISABLE_TAGS is #defined, this primarily checks the item + * data type against the allowed tag content types. It will also error out + * if the caller tries to require a tag because there is no way that can + * ever be fulfilled. */ static QCBORError CheckTagRequirement(const TagSpecification TagSpec, const QCBORItem *pItem) { + const int nItemType = pItem->uDataType; + const int nTagReq = TagSpec.uTagRequirement & ~QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS; + +#ifndef QCBOR_DISABLE_TAGS if(!(TagSpec.uTagRequirement & QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS) && pItem->uTags[0] != CBOR_TAG_INVALID16) { /* There are tags that QCBOR couldn't process on this item and - the caller has told us there should not be. */ + * the caller has told us there should not be. + */ return QCBOR_ERR_UNEXPECTED_TYPE; } - const int nTagReq = TagSpec.uTagRequirement & ~QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS; - const int nItemType = pItem->uDataType; - if(nTagReq == QCBOR_TAG_REQUIREMENT_TAG) { - // Must match the tag and only the tag + /* Must match the tag number and only the tag */ return CheckTypeList(nItemType, TagSpec.uTaggedTypes); } @@ -3272,26 +3368,35 @@ CheckTagRequirement(const TagSpecification TagSpec, const QCBORItem *pItem) if(nTagReq == QCBOR_TAG_REQUIREMENT_NOT_A_TAG) { /* Must match the content type and only the content type. - There was no match just above so it is a fail. */ + * There was no match just above so it is a fail. */ return QCBOR_ERR_UNEXPECTED_TYPE; } - /* If here it can match either the tag or the content - and it hasn't matched the content, so the end - result is whether it matches the tag. This is - also the case that the CBOR standard discourages. */ + /* QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG: If here it can match either the tag or the content + * and it hasn't matched the content, so the end + * result is whether it matches the tag. This is + * the tag optional case that the CBOR standard discourages. + */ return CheckTypeList(nItemType, TagSpec.uTaggedTypes); -} +#else /* QCBOR_DISABLE_TAGS */ + if(nTagReq == QCBOR_TAG_REQUIREMENT_TAG) { + return QCBOR_ERR_UNEXPECTED_TYPE; + } + + return CheckTypeList(nItemType, TagSpec.uAllowedContentTypes); + +#endif /* QCBOR_DISABLE_TAGS */ +} // This could be semi-private if need be static inline -void QCBORDecode_GetTaggedItemInMapN(QCBORDecodeContext *pMe, - int64_t nLabel, - TagSpecification TagSpec, - QCBORItem *pItem) +void QCBORDecode_GetTaggedItemInMapN(QCBORDecodeContext *pMe, + const int64_t nLabel, + const TagSpecification TagSpec, + QCBORItem *pItem) { QCBORDecode_GetItemInMapN(pMe, nLabel, QCBOR_TYPE_ANY, pItem); if(pMe->uLastError != QCBOR_SUCCESS) { @@ -3304,10 +3409,10 @@ void QCBORDecode_GetTaggedItemInMapN(QCBORDecodeContext *pMe, // This could be semi-private if need be static inline -void QCBORDecode_GetTaggedItemInMapSZ(QCBORDecodeContext *pMe, - const char *szLabel, - TagSpecification TagSpec, - QCBORItem *pItem) +void QCBORDecode_GetTaggedItemInMapSZ(QCBORDecodeContext *pMe, + const char *szLabel, + const TagSpecification TagSpec, + QCBORItem *pItem) { QCBORDecode_GetItemInMapSZ(pMe, szLabel, QCBOR_TYPE_ANY, pItem); if(pMe->uLastError != QCBOR_SUCCESS) { @@ -3321,7 +3426,7 @@ void QCBORDecode_GetTaggedItemInMapSZ(QCBORDecodeContext *pMe, void QCBORDecode_GetTaggedStringInMapN(QCBORDecodeContext *pMe, int64_t nLabel, TagSpecification TagSpec, - UsefulBufC *pString) + UsefulBufC *pString) { QCBORItem Item; QCBORDecode_GetTaggedItemInMapN(pMe, nLabel, TagSpec, &Item); @@ -3631,10 +3736,11 @@ void QCBORDecode_ExitBoundedMapOrArray(QCBORDecodeContext *pMe, uint8_t uType) -static QCBORError InternalEnterBstrWrapped(QCBORDecodeContext *pMe, - const QCBORItem *pItem, - uint8_t uTagRequirement, - UsefulBufC *pBstr) +static QCBORError +InternalEnterBstrWrapped(QCBORDecodeContext *pMe, + const QCBORItem *pItem, + const uint8_t uTagRequirement, + UsefulBufC *pBstr) { if(pBstr) { *pBstr = NULLUsefulBufC; @@ -3875,7 +3981,7 @@ void QCBORDecode_GetBoolInMapSZ(QCBORDecodeContext *pMe, const char *szLabel, bo static void ProcessEpochDate(QCBORDecodeContext *pMe, QCBORItem *pItem, - uint8_t uTagRequirement, + const uint8_t uTagRequirement, int64_t *pnTime) { if(pMe->uLastError != QCBOR_SUCCESS) { @@ -4054,13 +4160,16 @@ QCBORDecode_GetEpochDaysInMapSZ(QCBORDecodeContext *pMe, - -void QCBORDecode_GetTaggedStringInternal(QCBORDecodeContext *pMe, - TagSpecification TagSpec, - UsefulBufC *pBstr) +/* + * @brief Get a string that matches the type/tag specification. + */ +void +QCBORDecode_GetTaggedStringInternal(QCBORDecodeContext *pMe, + const TagSpecification TagSpec, + UsefulBufC *pBstr) { if(pMe->uLastError != QCBOR_SUCCESS) { - // Already in error state, do nothing + /* Already in error state, do nothing */ return; } @@ -4085,10 +4194,11 @@ void QCBORDecode_GetTaggedStringInternal(QCBORDecodeContext *pMe, -static QCBORError ProcessBigNum(uint8_t uTagRequirement, - const QCBORItem *pItem, - UsefulBufC *pValue, - bool *pbIsNegative) +static QCBORError +ProcessBigNum(const uint8_t uTagRequirement, + const QCBORItem *pItem, + UsefulBufC *pValue, + bool *pbIsNegative) { const TagSpecification TagSpec = { @@ -4179,10 +4289,11 @@ void QCBORDecode_GetBignumInMapSZ(QCBORDecodeContext *pMe, // Semi private -QCBORError QCBORDecode_GetMIMEInternal(uint8_t uTagRequirement, - const QCBORItem *pItem, - UsefulBufC *pMessage, - bool *pbIsTag257) +QCBORError +QCBORDecode_GetMIMEInternal(const uint8_t uTagRequirement, + const QCBORItem *pItem, + UsefulBufC *pMessage, + bool *pbIsTag257) { const TagSpecification TagSpecText = { @@ -4230,7 +4341,20 @@ QCBORError QCBORDecode_GetMIMEInternal(uint8_t uTagRequirement, typedef QCBORError (*fExponentiator)(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult); -// The exponentiator that works on only positive numbers +/** + * @brief Base 10 exponentiate a mantissa and exponent into an unsigned 64-bit integer. + * + * @param[in] uMantissa The unsigned integer mantissa. + * @param[in] nExponent The signed integer exponent. + * @param[out] puResult Place to return the unsigned integer result. + * + * This computes: mantissa * 10 ^^ exponent as for a decimal fraction. The output is a 64-bit + * unsigned integer. + * + * There are many inputs for which the result will not fit in the + * 64-bit integer and \ref QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW will + * be returned. + */ static QCBORError Exponentitate10(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) { @@ -4243,7 +4367,7 @@ Exponentitate10(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) */ for(; nExponent > 0; nExponent--) { if(uResult > UINT64_MAX / 10) { - return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; // Error overflow + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; } uResult = uResult * 10; } @@ -4251,7 +4375,7 @@ Exponentitate10(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) for(; nExponent < 0; nExponent++) { uResult = uResult / 10; if(uResult == 0) { - return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; // Underflow error + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; } } } @@ -4263,7 +4387,20 @@ Exponentitate10(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) } -// The exponentiator that works on only positive numbers +/** + * @brief Base 2 exponentiate a mantissa and exponent into an unsigned 64-bit integer. + * + * @param[in] uMantissa The unsigned integer mantissa. + * @param[in] nExponent The signed integer exponent. + * @param[out] puResult Place to return the unsigned integer result. + * + * This computes: mantissa * 2 ^^ exponent as for a big float. The + * output is a 64-bit unsigned integer. + * + * There are many inputs for which the result will not fit in the + * 64-bit integer and \ref QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW will + * be returned. + */ static QCBORError Exponentitate2(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) { @@ -4271,13 +4408,12 @@ Exponentitate2(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) uResult = uMantissa; - /* This loop will run a maximum of 64 times because - * INT64_MAX < 2^31. More than that will cause - * exit with the overflow error + /* This loop will run a maximum of 64 times because INT64_MAX < + * 2^31. More than that will cause exit with the overflow error */ while(nExponent > 0) { if(uResult > UINT64_MAX >> 1) { - return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; // Error overflow + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; } uResult = uResult << 1; nExponent--; @@ -4285,7 +4421,7 @@ Exponentitate2(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) while(nExponent < 0 ) { if(uResult == 0) { - return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; // Underflow error + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; } uResult = uResult >> 1; nExponent++; @@ -4297,77 +4433,145 @@ Exponentitate2(uint64_t uMantissa, int64_t nExponent, uint64_t *puResult) } -/* - Compute value with signed mantissa and signed result. Works with - exponent of 2 or 10 based on exponentiator. +/** + * @brief Exponentiate a signed mantissa and signed exponent to produce a signed result. + * + * @param[in] nMantissa Signed integer mantissa. + * @param[in] nExponent Signed integer exponent. + * @param[out] pnResult Place to put the signed integer result. + * @param[in] pfExp Exponentiation function. + * + * @returns Error code + * + * \c pfExp performs exponentiation on and unsigned mantissa and + * produces an unsigned result. This converts the mantissa from signed + * and converts the result to signed. The exponentiation function is + * either for base 2 or base 10 (and could be other if needed). */ -static inline QCBORError ExponentiateNN(int64_t nMantissa, - int64_t nExponent, - int64_t *pnResult, - fExponentiator pfExp) +static QCBORError +ExponentiateNN(int64_t nMantissa, + int64_t nExponent, + int64_t *pnResult, + fExponentiator pfExp) { uint64_t uResult; + uint64_t uMantissa; - // Take the absolute value of the mantissa and convert to unsigned. - // Improvement: this should be possible in one instruction - uint64_t uMantissa = nMantissa > 0 ? (uint64_t)nMantissa : (uint64_t)-nMantissa; + /* Take the absolute value and put it into an unsigned. */ + if(nMantissa >= 0) { + /* Positive case is straightforward */ + uMantissa = (uint64_t)nMantissa; + } else if(nMantissa != INT64_MIN) { + /* The common negative case. See next. */ + uMantissa = (uint64_t)-nMantissa; + } else { + /* int64_t and uint64_t are always two's complement per the + * C standard (and since QCBOR uses these it only works with + * two's complement, which is pretty much universal these + * days). The range of a negative two's complement integer is + * one more that than a positive, so the simple code above might + * not work all the time because you can't simply negate the + * value INT64_MIN because it can't be represented in an + * int64_t. -INT64_MIN can however be represented in a + * uint64_t. Some compilers seem to recognize this case for the + * above code and put the correct value in uMantissa, however + * they are not required to do this by the C standard. This next + * line does however work for all compilers. + * + * This does assume two's complement where -INT64_MIN == + * INT64_MAX + 1 (which wouldn't be true for one's complement or + * sign and magnitude (but we know we're using two's complement + * because int64_t requires it)). + * + * See these, particularly the detailed commentary: + * https://stackoverflow.com/questions/54915742/does-c99-mandate-a-int64-t-type-be-available-always + * https://stackoverflow.com/questions/37301078/is-negating-int-min-undefined-behaviour + */ + uMantissa = (uint64_t)INT64_MAX+1; + } - // Do the exponentiation of the positive mantissa + /* Call the exponentiator passed for either base 2 or base 10. + * Here is where most of the overflow errors are caught. */ QCBORError uReturn = (*pfExp)(uMantissa, nExponent, &uResult); if(uReturn) { return uReturn; } - - /* (uint64_t)INT64_MAX+1 is used to represent the absolute value - of INT64_MIN. This assumes two's compliment representation where - INT64_MIN is one increment farther from 0 than INT64_MAX. - Trying to write -INT64_MIN doesn't work to get this because the - compiler tries to work with an int64_t which can't represent - -INT64_MIN. - */ - uint64_t uMax = nMantissa > 0 ? INT64_MAX : (uint64_t)INT64_MAX+1; - - // Error out if too large - if(uResult > uMax) { - return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; + /* Convert back to the sign of the original mantissa */ + if(nMantissa >= 0) { + if(uResult > INT64_MAX) { + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; + } + *pnResult = (int64_t)uResult; + } else { + /* (uint64_t)INT64_MAX+1 is used to represent the absolute value + * of INT64_MIN. This assumes two's compliment representation + * where INT64_MIN is one increment farther from 0 than + * INT64_MAX. Trying to write -INT64_MIN doesn't work to get + * this because the compiler makes it an int64_t which can't + * represent -INT64_MIN. Also see above. + */ + if(uResult > (uint64_t)INT64_MAX+1) { + return QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW; + } + *pnResult = -(int64_t)uResult; } - // Casts are safe because of checks above - *pnResult = nMantissa > 0 ? (int64_t)uResult : -(int64_t)uResult; - return QCBOR_SUCCESS; } -/* - Compute value with signed mantissa and unsigned result. Works with - exponent of 2 or 10 based on exponentiator. +/** + * @brief Exponentiate an unsigned mantissa and signed exponent to produce an unsigned result. + * + * @param[in] nMantissa Signed integer mantissa. + * @param[in] nExponent Signed integer exponent. + * @param[out] puResult Place to put the signed integer result. + * @param[in] pfExp Exponentiation function. + * + * @returns Error code + * + * \c pfExp performs exponentiation on and unsigned mantissa and + * produces an unsigned result. This errors out if the mantissa + * is negative because the output is unsigned. */ -static inline QCBORError ExponentitateNU(int64_t nMantissa, - int64_t nExponent, - uint64_t *puResult, - fExponentiator pfExp) +static QCBORError +ExponentitateNU(int64_t nMantissa, + int64_t nExponent, + uint64_t *puResult, + fExponentiator pfExp) { if(nMantissa < 0) { return QCBOR_ERR_NUMBER_SIGN_CONVERSION; } - // Cast to unsigned is OK because of check for negative - // Cast to unsigned is OK because UINT64_MAX > INT64_MAX - // Exponentiation is straight forward + /* Cast to unsigned is OK because of check for negative. + * Cast to unsigned is OK because UINT64_MAX > INT64_MAX. + * Exponentiation is straight forward + */ return (*pfExp)((uint64_t)nMantissa, nExponent, puResult); } -/* - Compute value with signed mantissa and unsigned result. Works with - exponent of 2 or 10 based on exponentiator. +/** + * @brief Exponentiate an usnigned mantissa and unsigned exponent to produce an unsigned result. + * + * @param[in] uMantissa Unsigned integer mantissa. + * @param[in] nExponent Unsigned integer exponent. + * @param[out] puResult Place to put the unsigned integer result. + * @param[in] pfExp Exponentiation function. + * + * @returns Error code + * + * \c pfExp performs exponentiation on and unsigned mantissa and + * produces an unsigned result so this is just a wrapper that does + * nothing (and is likely inlined). */ -static inline QCBORError ExponentitateUU(uint64_t uMantissa, - int64_t nExponent, - uint64_t *puResult, - fExponentiator pfExp) +static QCBORError +ExponentitateUU(uint64_t uMantissa, + int64_t nExponent, + uint64_t *puResult, + fExponentiator pfExp) { return (*pfExp)(uMantissa, nExponent, puResult); } @@ -4377,8 +4581,20 @@ static inline QCBORError ExponentitateUU(uint64_t uMantissa, - -static QCBORError ConvertBigNumToUnsigned(const UsefulBufC BigNum, uint64_t uMax, uint64_t *pResult) +/** + * @brief Convert a CBOR big number to a uint64_t. + * + * @param[in] BigNum Bytes of the big number to convert. + * @param[in] uMax Maximum value allowed for the result. + * @param[out] pResult Place to put the unsigned integer result. + * + * @returns Error code + * + * Many values will overflow because a big num can represent a much + * larger range than uint64_t. + */ +static QCBORError +ConvertBigNumToUnsigned(const UsefulBufC BigNum, const uint64_t uMax, uint64_t *pResult) { uint64_t uResult; @@ -4397,49 +4613,85 @@ static QCBORError ConvertBigNumToUnsigned(const UsefulBufC BigNum, uint64_t uMax } -static inline QCBORError ConvertPositiveBigNumToUnsigned(const UsefulBufC BigNum, uint64_t *pResult) +/** + * @brief Convert a CBOR postive big number to a uint64_t. + * + * @param[in] BigNum Bytes of the big number to convert. + * @param[out] pResult Place to put the unsigned integer result. + * + * @returns Error code + * + * Many values will overflow because a big num can represent a much + * larger range than uint64_t. + */ +static QCBORError +ConvertPositiveBigNumToUnsigned(const UsefulBufC BigNum, uint64_t *pResult) { return ConvertBigNumToUnsigned(BigNum, UINT64_MAX, pResult); } -static inline QCBORError ConvertPositiveBigNumToSigned(const UsefulBufC BigNum, int64_t *pResult) +/** + * @brief Convert a CBOR positive big number to an int64_t. + * + * @param[in] BigNum Bytes of the big number to convert. + * @param[out] pResult Place to put the signed integer result. + * + * @returns Error code + * + * Many values will overflow because a big num can represent a much + * larger range than int64_t. + */ +static QCBORError +ConvertPositiveBigNumToSigned(const UsefulBufC BigNum, int64_t *pResult) { uint64_t uResult; - QCBORError uError = ConvertBigNumToUnsigned(BigNum, INT64_MAX, &uResult); + QCBORError uError = ConvertBigNumToUnsigned(BigNum, INT64_MAX, &uResult); if(uError) { return uError; } - /* Cast is safe because ConvertBigNum is told to limit to INT64_MAX */ + /* Cast is safe because ConvertBigNumToUnsigned is told to limit to INT64_MAX */ *pResult = (int64_t)uResult; return QCBOR_SUCCESS; } -static inline QCBORError ConvertNegativeBigNumToSigned(const UsefulBufC BigNum, int64_t *pnResult) +/** + * @brief Convert a CBOR negative big number to an int64_t. + * + * @param[in] BigNum Bytes of the big number to convert. + * @param[out] pnResult Place to put the signed integer result. + * + * @returns Error code + * + * Many values will overflow because a big num can represent a much + * larger range than int64_t. + */ +static QCBORError +ConvertNegativeBigNumToSigned(const UsefulBufC BigNum, int64_t *pnResult) { uint64_t uResult; /* The negative integer furthest from zero for a C int64_t is - INT64_MIN which is expressed as -INT64_MAX - 1. The value of a - negative number in CBOR is computed as -n - 1 where n is the - encoded integer, where n is what is in the variable BigNum. When - converting BigNum to a uint64_t, the maximum value is thus - INT64_MAX, so that when it -n - 1 is applied to it the result will - never be further from 0 than INT64_MIN. - - -n - 1 <= INT64_MIN. - -n - 1 <= -INT64_MAX - 1 - n <= INT64_MAX. + * INT64_MIN which is expressed as -INT64_MAX - 1. The value of a + * negative number in CBOR is computed as -n - 1 where n is the + * encoded integer, where n is what is in the variable BigNum. When + * converting BigNum to a uint64_t, the maximum value is thus + * INT64_MAX, so that when it -n - 1 is applied to it the result + * will never be further from 0 than INT64_MIN. + * + * -n - 1 <= INT64_MIN. + * -n - 1 <= -INT64_MAX - 1 + * n <= INT64_MAX. */ QCBORError uError = ConvertBigNumToUnsigned(BigNum, INT64_MAX, &uResult); if(uError != QCBOR_SUCCESS) { return uError; } - /// Now apply -n - 1. The cast is safe because - // ConvertBigNumToUnsigned() is limited to INT64_MAX which does fit - // is the largest positive integer that an int64_t can - // represent. */ + /* Now apply -n - 1. The cast is safe because + * ConvertBigNumToUnsigned() is limited to INT64_MAX which does fit + * is the largest positive integer that an int64_t can + * represent. */ *pnResult = -(int64_t)uResult - 1; return QCBOR_SUCCESS; @@ -5469,38 +5721,105 @@ static inline UsefulBufC ConvertIntToBigNum(uint64_t uInt, UsefulBuf Buffer) } -static QCBORError MantissaAndExponentTypeHandler(QCBORDecodeContext *pMe, - TagSpecification TagSpec, - QCBORItem *pItem) +/** + * @brief Check and/or complete mantissa and exponent item. + * + * @param[in] pMe The decoder context + * @param[in] TagSpec Expected type(s) + * @param[in,out] pItem See below + * + * This is for decimal fractions and big floats, both of which are a + * mantissa and exponent. + * + * The input item is either a fully decoded decimal faction or big + * float, or a just the decoded first item of a decimal fraction or + * big float. + * + * On output, the item is always a fully decoded decimal fraction or + * big float. + * + * This errors out if the input type does not meet the TagSpec. + */ +// TODO: document and see tests for the bug that was fixed by this rewrite +static QCBORError +MantissaAndExponentTypeHandler(QCBORDecodeContext *pMe, + const TagSpecification TagSpec, + QCBORItem *pItem) { QCBORError uErr; - // Loops runs at most 1.5 times. Making it a loop saves object code. - while(1) { - uErr = CheckTagRequirement(TagSpec, pItem); - if(uErr != QCBOR_SUCCESS) { - goto Done; - } - if(pItem->uDataType != QCBOR_TYPE_ARRAY) { - break; // Successful exit. Moving on to finish decoding. - } + /* pItem could either be an auto-decoded mantissa and exponent or + * the opening array of an undecoded mantissa and exponent. This + * check will succeed on either, but doesn't say which it was. + */ + uErr = CheckTagRequirement(TagSpec, pItem); + if(uErr != QCBOR_SUCCESS) { + goto Done; + } - // The item is an array, which means an undecoded - // mantissa and exponent, so decode it. It will then - // have a different type and exit the loop if. + if(pItem->uDataType == QCBOR_TYPE_ARRAY) { + /* The item is an array, which means is is an undecoded mantissa + * and exponent. This call consumes the items in the array and + * results in a decoded mantissa and exponent in pItem. This is + * the case where there was no tag. + */ uErr = QCBORDecode_MantissaAndExponent(pMe, pItem); if(uErr != QCBOR_SUCCESS) { goto Done; } - // Second time around, the type must match. - TagSpec.uTagRequirement = QCBOR_TAG_REQUIREMENT_TAG; + /* The above decode didn't determine whether it is a decimal + * fraction or big num. Which of these two depends on what the + * caller wants it decoded as since there is no tag, so fish the + * type out of the TagSpec. */ + pItem->uDataType = MantissaExponentDataType(TagSpec.uTaggedTypes[0], pItem); + + /* No need to check the type again. All that we need to know was + * that it decoded correctly as a mantissa and exponent. The + * QCBOR type is set out by what was requested. + */ } + + /* If the item was not an array and the check passed, then + * it is a fully decoded big float or decimal fraction and + * matches what is requested. + */ + Done: return uErr; } +/* Some notes from the work to disable tags. + * + * The API for big floats and decimal fractions seems good. + * If there's any issue with it it's that the code size to + * implement is a bit large because of the conversion + * to/from int and bignum that is required. There is no API + * that doesn't do the conversion so dead stripping will never + * leave that code out. + * + * The implementation itself seems correct, but not as clean + * and neat as it could be. It could probably be smaller too. + * + * The implementation has three main parts / functions + * - The decoding of the array of two + * - All the tag and type checking for the various API functions + * - Conversion to/from bignum and int + * + * The type checking seems like it wastes the most code for + * what it needs to do. + * + * The inlining for the conversion is probably making the + * overall code base larger. + * + * The tests cases could be organized a lot better and be + * more thorough. + * + * Seems also like there could be more common code in the + * first tier part of the public API. Some functions only + * vary by a TagSpec. + */ static void ProcessMantissaAndExponent(QCBORDecodeContext *pMe, TagSpecification TagSpec, QCBORItem *pItem, @@ -5518,10 +5837,12 @@ static void ProcessMantissaAndExponent(QCBORDecodeContext *pMe, case QCBOR_TYPE_DECIMAL_FRACTION: case QCBOR_TYPE_BIGFLOAT: - *pnMantissa = pItem->val.expAndMantissa.Mantissa.nInt; *pnExponent = pItem->val.expAndMantissa.nExponent; + *pnMantissa = pItem->val.expAndMantissa.Mantissa.nInt; break; +#ifndef QCBOR_DISABLE_TAGS + /* If tags are disabled, mantissas can never be big nums */ case QCBOR_TYPE_DECIMAL_FRACTION_POS_BIGNUM: case QCBOR_TYPE_BIGFLOAT_POS_BIGNUM: *pnExponent = pItem->val.expAndMantissa.nExponent; @@ -5533,6 +5854,7 @@ static void ProcessMantissaAndExponent(QCBORDecodeContext *pMe, *pnExponent = pItem->val.expAndMantissa.nExponent; uErr = ConvertNegativeBigNumToSigned(pItem->val.expAndMantissa.Mantissa.bigNum, pnMantissa); break; +#endif /* QCBOR_DISABLE_TAGS */ default: uErr = QCBOR_ERR_UNEXPECTED_TYPE; @@ -5564,17 +5886,23 @@ static void ProcessMantissaAndExponentBig(QCBORDecodeContext *pMe, case QCBOR_TYPE_DECIMAL_FRACTION: case QCBOR_TYPE_BIGFLOAT: + /* See comments in ExponentiateNN() on handling INT64_MIN */ if(pItem->val.expAndMantissa.Mantissa.nInt >= 0) { uMantissa = (uint64_t)pItem->val.expAndMantissa.Mantissa.nInt; *pbIsNegative = false; - } else { + } else if(pItem->val.expAndMantissa.Mantissa.nInt != INT64_MIN) { uMantissa = (uint64_t)-pItem->val.expAndMantissa.Mantissa.nInt; *pbIsNegative = true; + } else { + uMantissa = (uint64_t)INT64_MAX+1; + *pbIsNegative = true; } *pMantissa = ConvertIntToBigNum(uMantissa, BufferForMantissa); *pnExponent = pItem->val.expAndMantissa.nExponent; break; +#ifndef QCBOR_DISABLE_TAGS + /* If tags are disabled, mantissas can never be big nums */ case QCBOR_TYPE_DECIMAL_FRACTION_POS_BIGNUM: case QCBOR_TYPE_BIGFLOAT_POS_BIGNUM: *pnExponent = pItem->val.expAndMantissa.nExponent; @@ -5588,6 +5916,7 @@ static void ProcessMantissaAndExponentBig(QCBORDecodeContext *pMe, *pMantissa = pItem->val.expAndMantissa.Mantissa.bigNum; *pbIsNegative = true; break; +#endif /* QCBOR_DISABLE_TAGS */ default: uErr = QCBOR_ERR_UNEXPECTED_TYPE; @@ -5690,7 +6019,7 @@ void QCBORDecode_GetDecimalFractionInMapSZ(QCBORDecodeContext *pMe, */ void QCBORDecode_GetDecimalFractionBig(QCBORDecodeContext *pMe, uint8_t uTagRequirement, - UsefulBuf MantissaBuffer, + UsefulBuf MantissaBuffer, UsefulBufC *pMantissa, bool *pbMantissaIsNegative, int64_t *pnExponent) diff --git a/3rdparty/exported/QCBOR/src/qcbor_encode.c b/3rdparty/exported/QCBOR/src/qcbor_encode.c index 26d86ce1e075..53df657cfbb7 100644 --- a/3rdparty/exported/QCBOR/src/qcbor_encode.c +++ b/3rdparty/exported/QCBOR/src/qcbor_encode.c @@ -658,8 +658,13 @@ void QCBOREncode_AddInt64(QCBOREncodeContext *me, int64_t nNum) uint64_t uValue; if(nNum < 0) { - /* In CBOR -1 encodes as 0x00 with major type negative int. */ - uValue = (uint64_t)(-nNum - 1); + /* In CBOR -1 encodes as 0x00 with major type negative int. + * First add one as a signed integer because that will not + * overflow. Then change the sign as needed for encoding. (The + * opposite order, changing the sign and subtracting, can cause + * an overflow when encoding INT64_MIN. */ + int64_t nTmp = nNum + 1; + uValue = (uint64_t)-nTmp; uMajorType = CBOR_MAJOR_TYPE_NEGATIVE_INT; } else { uValue = (uint64_t)nNum; diff --git a/3rdparty/exported/QCBOR/test/CMakeLists.txt b/3rdparty/exported/QCBOR/test/CMakeLists.txt new file mode 100644 index 000000000000..ab53f70dc0d4 --- /dev/null +++ b/3rdparty/exported/QCBOR/test/CMakeLists.txt @@ -0,0 +1,71 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2022-2023, Arm Limited. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# See BSD-3-Clause license in README.md +#------------------------------------------------------------------------------- + +cmake_minimum_required(VERSION 3.15) + +# Validate value of BUILD_QCBOR_TEST config option +if ((NOT BUILD_QCBOR_TEST STREQUAL "LIB") AND (NOT BUILD_QCBOR_TEST STREQUAL "APP")) + message(FATAL_ERROR "QCBOR | Invalid Config: BUILD_QCBOR_TEST=${BUILD_QCBOR_TEST}") +endif() + +add_library(qcbor_test STATIC) + +target_sources(qcbor_test + PRIVATE + float_tests.c + half_to_double_from_rfc7049.c + qcbor_decode_tests.c + qcbor_encode_tests.c + run_tests.c + UsefulBuf_Tests.c +) + +target_include_directories(qcbor_test + PUBLIC + . + PRIVATE + ../inc +) + +target_compile_definitions(qcbor_test + PUBLIC + $<$:QCBOR_DISABLE_FLOAT_HW_USE> + $<$:QCBOR_DISABLE_PREFERRED_FLOAT> + $<$:USEFULBUF_DISABLE_ALL_FLOAT> +) + +target_link_libraries(qcbor_test + PRIVATE + qcbor + # The math library is needed for floating-point support. + # To avoid need for it #define QCBOR_DISABLE_FLOAT_HW_USE + # Using GCC + $<$,$>>:m> +) + +if (BUILD_QCBOR_TEST STREQUAL "APP") + add_executable(qcbortest) + + target_sources(qcbortest + PRIVATE + ../cmd_line_main.c + ../example.c + ../ub-example.c + ) + + target_include_directories(qcbortest + PRIVATE + ../ + ) + + target_link_libraries(qcbortest + PRIVATE + qcbor + qcbor_test + ) +endif() diff --git a/3rdparty/exported/QCBOR/test/UsefulBuf_Tests.c b/3rdparty/exported/QCBOR/test/UsefulBuf_Tests.c index e729ff139ffb..e93a011b86f6 100644 --- a/3rdparty/exported/QCBOR/test/UsefulBuf_Tests.c +++ b/3rdparty/exported/QCBOR/test/UsefulBuf_Tests.c @@ -57,6 +57,12 @@ const char *AddStuffToUOB(UsefulOutBuf *pUOB) /* add a space to end */ UsefulOutBuf_AppendByte(pUOB, ' '); + /* Add an empty string */ + UsefulOutBuf_AppendUsefulBuf(pUOB, NULLUsefulBufC); + + /* Add a zero length string (valid pointer, 0 length) */ + UsefulOutBuf_AppendData(pUOB, "xxx", 0); + /* Add 6 bytes to the end */ UsefulBufC UBC = {"hunny ", 6}; UsefulOutBuf_AppendUsefulBuf(pUOB, UBC); diff --git a/3rdparty/exported/QCBOR/test/float_tests.c b/3rdparty/exported/QCBOR/test/float_tests.c index 3484084a18dd..2bf5fad311b7 100644 --- a/3rdparty/exported/QCBOR/test/float_tests.c +++ b/3rdparty/exported/QCBOR/test/float_tests.c @@ -127,7 +127,7 @@ inline static bool CheckDouble(double d, uint64_t u) } -int32_t HalfPrecisionDecodeBasicTests() +int32_t HalfPrecisionDecodeBasicTests(void) { UsefulBufC HalfPrecision = UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spExpectedHalf); @@ -247,7 +247,7 @@ int32_t HalfPrecisionDecodeBasicTests() -int32_t HalfPrecisionAgainstRFCCodeTest() +int32_t HalfPrecisionAgainstRFCCodeTest(void) { for(uint32_t uHalfP = 0; uHalfP < 0xffff; uHalfP += 60) { unsigned char x[2]; @@ -441,7 +441,7 @@ static const uint8_t spExpectedSmallest[] = { #define MAKE_DOUBLE(x) UsefulBufUtil_CopyUint64ToDouble(x) -int32_t DoubleAsSmallestTest() +int32_t DoubleAsSmallestTest(void) { UsefulBuf_MAKE_STACK_UB(EncodedHalfsMem, sizeof(spExpectedSmallest)); @@ -700,7 +700,7 @@ static const uint8_t spExpectedFloatsNoHalf[] = { 0x18, 0x6A, 0xFA, 0x00, 0x00, 0x00, 0x00}; -int32_t GeneralFloatEncodeTests() +int32_t GeneralFloatEncodeTests(void) { UsefulBufC ExpectedFloats; #ifndef QCBOR_DISABLE_PREFERRED_FLOAT @@ -774,7 +774,7 @@ static int CHECK_EXPECTED_DOUBLE(double val, double expected) #endif /* USEFULBUF_DISABLE_ALL_FLOAT */ -int32_t GeneralFloatDecodeTests() +int32_t GeneralFloatDecodeTests(void) { QCBORItem Item; QCBORError uErr; diff --git a/3rdparty/exported/QCBOR/test/not_well_formed_cbor.h b/3rdparty/exported/QCBOR/test/not_well_formed_cbor.h index 6734e9396dbf..e50588716720 100644 --- a/3rdparty/exported/QCBOR/test/not_well_formed_cbor.h +++ b/3rdparty/exported/QCBOR/test/not_well_formed_cbor.h @@ -15,7 +15,8 @@ #ifndef not_well_formed_cbor_h #define not_well_formed_cbor_h -#include // for size_t and uint8_t +#include // for size_t +#include // for uint8_t struct someBinaryBytes { @@ -50,8 +51,10 @@ static const struct someBinaryBytes paNotWellFormedCBOR[] = { {(uint8_t[]){0x5f, 0x80, 0xff}, 3}, // indefinite length byte string with an map chunk {(uint8_t[]){0x5f, 0xa0, 0xff}, 3}, +#ifndef QCBOR_DISABLE_TAGS // indefinite length byte string with tagged integer chunk {(uint8_t[]){0x5f, 0xc0, 0x00, 0xff}, 4}, +#endif /* QCBOR_DISABLE_TAGS */ // indefinite length byte string with an simple type chunk {(uint8_t[]){0x5f, 0xe0, 0xff}, 3}, // indefinite length byte string with indefinite string inside @@ -246,11 +249,12 @@ static const struct someBinaryBytes paNotWellFormedCBOR[] = { {(uint8_t[]){0x1f}, 1}, // Negative integer with "argument" an indefinite length {(uint8_t[]){0x3f}, 1}, +#ifndef QCBOR_DISABLE_TAGS // CBOR tag with "argument" an indefinite length {(uint8_t[]){0xdf, 0x00}, 2}, // CBOR tag with "argument" an indefinite length alternate vector {(uint8_t[]){0xdf}, 1}, - +#endif /* QCBOR_DISABLE_TAGS */ // Missing content bytes from a definite length string diff --git a/3rdparty/exported/QCBOR/test/qcbor_decode_tests.c b/3rdparty/exported/QCBOR/test/qcbor_decode_tests.c index 431836cdca30..e913854d7a3d 100644 --- a/3rdparty/exported/QCBOR/test/qcbor_decode_tests.c +++ b/3rdparty/exported/QCBOR/test/qcbor_decode_tests.c @@ -484,7 +484,7 @@ static const uint8_t spTooSmallNegative[] = { Tests the decoding of lots of different integers sizes and values. */ -int32_t IntegerValuesParseTest() +int32_t IntegerValuesParseTest(void) { int nReturn; QCBORDecodeContext DCtx; @@ -691,7 +691,7 @@ static int32_t ParseOrderedArray(const uint8_t *pEncoded, -int32_t SimpleArrayTest() +int32_t SimpleArrayTest(void) { uint8_t *pEncoded; size_t nEncodedLen; @@ -898,7 +898,7 @@ static int32_t CheckEmpties(UsefulBufC input, bool bCheckCounts) } -int32_t EmptyMapsAndArraysTest() +int32_t EmptyMapsAndArraysTest(void) { int nResult; nResult = CheckEmpties(UsefulBuf_FROM_BYTE_ARRAY_LITERAL(sEmpties), @@ -966,7 +966,7 @@ static const uint8_t spDeepArrays[] = { 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x80}; -int32_t ParseDeepArrayTest() +int32_t ParseDeepArrayTest(void) { QCBORDecodeContext DCtx; int nReturn = 0; @@ -999,7 +999,7 @@ static const uint8_t spTooDeepArrays[] = { 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x80}; -int32_t ParseTooDeepArrayTest() +int32_t ParseTooDeepArrayTest(void) { QCBORDecodeContext DCtx; int nReturn = 0; @@ -1030,7 +1030,7 @@ int32_t ParseTooDeepArrayTest() -int32_t ShortBufferParseTest() +int32_t ShortBufferParseTest(void) { int nResult = 0; @@ -1054,7 +1054,7 @@ int32_t ShortBufferParseTest() -int32_t ShortBufferParseTest2() +int32_t ShortBufferParseTest2(void) { uint8_t *pEncoded; int nReturn; @@ -1214,7 +1214,7 @@ static int32_t ParseMapTest1(QCBORDecodeMode nMode) Decode and thoroughly check a moderately complex set of maps in the QCBOR_DECODE_MODE_MAP_AS_ARRAY mode. */ -int32_t ParseMapAsArrayTest() +int32_t ParseMapAsArrayTest(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -1637,7 +1637,7 @@ static int32_t ExtraBytesTest(int nLevel) -int32_t ParseMapTest() +int32_t ParseMapTest(void) { // Parse a moderatly complex map structure very thoroughly int32_t nResult = ParseMapTest1(QCBOR_DECODE_MODE_NORMAL); @@ -1672,7 +1672,7 @@ static const uint8_t spSimpleValues[] = { 0xf8, 0x00, 0xf8, 0x13, 0xf8, 0x1f, 0xf8, 0x20, 0xf8, 0xff}; -int32_t ParseSimpleTest() +int32_t ParseSimpleTest(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -1748,7 +1748,7 @@ int32_t ParseSimpleTest() } -int32_t NotWellFormedTests() +int32_t NotWellFormedTests(void) { // Loop over all the not-well-formed instance of CBOR // that are test vectors in not_well_formed_cbor.h @@ -1758,6 +1758,10 @@ int32_t NotWellFormedTests() const struct someBinaryBytes *pBytes = &paNotWellFormedCBOR[nIterate]; const UsefulBufC Input = (UsefulBufC){pBytes->p, pBytes->n}; + if(nIterate == 86) { + nIterate = 86; + } + // Set up decoder context. String allocator needed for indefinite // string test cases QCBORDecodeContext DCtx; @@ -1814,6 +1818,11 @@ static int32_t ProcessFailures(const struct FailInput *pFailInputs, size_t nNumF } #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ + const size_t nIndexx = (size_t)(pF - pFailInputs); + if(nIndexx == 8) { + uCBORError = 9; + } + // Iterate until there is an error of some sort error QCBORItem Item; @@ -1867,8 +1876,13 @@ static const struct FailInput Failures[] = { { {(uint8_t[]){0x5f, 0x80, 0xff}, 3}, QCBOR_ERR_INDEFINITE_STRING_CHUNK }, // indefinite length byte string with an map chunk { {(uint8_t[]){0x5f, 0xa0, 0xff}, 3}, QCBOR_ERR_INDEFINITE_STRING_CHUNK }, +#ifndef QCBOR_DISABLE_TAGS // indefinite length byte string with tagged integer chunk { {(uint8_t[]){0x5f, 0xc0, 0x00, 0xff}, 4}, QCBOR_ERR_INDEFINITE_STRING_CHUNK }, +#else + // indefinite length byte string with tagged integer chunk + { {(uint8_t[]){0x5f, 0xc0, 0x00, 0xff}, 4}, QCBOR_ERR_TAGS_DISABLED }, +#endif /* QCBOR_DISABLE_TAGS */ // indefinite length byte string with an simple type chunk { {(uint8_t[]){0x5f, 0xe0, 0xff}, 3}, QCBOR_ERR_INDEFINITE_STRING_CHUNK }, { {(uint8_t[]){0x5f, 0x5f, 0x41, 0x00, 0xff, 0xff}, 6}, QCBOR_ERR_INDEFINITE_STRING_CHUNK}, @@ -1999,10 +2013,12 @@ static const struct FailInput Failures[] = { // double-precision with 3 byte argument { {(uint8_t[]){0xfb, 0x00, 0x00, 0x00}, 4}, QCBOR_ERR_HIT_END }, - +#ifndef QCBOR_DISABLE_TAGS // Tag with no content { {(uint8_t[]){0xc0}, 1}, QCBOR_ERR_HIT_END }, - +#else /* QCBOR_DISABLE_TAGS */ + { {(uint8_t[]){0xc0}, 1}, QCBOR_ERR_TAGS_DISABLED }, +#endif /* QCBOR_DISABLE_TAGS */ // Breaks must not occur in definite length arrays and maps // Array of length 1 with sole member replaced by a break @@ -2094,11 +2110,16 @@ static const struct FailInput Failures[] = { { {(uint8_t[]){0x1f}, 1}, QCBOR_ERR_BAD_INT }, // Negative integer with additional info indefinite length { {(uint8_t[]){0x3f}, 1}, QCBOR_ERR_BAD_INT }, + +#ifndef QCBOR_DISABLE_TAGS // CBOR tag with "argument" an indefinite length { {(uint8_t[]){0xdf, 0x00}, 2}, QCBOR_ERR_BAD_INT }, // CBOR tag with "argument" an indefinite length alternate vector { {(uint8_t[]){0xdf}, 1}, QCBOR_ERR_BAD_INT }, - +#else /* QCBOR_DISABLE_TAGS */ + { {(uint8_t[]){0xdf, 0x00}, 2}, QCBOR_ERR_TAGS_DISABLED }, + { {(uint8_t[]){0xdf}, 1}, QCBOR_ERR_TAGS_DISABLED }, +#endif /* QCBOR_DISABLE_TAGS */ // Missing bytes from a deterministic length string // A byte string is of length 1 without the 1 byte @@ -2191,7 +2212,7 @@ static const struct FailInput Failures[] = { { {(uint8_t[]){0xbf, 0x00, 0x00, 0x00, 0xff}, 5}, QCBOR_ERR_BAD_BREAK }, #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_ARRAYS */ - +#ifndef QCBOR_DISABLE_TAGS // In addition to not-well-formed, some invalid CBOR // Text-based date, with an integer { {(uint8_t[]){0xc0, 0x00}, 2}, QCBOR_ERR_BAD_OPT_TAG }, @@ -2201,9 +2222,20 @@ static const struct FailInput Failures[] = { { {(uint8_t[]){0xc1, 0xc0, 0x00}, 3}, QCBOR_ERR_BAD_OPT_TAG }, // big num tagged an int, not a byte string { {(uint8_t[]){0xc2, 0x00}, 2}, QCBOR_ERR_BAD_OPT_TAG }, +#else /* QCBOR_DISABLE_TAGS */ + // Text-based date, with an integer + { {(uint8_t[]){0xc0, 0x00}, 2}, QCBOR_ERR_TAGS_DISABLED }, + // Epoch date, with an byte string + { {(uint8_t[]){0xc1, 0x41, 0x33}, 3}, QCBOR_ERR_TAGS_DISABLED }, + // tagged as both epoch and string dates + { {(uint8_t[]){0xc1, 0xc0, 0x00}, 3}, QCBOR_ERR_TAGS_DISABLED }, + // big num tagged an int, not a byte string + { {(uint8_t[]){0xc2, 0x00}, 2}, QCBOR_ERR_TAGS_DISABLED }, +#endif /* QCBOR_DISABLE_TAGS */ + }; -int32_t DecodeFailureTests() +int32_t DecodeFailureTests(void) { int32_t nResult; @@ -2303,7 +2335,7 @@ static void ComprehensiveInputRecurser(uint8_t *pBuf, size_t nLen, size_t nLenMa } -int32_t ComprehensiveInputTest() +int32_t ComprehensiveInputTest(void) { // Size 2 tests 64K inputs and runs quickly uint8_t pBuf[2]; @@ -2314,7 +2346,7 @@ int32_t ComprehensiveInputTest() } -int32_t BigComprehensiveInputTest() +int32_t BigComprehensiveInputTest(void) { // size 3 tests 16 million inputs and runs OK // in seconds on fast machines. Size 4 takes @@ -2401,8 +2433,8 @@ static int CHECK_EXPECTED_DOUBLE(double val, double expected) { #endif /* QCBOR_DISABLE_FLOAT_HW_USE */ - -int32_t DateParseTest() +/* Test date decoding using GetNext() */ +int32_t DateParseTest(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -2532,6 +2564,7 @@ int32_t DateParseTest() return 0; } + /* Test cases covered here. Some items cover more than one of these. positive integer (zero counts as a positive integer) @@ -2561,76 +2594,82 @@ int32_t DateParseTest() Untagged values */ static const uint8_t spSpiffyDateTestInput[] = { - 0x86, // array of 6 items + 0x87, // array of 7 items - 0xc1, - 0xfb, 0xc3, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // -9.2233720368547748E+18, too negative + 0xa6, // Open a map for tests involving untagged items with labels. - 0xc1, // tag for epoch date - 0x1b, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, // Too-large integer + // Untagged integer 0 + 0x08, + 0x00, - 0xc1, // tag for epoch date - 0xf9, 0xfc, 0x00, // Half-precision -Infinity + // Utagged date string with string label y + 0x61, 0x79, + 0x6a, '2','0','8','5','-','0','4','-','1','2', // Untagged date string - 0xad, // Open a map for tests involving labels. + // Untagged single-precision float with value 3.14 with string label x + 0x61, 0x78, + 0xFA, 0x40, 0x48, 0xF5, 0xC3, + + // Untagged half-precision float with value -2 + 0x09, + 0xF9, 0xC0, 0x00, + + /* Untagged date-only date string */ + 0x18, 0x63, + 0x6A, 0x31, 0x39, 0x38, 0x35, 0x2D, 0x30, 0x34, 0x2D, 0x31, 0x32, /* "1985-04-12" */ + + /* Untagged days-count epoch date */ + 0x11, + 0x19, 0x0F, 0x9A, /* 3994 */ + + // End of map, back to array + + 0xa7, // Open map of tagged items with labels 0x00, 0xc0, // tag for string date 0x6a, '1','9','8','5','-','0','4','-','1','2', // Tagged date string + 0x01, 0xda, 0x03, 0x03, 0x03, 0x03, // An additional tag 0xc1, // tag for epoch date 0x1a, 0x53, 0x72, 0x4E, 0x00, // Epoch date 1400000000; Tue, 13 May 2014 16:53:20 GMT - // Untagged integer 0 - 0x08, - 0x00, - - // Utagged date string with string label y - 0x61, 0x79, - 0x6a, '2','0','8','5','-','0','4','-','1','2', // Untagged date string + 0x05, + 0xc1, + 0xfb, 0xc3, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, // -9223372036854773760 largest negative - // Untagged -1000 with label z - 0x61, 0x7a, - 0xda, 0x01, 0x01, 0x01, 0x01, // An additional tag - 0x39, 0x03, 0xe7, 0x07, 0xc1, // tag for epoch date 0xfb, 0x43, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, // 9223372036854773760 largest supported - 0x05, - 0xc1, - 0xfb, 0xc3, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, // -9223372036854773760 largest negative - - // Untagged single-precision float with value 3.14 with string label x - 0x61, 0x78, - 0xFA, 0x40, 0x48, 0xF5, 0xC3, + /* Tagged days-count epoch date */ + 0x63, 0x53, 0x44, 0x45, + 0xD8, 0x64, /* tag(100) */ + 0x39, 0x29, 0xB3, /* -10676 */ - // Untagged half-precision float with value -2 - 0x09, - 0xF9, 0xC0, 0x00, + // Untagged -1000 with label z + 0x61, 0x7a, + 0xda, 0x01, 0x01, 0x01, 0x01, // An additional tag + 0x39, 0x03, 0xe7, /* Tagged date-only date string */ 0x63, 0x53, 0x44, 0x53, 0xD9, 0x03, 0xEC, 0x6A, 0x31, 0x39, 0x38, 0x35, 0x2D, 0x30, 0x34, 0x2D, 0x31, 0x32, /* "1985-04-12" */ - /* Untagged date-only date string */ - 0x18, 0x63, - 0x6A, 0x31, 0x39, 0x38, 0x35, 0x2D, 0x30, 0x34, 0x2D, 0x31, 0x32, /* "1985-04-12" */ + // End of map of tagged items - /* Tagged days-count epoch date */ - 0x63, 0x53, 0x44, 0x45, - 0xD8, 0x64, /* tag(100) */ - 0x39, 0x29, 0xB3, /* -10676 */ + 0xc1, + 0xfb, 0xc3, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // -9.2233720368547748E+18, too negative - /* Untagged days-count epoch date */ - 0x11, - 0x19, 0x0F, 0x9A, /* 3994 */ + 0xc1, // tag for epoch date + 0x1b, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, // Too-large integer - // End of map, back to array + 0xc1, // tag for epoch date + 0xf9, 0xfc, 0x00, // Half-precision -Infinity // These two at the end because they are unrecoverable errors 0xc1, // tag for epoch date @@ -2638,78 +2677,28 @@ static const uint8_t spSpiffyDateTestInput[] = { 0xc0, // tag for string date 0xa0 // Erroneous empty map as content for date - }; -int32_t SpiffyDateDecodeTest() +int32_t SpiffyDateDecodeTest(void) { QCBORDecodeContext DC; QCBORError uError; - int64_t nEpochDate2, nEpochDate3, nEpochDate5, - nEpochDate4, nEpochDate6, nEpochDateFail, - nEpochDate1400000000, nEpochDays1, nEpochDays2; - UsefulBufC StringDate1, StringDate2, StringDays1, StringDays2; - uint64_t uTag1, uTag2; + int64_t nEpochDate3, nEpochDate5, + nEpochDate4, nEpochDate6, + nEpochDays2; + UsefulBufC StringDate1, StringDate2, StringDays2; QCBORDecode_Init(&DC, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spSpiffyDateTestInput), QCBOR_DECODE_MODE_NORMAL); - QCBORDecode_EnterArray(&DC, NULL); - - // Too-negative float, -9.2233720368547748E+18 - QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); - uError = QCBORDecode_GetAndResetError(&DC); - if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_DATE_OVERFLOW)) { - return 1111; - } - - // Too-large integer - QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); - uError = QCBORDecode_GetAndResetError(&DC); - if(uError != QCBOR_ERR_DATE_OVERFLOW) { - return 1; - } - - // Half-precision minus infinity - QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); - uError = QCBORDecode_GetAndResetError(&DC); - if(uError != FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_ERR_DATE_OVERFLOW)) { - return 2; - } - + /* Items are in an array or map to test look up by label and other + * that might not occur in isolated items. But it does make the + * test a bit messy. */ + QCBORDecode_EnterArray(&DC, NULL); QCBORDecode_EnterMap(&DC, NULL); - // Get largest negative double precision epoch date allowed - QCBORDecode_GetEpochDateInMapN(&DC, - 5, - QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG | - QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS, - &nEpochDate2); - uError = QCBORDecode_GetAndResetError(&DC); - if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS)) { - return 102; - } - if(uError == QCBOR_SUCCESS) { - if(nEpochDate2 != -9223372036854773760LL) { - return 101; - } - } - - // Get largest double precision epoch date allowed - QCBORDecode_GetEpochDateInMapN(&DC, 7, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, - &nEpochDate2); - uError = QCBORDecode_GetAndResetError(&DC); - if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS)) { - return 112; - } - if(uError == QCBOR_SUCCESS) { - if(nEpochDate2 != 9223372036854773760ULL) { - return 111; - } - } - // A single-precision date QCBORDecode_GetEpochDateInMapSZ(&DC, "x", QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, &nEpochDate5); @@ -2772,6 +2761,39 @@ int32_t SpiffyDateDecodeTest() // The rest of these succeed even if float features are disabled + + // Untagged integer 0 + QCBORDecode_GetEpochDateInMapN(&DC, 8, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, + &nEpochDate3); + // Untagged date string + QCBORDecode_GetDateStringInMapSZ(&DC, "y", QCBOR_TAG_REQUIREMENT_NOT_A_TAG, + &StringDate2); + + QCBORDecode_GetDaysStringInMapN(&DC, 99, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, + &StringDays2); + + QCBORDecode_GetEpochDaysInMapN(&DC, 17, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, + &nEpochDays2); + + QCBORDecode_ExitMap(&DC); + if(QCBORDecode_GetError(&DC) != QCBOR_SUCCESS) { + return 3001; + } + + // The map of tagged items + QCBORDecode_EnterMap(&DC, NULL); + +#ifndef QCBOR_DISABLE_TAGS + int64_t nEpochDate2, + nEpochDateFail, + nEpochDate1400000000, nEpochDays1; + UsefulBufC StringDays1; + uint64_t uTag1, uTag2; + + // Tagged date string + QCBORDecode_GetDateStringInMapN(&DC, 0, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + &StringDate1); + // Epoch date 1400000000; Tue, 13 May 2014 16:53:20 GMT QCBORDecode_GetEpochDateInMapN(&DC, 1, @@ -2779,15 +2801,23 @@ int32_t SpiffyDateDecodeTest() QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS, &nEpochDate1400000000); uTag1 = QCBORDecode_GetNthTagOfLast(&DC, 0); - // Tagged date string - QCBORDecode_GetDateStringInMapN(&DC, 0, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, - &StringDate1); - // Untagged integer 0 - QCBORDecode_GetEpochDateInMapN(&DC, 8, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, - &nEpochDate3); - // Untagged date string - QCBORDecode_GetDateStringInMapSZ(&DC, "y", QCBOR_TAG_REQUIREMENT_NOT_A_TAG, - &StringDate2); + + // Get largest negative double precision epoch date allowed + QCBORDecode_GetEpochDateInMapN(&DC, + 5, + QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG | + QCBOR_TAG_REQUIREMENT_ALLOW_ADDITIONAL_TAGS, + &nEpochDate2); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS)) { + return 102; + } + if(uError == QCBOR_SUCCESS) { + if(nEpochDate2 != -9223372036854773760LL) { + return 101; + } + } + // Untagged -1000 with label z QCBORDecode_GetEpochDateInMapSZ(&DC, "z", @@ -2796,6 +2826,20 @@ int32_t SpiffyDateDecodeTest() &nEpochDate6); uTag2 = QCBORDecode_GetNthTagOfLast(&DC, 0); + + // Get largest double precision epoch date allowed + QCBORDecode_GetEpochDateInMapN(&DC, 7, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + &nEpochDate2); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS)) { + return 112; + } + if(uError == QCBOR_SUCCESS) { + if(nEpochDate2 != 9223372036854773760ULL) { + return 111; + } + } + /* The days format is much simpler than the date format * because it can't be a floating point value. The test * of the spiffy decode functions sufficiently covers @@ -2807,20 +2851,36 @@ int32_t SpiffyDateDecodeTest() QCBORDecode_GetDaysStringInMapSZ(&DC, "SDS", QCBOR_TAG_REQUIREMENT_TAG, &StringDays1); - QCBORDecode_GetDaysStringInMapN(&DC, 99, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, - &StringDays2); - QCBORDecode_GetEpochDaysInMapSZ(&DC, "SDE", QCBOR_TAG_REQUIREMENT_TAG, &nEpochDays1); - QCBORDecode_GetEpochDaysInMapN(&DC, 17, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, - &nEpochDays2); - QCBORDecode_ExitMap(&DC); if(QCBORDecode_GetError(&DC) != QCBOR_SUCCESS) { return 3001; } + // Too-negative float, -9.2233720368547748E+18 + QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_DATE_OVERFLOW)) { + return 1111; + } + + // Too-large integer + QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != QCBOR_ERR_DATE_OVERFLOW) { + return 1; + } + + // Half-precision minus infinity + QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_ERR_DATE_OVERFLOW)) { + return 2; + } + + // Bad content for epoch date QCBORDecode_GetEpochDate(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nEpochDateFail); uError = QCBORDecode_GetAndResetError(&DC); @@ -2840,6 +2900,17 @@ int32_t SpiffyDateDecodeTest() if(uError != QCBOR_ERR_UNRECOVERABLE_TAG_CONTENT) { return 1000 + (int32_t)uError; } +#else /* QCBOR_DISABLE_TAGS */ + QCBORDecode_GetDateStringInMapN(&DC, 0, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + &StringDate1); + uError = QCBORDecode_GetAndResetError(&DC); + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return 4; + } +#endif /* QCBOR_DISABLE_TAGS */ + + +#ifndef QCBOR_DISABLE_TAGS if(nEpochDate1400000000 != 1400000000) { return 200; @@ -2849,36 +2920,38 @@ int32_t SpiffyDateDecodeTest() return 201; } - if(nEpochDate3 != 0) { - return 202; + if(nEpochDays1 != -10676) { + return 205; } - if(nEpochDate6 != -1000) { - return 203; + if(UsefulBuf_Compare(StringDays1, UsefulBuf_FromSZ("1985-04-12"))) { + return 207; } if(uTag2 != 0x01010101) { return 204; } - if(nEpochDays1 != -10676) { - return 205; - } - - if(nEpochDays2 != 3994) { - return 206; + if(nEpochDate6 != -1000) { + return 203; } if(UsefulBuf_Compare(StringDate1, UsefulBuf_FromSZ("1985-04-12"))) { return 205; } - if(UsefulBuf_Compare(StringDate2, UsefulBuf_FromSZ("2085-04-12"))) { +#endif /* QCBOR_DISABLE_TAGS */ + + if(nEpochDate3 != 0) { + return 202; + } + + if(nEpochDays2 != 3994) { return 206; } - if(UsefulBuf_Compare(StringDays1, UsefulBuf_FromSZ("1985-04-12"))) { - return 207; + if(UsefulBuf_Compare(StringDate2, UsefulBuf_FromSZ("2085-04-12"))) { + return 206; } if(UsefulBuf_Compare(StringDays2, UsefulBuf_FromSZ("1985-04-12"))) { @@ -2889,7 +2962,6 @@ int32_t SpiffyDateDecodeTest() } - // Input for one of the tagging tests static const uint8_t spTagInput[] = { 0xd9, 0xd9, 0xf7, // CBOR magic number @@ -3050,7 +3122,7 @@ static const uint8_t spSpiffyTagInput[] = { static int32_t CheckCSRMaps(QCBORDecodeContext *pDC); -int32_t OptTagParseTest() +int32_t OptTagParseTest(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -3530,8 +3602,19 @@ int32_t OptTagParseTest() return 0; } - - +/* + * These are showing the big numbers converted to integers. + * The tag numbers are not shown. + * + * [ 18446744073709551616, + * -18446744073709551617, + * {"BN+": 18446744073709551616, + * 64: 18446744073709551616, + * "BN-": -18446744073709551617, + * -64: -18446744073709551617 + * } + * ] + */ static const uint8_t spBigNumInput[] = { 0x83, @@ -3547,13 +3630,15 @@ static const uint8_t spBigNumInput[] = { 0x38, 0x3F, 0xC3, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#ifndef QCBOR_DISABLE_TAGS /* The expected big num */ static const uint8_t spBigNum[] = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#endif /* QCBOR_DISABLE_TAGS */ -int32_t BignumParseTest() +int32_t BignumParseTest(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -3571,6 +3656,7 @@ int32_t BignumParseTest() return -2; } +#ifndef QCBOR_DISABLE_TAGS // if((nCBORError = QCBORDecode_GetNext(&DCtx, &Item))) return -3; @@ -3627,6 +3713,12 @@ int32_t BignumParseTest() UsefulBuf_Compare(Item.val.bigNum, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spBigNum))){ return -16; } +#else + + if(QCBORDecode_GetNext(&DCtx, &Item) != QCBOR_ERR_TAGS_DISABLED) { + return -100; + } +#endif /* QCBOR_DISABLE_TAGS */ return 0; } @@ -3743,7 +3835,7 @@ static const uint8_t spCSRInputIndefLen[] = { 0x35, 0xbf, 0x24, 0x22, 0xff, 0xff}; -int32_t NestedMapTest() +int32_t NestedMapTest(void) { QCBORDecodeContext DCtx; @@ -3756,7 +3848,7 @@ int32_t NestedMapTest() -int32_t StringDecoderModeFailTest() +int32_t StringDecoderModeFailTest(void) { QCBORDecodeContext DCtx; @@ -3784,7 +3876,7 @@ int32_t StringDecoderModeFailTest() -int32_t NestedMapTestIndefLen() +int32_t NestedMapTestIndefLen(void) { QCBORDecodeContext DCtx; @@ -3848,7 +3940,7 @@ static int32_t parse_indeflen_nested(UsefulBufC Nested, int nNestLevel) } -int32_t IndefiniteLengthNestTest() +int32_t IndefiniteLengthNestTest(void) { UsefulBuf_MAKE_STACK_UB(Storage, 50); int i; @@ -3875,7 +3967,7 @@ static const uint8_t spIndefiniteArrayBad4[] = {0x81, 0x9f}; // confused tag static const uint8_t spIndefiniteArrayBad5[] = {0x9f, 0xd1, 0xff}; -int32_t IndefiniteLengthArrayMapTest() +int32_t IndefiniteLengthArrayMapTest(void) { QCBORError nResult; // --- first test ----- @@ -4015,6 +4107,8 @@ int32_t IndefiniteLengthArrayMapTest() QCBORDecode_Init(&DC, IndefLen, QCBOR_DECODE_MODE_NORMAL); nResult = QCBORDecode_GetNext(&DC, &Item); + +#ifndef QCBOR_DISABLE_TAGS if(nResult || Item.uDataType != QCBOR_TYPE_ARRAY) { return -18; } @@ -4023,6 +4117,11 @@ int32_t IndefiniteLengthArrayMapTest() if(nResult != QCBOR_ERR_BAD_BREAK) { return -19; } +#else /* QCBOR_DISABLE_TAGS */ + if(nResult != QCBOR_ERR_TAGS_DISABLED) { + return -20; + } +#endif /* QCBOR_DISABLE_TAGS */ return 0; } @@ -4118,7 +4217,7 @@ static int CheckBigString(UsefulBufC BigString) } -int32_t IndefiniteLengthStringTest() +int32_t IndefiniteLengthStringTest(void) { QCBORDecodeContext DC; QCBORItem Item; @@ -4307,7 +4406,7 @@ int32_t IndefiniteLengthStringTest() } -int32_t AllocAllStringsTest() +int32_t AllocAllStringsTest(void) { QCBORDecodeContext DC; QCBORError nCBORError; @@ -4549,155 +4648,534 @@ int32_t SetUpAllocatorTest(void) #ifndef QCBOR_DISABLE_EXP_AND_MANTISSA -/* exponent, mantissa - [ - 4([-1, 3]), - 4([-20, 4759477275222530853136]), - 4([9223372036854775807, -4759477275222530853137]), - 5([300, 100]), - 5([-20, 4759477275222530853136]), - 5([-9223372036854775807, -4759477275222530853137]) - 5([ 9223372036854775806, -4759477275222530853137]) - 5([ 9223372036854775806, 9223372036854775806])] - ] - */ -static const uint8_t spExpectedExponentsAndMantissas[] = { - 0x88, - 0xC4, 0x82, 0x20, - 0x03, - 0xC4, 0x82, 0x33, - 0xC2, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, - 0xC4, 0x82, 0x1B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xC3, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, - 0xC5, 0x82, 0x19, 0x01, 0x2C, - 0x18, 0x64, - 0xC5, 0x82, 0x33, - 0xC2, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, - 0xC5, 0x82, 0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, - 0xC3, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, - 0xC5, 0x82, 0x1B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, - 0xC3, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, - 0xC5, 0x82, 0x1B, 0x7f, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, - 0x1B, 0x7f, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE +struct EaMTest { + const char *szName; + UsefulBufC Input; + uint8_t uTagRequirement; + bool bHasTags; + + /* Expected values for GetNext */ + QCBORError uExpectedErrorGN; + uint8_t uQCBORTypeGN; + int64_t nExponentGN; + int64_t nMantissaGN; + UsefulBufC MantissaGN; + + /* Expected values for GetDecimalFraction */ + QCBORError uExpectedErrorGDF; + int64_t nExponentGDF; + int64_t nMantissaGDF; + + /* Expected values for GetDecimalFractionBig */ + QCBORError uExpectedErrorGDFB; + int64_t nExponentGDFB; + UsefulBufC MantissaGDFB; + bool IsNegativeGDFB; + + /* Expected values for GetBigFloat */ + QCBORError uExpectedErrorGBF; + int64_t nExponentGBF; + int64_t nMantissaGBF; + + /* Expected values for GetBigFloatBig */ + QCBORError uExpectedErrorGBFB; + int64_t nExponentGBFB; + UsefulBufC MantissaGBFB; + bool IsNegativeGBFB; }; -int32_t ExponentAndMantissaDecodeTests(void) -{ - QCBORDecodeContext DC; - QCBORError uErr; - QCBORItem item; - static const uint8_t spBigNumMantissa[] = {0x01, 0x02, 0x03, 0x04, 0x05, - 0x06, 0x07, 0x08, 0x09, 0x010}; - UsefulBufC BN = UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spBigNumMantissa); +static const struct EaMTest pEaMTests[] = { + { + "1. Untagged pair (big float or decimal fraction), no tag required", + {(const uint8_t []){0x82, 0x20, 0x03}, 3}, + QCBOR_TAG_REQUIREMENT_NOT_A_TAG, + false, + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_ARRAY, + 0, + 0, + {(const uint8_t []){0x00}, 1}, - QCBORDecode_Init(&DC, - UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spExpectedExponentsAndMantissas), - QCBOR_DECODE_MODE_NORMAL); + QCBOR_SUCCESS, /* GetDecimalFraction */ + -1, + 3, - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 1; - } + QCBOR_SUCCESS, /* for GetDecimalFractionBig */ + -1, + {(const uint8_t []){0x03}, 1}, + false, - if(item.uDataType != QCBOR_TYPE_ARRAY) { - return 2; - } + QCBOR_SUCCESS, /* for GetBigFloat */ + -1, + 3, - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 3; - } + QCBOR_SUCCESS, /* for GetBigFloatBig */ + -1, + {(const uint8_t []){0x03}, 1}, + false + }, - if(item.uDataType != QCBOR_TYPE_DECIMAL_FRACTION || - item.val.expAndMantissa.Mantissa.nInt != 3 || - item.val.expAndMantissa.nExponent != -1) { - return 4; - } + { + "2. Untagged pair (big float or decimal fraction), tag required", + {(const uint8_t []){0x82, 0x20, 0x03}, 3}, + QCBOR_TAG_REQUIREMENT_TAG, + false, - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 5; - } + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_ARRAY, + 0, + 0, + {(const uint8_t []){0x00}, 1}, - if(item.uDataType != QCBOR_TYPE_DECIMAL_FRACTION_POS_BIGNUM || - item.val.expAndMantissa.nExponent != -20 || - UsefulBuf_Compare(item.val.expAndMantissa.Mantissa.bigNum, BN)) { - return 6; - } + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFraction */ + 0, + 0, - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 7; - } + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFractionBig */ + 0, + {(const uint8_t []){0x00}, 1}, + false, - if(item.uDataType != QCBOR_TYPE_DECIMAL_FRACTION_NEG_BIGNUM || - item.val.expAndMantissa.nExponent != 9223372036854775807 || - UsefulBuf_Compare(item.val.expAndMantissa.Mantissa.bigNum, BN)) { - return 8; - } + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloat */ + 0, + 0, - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 9; - } + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloatBig */ + 0, + {(const uint8_t []){0x00}, 1}, + false - if(item.uDataType != QCBOR_TYPE_BIGFLOAT || - item.val.expAndMantissa.Mantissa.nInt != 100 || - item.val.expAndMantissa.nExponent != 300) { - return 10; - } + }, - // 5([-20, 4759477275222530853136]), - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 11; - } - if(item.uDataType != QCBOR_TYPE_BIGFLOAT_POS_BIGNUM || - item.val.expAndMantissa.nExponent != -20 || - UsefulBuf_Compare(item.val.expAndMantissa.Mantissa.bigNum, BN)) { - return 12; - } + { + "3. Tagged 1.5 decimal fraction, tag 4 optional", + {(const uint8_t []){0xC4, 0x82, 0x20, 0x03}, 4}, + QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + true, - // 5([-9223372036854775807, -4759477275222530853137]) - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 13; - } - if(item.uDataType != QCBOR_TYPE_BIGFLOAT_NEG_BIGNUM || - item.val.expAndMantissa.nExponent != -9223372036854775807 || - UsefulBuf_Compare(item.val.expAndMantissa.Mantissa.bigNum, BN)) { - return 14; + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_DECIMAL_FRACTION, + -1, + 3, + {(const uint8_t []){0x00}, 1}, + + + QCBOR_SUCCESS, /* for GetDecimalFraction */ + -1, + 3, + + QCBOR_SUCCESS, /* for GetDecimalFractionBig */ + -1, + {(const uint8_t []){0x03}, 1}, + false, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloat */ + 0, + 0, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloatBig */ + 0, + {(const uint8_t []){0x00}, 1}, + false + }, + { + "4. Tagged 100 * 2^300 big float, tag 5 optional", + {(const uint8_t []){0xC5, 0x82, 0x19, 0x01, 0x2C, 0x18, 0x64}, 7}, + QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + true, + + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_BIGFLOAT, + 300, + 100, + {(const uint8_t []){0x00}, 1}, + + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFraction */ + 0, + 0, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFractionBig */ + 0, + {(const uint8_t []){0x03}, 1}, + false, + + QCBOR_SUCCESS, /* for GetBigFloat */ + 300, + 100, + + QCBOR_SUCCESS, /* for GetBigFloatBig */ + 300, + {(const uint8_t []){0x64}, 1}, + false + }, + + { + "5. Tagged 4([-20, 4759477275222530853136]) decimal fraction, tag 4 required", + {(const uint8_t []){0xC4, 0x82, 0x33, + 0xC2, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10,}, 15}, + QCBOR_TAG_REQUIREMENT_TAG, + true, + + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_DECIMAL_FRACTION_POS_BIGNUM, + -20, + 0, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, /* for GetDecimalFraction */ + 0, + 0, + + QCBOR_SUCCESS, /* for GetDecimalFractionBig */ + -20, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + false, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloat */ + 0, + 0, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloatBig */ + 0, + {(const uint8_t []){0x00}, 0}, + false + }, + + { + "6. Error: Mantissa and exponent inside a Mantissa and exponent", + {(const uint8_t []){0xC4, 0x82, 0x33, + 0xC5, 0x82, 0x19, 0x01, 0x2C, 0x18, 0x64}, 10}, + QCBOR_TAG_REQUIREMENT_TAG, + true, + + QCBOR_ERR_BAD_EXP_AND_MANTISSA, /* for GetNext */ + QCBOR_TYPE_DECIMAL_FRACTION_POS_BIGNUM, + 0, + 0, + {(const uint8_t []){0x00}, 0}, + + QCBOR_ERR_BAD_EXP_AND_MANTISSA, /* for GetDecimalFraction */ + 0, + 0, + + QCBOR_ERR_BAD_EXP_AND_MANTISSA, /* for GetDecimalFractionBig */ + 0, + {(const uint8_t []){0x00}, 0}, + false, + + QCBOR_ERR_BAD_EXP_AND_MANTISSA, /* for GetBigFloat */ + 0, + 0, + + QCBOR_ERR_BAD_EXP_AND_MANTISSA, /* for GetBigFloatBig */ + 0, + {(const uint8_t []){0x00}, 0}, + false + }, + { + "7. Tagged 5([-20, 4294967295]) big float, big num mantissa, tag 5 required", + {(const uint8_t []){0xC5, 0x82, 0x33, + 0xC2, 0x44, 0xff, 0xff, 0xff, 0xff}, 9}, + QCBOR_TAG_REQUIREMENT_TAG, + true, + + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_BIGFLOAT_POS_BIGNUM, + -20, + 0, + {(const uint8_t []){0xff, 0xff, 0xff, 0xff}, 4}, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFraction */ + 0, + 0, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetDecimalFractionBig */ + -20, + {(const uint8_t []){0x00}, 1}, + false, + + QCBOR_SUCCESS, /* for GetBigFloat */ + -20, + 4294967295, + + QCBOR_SUCCESS, /* for GetBigFloatBig */ + -20, + {(const uint8_t []){0xff, 0xff, 0xff, 0xff}, 4}, + false + }, + + { + /* Special case for test 8. Don't renumber it. */ + "8. Untagged pair with big num (big float or decimal fraction), tag optional", + {(const uint8_t []){0x82, 0x33, 0xC2, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 14}, + QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + true, + + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_ARRAY, + 0, + 0, + {(const uint8_t []){0x00}, 1}, + + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, /* GetDecimalFraction */ + 0, + 0, + + QCBOR_SUCCESS, /* for GetDecimalFractionBig */ + -20, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + false, + + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, /* for GetBigFloat */ + 0, + 0, + + QCBOR_SUCCESS, /* for GetBigFloatBig */ + -20, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + false + }, + + { + "9. decimal fraction with large exponent and negative big num mantissa", + {(const uint8_t []){0xC4, 0x82, 0x1B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xC3, 0x4A, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 23}, + QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, + true, + + QCBOR_SUCCESS, /* for GetNext */ + QCBOR_TYPE_DECIMAL_FRACTION_NEG_BIGNUM, + 9223372036854775807, + 0, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, /* GetDecimalFraction */ + 0, + 0, + + QCBOR_SUCCESS, /* for GetDecimalFractionBig */ + 9223372036854775807, + {(const uint8_t []){0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}, 10}, + true, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloat */ + 0, + 0, + + QCBOR_ERR_UNEXPECTED_TYPE, /* for GetBigFloatBig */ + 0, + {(const uint8_t []){0x00}, 1}, + false + }, +}; + + + +int32_t ProcessEaMTests(void) +{ + size_t uIndex; + QCBORDecodeContext DCtx; + QCBORItem Item; + QCBORError uError; + int64_t nMantissa, nExponent; + MakeUsefulBufOnStack( MantissaBuf, 200); + UsefulBufC Mantissa; + bool bMantissaIsNegative; + + for(uIndex = 0; uIndex < C_ARRAY_COUNT(pEaMTests, struct EaMTest); uIndex++) { + const struct EaMTest *pT = &pEaMTests[uIndex]; + /* Decode with GetNext */ + QCBORDecode_Init(&DCtx, pT->Input, 0); + + if(uIndex + 1 == 9) { + nExponent = 99; // just to set a break point + } + + uError = QCBORDecode_GetNext(&DCtx, &Item); +#ifdef QCBOR_DISABLE_TAGS + /* Test 8 is a special case when tags are disabled */ + if(pT->bHasTags && uIndex + 1 != 8) { + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return (int32_t)(1+uIndex) * 1000 + 9; + } + } else { +#endif + /* Now check return code, data type, mantissa and exponent */ + if(pT->uExpectedErrorGN != uError) { + return (int32_t)(1+uIndex) * 1000 + 1; + } + if(uError == QCBOR_SUCCESS && pT->uQCBORTypeGN != QCBOR_TYPE_ARRAY) { + if(pT->uQCBORTypeGN != Item.uDataType) { + return (int32_t)(1+uIndex) * 1000 + 2; + } + if(pT->nExponentGN != Item.val.expAndMantissa.nExponent) { + return (int32_t)(1+uIndex) * 1000 + 3; + } + if(Item.uDataType == QCBOR_TYPE_DECIMAL_FRACTION || Item.uDataType == QCBOR_TYPE_BIGFLOAT ) { + if(pT->nMantissaGN != Item.val.expAndMantissa.Mantissa.nInt) { + return (int32_t)(1+uIndex) * 1000 + 4; + } + } else { + if(UsefulBuf_Compare(Item.val.expAndMantissa.Mantissa.bigNum, pT->MantissaGN)) { + return (int32_t)(1+uIndex) * 1000 + 5; + } + } + } +#ifdef QCBOR_DISABLE_TAGS + } +#endif + + /* Decode with GetDecimalFraction */ + QCBORDecode_Init(&DCtx, pT->Input, 0); + QCBORDecode_GetDecimalFraction(&DCtx, + pT->uTagRequirement, + &nMantissa, + &nExponent); + uError = QCBORDecode_GetAndResetError(&DCtx); +#ifdef QCBOR_DISABLE_TAGS + if(pT->bHasTags) { + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return (int32_t)(1+uIndex) * 1000 + 39; + } + } else { +#endif + /* Now check return code, mantissa and exponent */ + if(pT->uExpectedErrorGDF != uError) { + return (int32_t)(1+uIndex) * 1000 + 31; + } + if(uError == QCBOR_SUCCESS) { + if(pT->nExponentGDF != nExponent) { + return (int32_t)(1+uIndex) * 1000 + 32; + } + if(pT->nMantissaGDF != nMantissa) { + return (int32_t)(1+uIndex) * 1000 + 33; + } + } +#ifdef QCBOR_DISABLE_TAGS + } +#endif + + /* Decode with GetDecimalFractionBig */ + QCBORDecode_Init(&DCtx, pT->Input, 0); + QCBORDecode_GetDecimalFractionBig(&DCtx, + pT->uTagRequirement, + MantissaBuf, + &Mantissa, + &bMantissaIsNegative, + &nExponent); + uError = QCBORDecode_GetAndResetError(&DCtx); +#ifdef QCBOR_DISABLE_TAGS + if(pT->bHasTags) { + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return (int32_t)(1+uIndex) * 1000 + 49; + } + } else { +#endif + /* Now check return code, mantissa (bytes and sign) and exponent */ + if(pT->uExpectedErrorGDFB != uError) { + return (int32_t)(1+uIndex) * 1000 + 41; + } + if(uError == QCBOR_SUCCESS) { + if(pT->nExponentGDFB != nExponent) { + return (int32_t)(1+uIndex) * 1000 + 42; + } + if(pT->IsNegativeGDFB != bMantissaIsNegative) { + return (int32_t)(1+uIndex) * 1000 + 43; + } + if(UsefulBuf_Compare(Mantissa, pT->MantissaGDFB)) { + return (int32_t)(1+uIndex) * 1000 + 44; + } + } +#ifdef QCBOR_DISABLE_TAGS + } +#endif + + /* Decode with GetBigFloat */ + QCBORDecode_Init(&DCtx, pT->Input, 0); + QCBORDecode_GetBigFloat(&DCtx, + pT->uTagRequirement, + &nMantissa, + &nExponent); + uError = QCBORDecode_GetAndResetError(&DCtx); +#ifdef QCBOR_DISABLE_TAGS + if(pT->bHasTags) { + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return (int32_t)(1+uIndex) * 1000 + 19; + } + } else { +#endif + /* Now check return code, mantissa and exponent */ + if(pT->uExpectedErrorGBF != uError) { + return (int32_t)(1+uIndex) * 1000 + 11; + } + if(uError == QCBOR_SUCCESS) { + if(pT->nExponentGBF != nExponent) { + return (int32_t)(1+uIndex) * 1000 + 12; + } + if(pT->nMantissaGBF != nMantissa) { + return (int32_t)(1+uIndex) * 1000 + 13; + } + } +#ifdef QCBOR_DISABLE_TAGS + } +#endif + + /* Decode with GetBigFloatBig */ + QCBORDecode_Init(&DCtx, pT->Input, 0); + QCBORDecode_GetBigFloatBig(&DCtx, + pT->uTagRequirement, + MantissaBuf, + &Mantissa, + &bMantissaIsNegative, + &nExponent); + uError = QCBORDecode_GetAndResetError(&DCtx); +#ifdef QCBOR_DISABLE_TAGS + if(pT->bHasTags) { + if(uError != QCBOR_ERR_TAGS_DISABLED) { + return (int32_t)(1+uIndex) * 1000 + 29; + } + } else { +#endif + /* Now check return code, mantissa (bytes and sign) and exponent */ + if(pT->uExpectedErrorGBFB != uError) { + return (int32_t)(1+uIndex) * 1000 + 21; + } + if(uError == QCBOR_SUCCESS) { + if(pT->nExponentGBFB != nExponent) { + return (int32_t)(1+uIndex) * 1000 + 22; + } + if(pT->IsNegativeGBFB != bMantissaIsNegative) { + return (int32_t)(1+uIndex) * 1000 + 23; + } + if(UsefulBuf_Compare(Mantissa, pT->MantissaGBFB)) { + return (int32_t)(1+uIndex) * 1000 + 24; + } + } +#ifdef QCBOR_DISABLE_TAGS + } +#endif } - // 5([ 9223372036854775806, -4759477275222530853137]) - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 15; - } - if(item.uDataType != QCBOR_TYPE_BIGFLOAT_NEG_BIGNUM || - item.val.expAndMantissa.nExponent != 9223372036854775806 || - UsefulBuf_Compare(item.val.expAndMantissa.Mantissa.bigNum, BN)) { - return 16; - } + return 0; +} + + +int32_t ExponentAndMantissaDecodeTestsSecondary(void) +{ +#ifndef QCBOR_DISABLE_TAGS + QCBORDecodeContext DC; + QCBORError uErr; + QCBORItem item; + + static const uint8_t spBigNumMantissa[] = {0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x010}; + UsefulBufC BN = UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spBigNumMantissa); - // 5([ 9223372036854775806, 9223372036854775806])] - uErr = QCBORDecode_GetNext(&DC, &item); - if(uErr != QCBOR_SUCCESS) { - return 17; - } - if(item.uDataType != QCBOR_TYPE_BIGFLOAT || - item.val.expAndMantissa.nExponent != 9223372036854775806 || - item.val.expAndMantissa.Mantissa.nInt!= 9223372036854775806 ) { - return 18; - } - uErr = QCBORDecode_Finish(&DC); - if(uErr != QCBOR_SUCCESS) { - return 18; - } /* Now encode some stuff and then decode it */ uint8_t pBuf[40]; @@ -4752,55 +5230,20 @@ int32_t ExponentAndMantissaDecodeTests(void) return 106; } +#endif /* QCBOR_TAGS_DISABLED */ - int64_t nExp, nMant; - UsefulBuf_MAKE_STACK_UB( MantBuf, 20); - UsefulBufC Mant; - bool bIsNeg; - - QCBORDecode_Init(&DC, - UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spExpectedExponentsAndMantissas), - QCBOR_DECODE_MODE_NORMAL); - QCBORDecode_EnterArray(&DC, NULL); - - // 4([-1, 3]), - QCBORDecode_GetDecimalFraction(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nExp, &nMant); - - // 4([-20, 4759477275222530853136]), - QCBORDecode_GetDecimalFractionBig(&DC, QCBOR_TAG_REQUIREMENT_TAG, MantBuf, - &Mant, &bIsNeg, &nExp); - - // 4([9223372036854775807, -4759477275222530853137]), - QCBORDecode_GetDecimalFractionBig(&DC, QCBOR_TAG_REQUIREMENT_OPTIONAL_TAG, - MantBuf, &Mant, &bIsNeg, &nExp); - - // 5([300, 100]), - QCBORDecode_GetBigFloat(&DC, QCBOR_TAG_REQUIREMENT_TAG, &nExp, &nMant); - - // 5([-20, 4759477275222530853136]), - QCBORDecode_GetBigFloatBig(&DC, QCBOR_TAG_REQUIREMENT_TAG, MantBuf, &Mant, - &bIsNeg, &nExp); - - // 5([-9223372036854775807, -4759477275222530853137]) - QCBORDecode_GetBigFloatBig(&DC, QCBOR_TAG_REQUIREMENT_TAG, MantBuf, &Mant, - &bIsNeg, &nExp); - - // 5([ 9223372036854775806, -4759477275222530853137]) - QCBORDecode_GetBigFloatBig(&DC, QCBOR_TAG_REQUIREMENT_TAG, MantBuf, &Mant, - &bIsNeg, &nExp); - - // 5([ 9223372036854775806, 9223372036854775806])] - QCBORDecode_GetBigFloatBig(&DC, QCBOR_TAG_REQUIREMENT_TAG, MantBuf, &Mant, - &bIsNeg, &nExp); + return 0; +} - QCBORDecode_ExitArray(&DC); - uErr = QCBORDecode_Finish(&DC); - if(uErr != QCBOR_SUCCESS) { - return 200; +int32_t ExponentAndMantissaDecodeTests(void) +{ + int32_t rv = ProcessEaMTests(); + if(rv) { + return rv; } - return 0; + return ExponentAndMantissaDecodeTestsSecondary(); } @@ -4845,7 +5288,7 @@ static const struct FailInput ExponentAndMantissaFailures[] = { }; -int32_t ExponentAndMantissaDecodeFailTests() +int32_t ExponentAndMantissaDecodeFailTests(void) { return ProcessFailures(ExponentAndMantissaFailures, C_ARRAY_COUNT(ExponentAndMantissaFailures, @@ -5211,10 +5654,14 @@ static const uint8_t spMapOfEmpty[] = { } */ static const uint8_t spRecoverableMapErrors[] = { +#ifndef QCBOR_DISABLE_TAGS 0xa6, + 0x04, 0xc1, 0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c, 0x01, 0xd8, 0xe0, 0xd8, 0xe1, 0xd8, 0xe2, 0xd8, 0xe3, 0xd8, 0x04, 0x00, +#else + 0xa4, +#endif 0x03, 0x3b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x04, 0xc1, 0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c, 0x05, 0x00, 0x05, 0x00, 0x08, 0x08, @@ -5282,7 +5729,7 @@ const unsigned char spBadConsumeInput5[] = { -int32_t EnterMapTest() +int32_t EnterMapTest(void) { QCBORItem Item1; QCBORItem ArrayItem; @@ -5487,6 +5934,7 @@ int32_t EnterMapTest() int64_t nInt; QCBORDecode_Init(&DCtx, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spRecoverableMapErrors), 0); QCBORDecode_EnterMap(&DCtx, NULL); +#ifndef QCBOR_DISABLE_TAGS QCBORDecode_GetInt64InMapN(&DCtx, 0x01, &nInt); uErr = QCBORDecode_GetError(&DCtx); if(uErr != QCBOR_ERR_TOO_MANY_TAGS) { @@ -5496,6 +5944,7 @@ int32_t EnterMapTest() return 2121; } (void)QCBORDecode_GetAndResetError(&DCtx); +#endif QCBORDecode_GetInt64InMapN(&DCtx, 0x03, &nInt); @@ -5504,11 +5953,13 @@ int32_t EnterMapTest() return 2023; } +#ifndef QCBOR_DISABLE_TAGS QCBORDecode_GetEpochDateInMapN(&DCtx, 0x04, QCBOR_TAG_REQUIREMENT_TAG, &nInt); uErr = QCBORDecode_GetAndResetError(&DCtx); if(uErr != FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_DATE_OVERFLOW)) { return 2024; } +#endif QCBORDecode_GetInt64InMapN(&DCtx, 0x05, &nInt); uErr = QCBORDecode_GetAndResetError(&DCtx); @@ -5623,6 +6074,7 @@ int32_t EnterMapTest() return 2600; } +#ifndef QCBOR_DISABLE_TAGS QCBORDecode_Init(&DCtx, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spBadConsumeInput2), 0); QCBORDecode_VGetNextConsume(&DCtx, &Item1); if(QCBORDecode_GetError(&DCtx) != QCBOR_SUCCESS) { @@ -5634,6 +6086,8 @@ int32_t EnterMapTest() if(QCBORDecode_GetError(&DCtx) != QCBOR_ERR_UNRECOVERABLE_TAG_CONTENT) { return 2800; } +#endif + QCBORDecode_Init(&DCtx, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spBadConsumeInput4), 0); QCBORDecode_VGetNextConsume(&DCtx, &Item1); @@ -5676,6 +6130,19 @@ struct NumberConversion { static const struct NumberConversion NumberConversions[] = { +#ifndef QCBOR_DISABLE_TAGS + { + "Big float: INT64_MIN * 2e-1 to test handling of INT64_MIN", + {(uint8_t[]){0xC5, 0x82, 0x20, + 0x3B, 0x7f, 0xff, 0xff, 0xff, 0xff, 0x0ff, 0xff, 0xff, + }, 15}, + -4611686018427387904, /* INT64_MIN / 2 */ + EXP_AND_MANTISSA_ERROR(QCBOR_SUCCESS), + 0, + EXP_AND_MANTISSA_ERROR(QCBOR_ERR_NUMBER_SIGN_CONVERSION), + -4.6116860184273879E+18, + FLOAT_ERR_CODE_NO_FLOAT_HW(EXP_AND_MANTISSA_ERROR(QCBOR_SUCCESS)) + }, { "too large to fit into int64_t", {(uint8_t[]){0xc3, 0x48, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 10}, @@ -5793,77 +6260,6 @@ static const struct NumberConversion NumberConversions[] = { 12.0, FLOAT_ERR_CODE_NO_FLOAT_HW(EXP_AND_MANTISSA_ERROR(QCBOR_SUCCESS)) }, - { - "Positive integer 18446744073709551615", - {(uint8_t[]){0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 9}, - 0, - QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, - 18446744073709551615ULL, - QCBOR_SUCCESS, - 18446744073709551615.0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) - }, - { - "Positive bignum 0xffff", - {(uint8_t[]){0xC2, 0x42, 0xff, 0xff}, 4}, - 65536-1, - QCBOR_SUCCESS, - 0xffff, - QCBOR_SUCCESS, - 65535.0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) - }, - { - "Postive integer 0", - {(uint8_t[]){0x0}, 1}, - 0LL, - QCBOR_SUCCESS, - 0ULL, - QCBOR_SUCCESS, - 0.0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) - }, - { - "Negative integer -18446744073709551616", - {(uint8_t[]){0x3b, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, 9}, - -9223372036854775807-1, // INT64_MIN - QCBOR_SUCCESS, - 0ULL, - QCBOR_ERR_NUMBER_SIGN_CONVERSION, - -9223372036854775808.0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) - }, - { - "Double Floating point value 100.3", - {(uint8_t[]){0xfb, 0x40, 0x59, 0x13, 0x33, 0x33, 0x33, 0x33, 0x33}, 9}, - 100L, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), - 100ULL, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), - 100.3, - FLOAT_ERR_CODE_NO_FLOAT(QCBOR_SUCCESS), - }, - { - "Floating point value NaN 0xfa7fc00000", - {(uint8_t[]){0xfa, 0x7f, 0xc0, 0x00, 0x00}, 5}, - 0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), - 0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), - NAN, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), - }, - { - "half-precision Floating point value -4", - {(uint8_t[]){0xf9, 0xc4, 0x00}, 3}, - // Normal case with all enabled. - -4, - FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_SUCCESS), - 0, - FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_ERR_NUMBER_SIGN_CONVERSION), - -4.0, - FLOAT_ERR_CODE_NO_HALF_PREC(QCBOR_SUCCESS) - }, { "Decimal fraction 3/10", {(uint8_t[]){0xC4, 0x82, 0x20, 0x03}, 4}, @@ -5874,17 +6270,6 @@ static const struct NumberConversion NumberConversions[] = { 0.30000000000000004, FLOAT_ERR_CODE_NO_FLOAT_HW(EXP_AND_MANTISSA_ERROR(QCBOR_SUCCESS)) }, - { - "+inifinity single precision", - {(uint8_t[]){0xfa, 0x7f, 0x80, 0x00, 0x00}, 5}, - 0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), - 0, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW), - INFINITY, - FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) - }, - { "extreme pos bignum", {(uint8_t[]){0xc2, 0x59, 0x01, 0x90, @@ -6038,6 +6423,90 @@ static const struct NumberConversion NumberConversions[] = { -INFINITY, FLOAT_ERR_CODE_NO_FLOAT_HW(EXP_AND_MANTISSA_ERROR(QCBOR_SUCCESS)) }, + { + "Positive bignum 0xffff", + {(uint8_t[]){0xC2, 0x42, 0xff, 0xff}, 4}, + 65536-1, + QCBOR_SUCCESS, + 0xffff, + QCBOR_SUCCESS, + 65535.0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) + }, +#endif /* QCBOR_DISABLE_TAGS */ + { + "Positive integer 18446744073709551615", + {(uint8_t[]){0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 9}, + 0, + QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW, + 18446744073709551615ULL, + QCBOR_SUCCESS, + 18446744073709551615.0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) + }, + + { + "Postive integer 0", + {(uint8_t[]){0x0}, 1}, + 0LL, + QCBOR_SUCCESS, + 0ULL, + QCBOR_SUCCESS, + 0.0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) + }, + { + "Negative integer -18446744073709551616", + {(uint8_t[]){0x3b, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, 9}, + -9223372036854775807-1, // INT64_MIN + QCBOR_SUCCESS, + 0ULL, + QCBOR_ERR_NUMBER_SIGN_CONVERSION, + -9223372036854775808.0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) + }, + { + "Double Floating point value 100.3", + {(uint8_t[]){0xfb, 0x40, 0x59, 0x13, 0x33, 0x33, 0x33, 0x33, 0x33}, 9}, + 100L, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), + 100ULL, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), + 100.3, + FLOAT_ERR_CODE_NO_FLOAT(QCBOR_SUCCESS), + }, + { + "Floating point value NaN 0xfa7fc00000", + {(uint8_t[]){0xfa, 0x7f, 0xc0, 0x00, 0x00}, 5}, + 0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), + 0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), + NAN, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS), + }, + { + "half-precision Floating point value -4", + {(uint8_t[]){0xf9, 0xc4, 0x00}, 3}, + // Normal case with all enabled. + -4, + FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_SUCCESS), + 0, + FLOAT_ERR_CODE_NO_HALF_PREC_NO_FLOAT_HW(QCBOR_ERR_NUMBER_SIGN_CONVERSION), + -4.0, + FLOAT_ERR_CODE_NO_HALF_PREC(QCBOR_SUCCESS) + }, + { + "+inifinity single precision", + {(uint8_t[]){0xfa, 0x7f, 0x80, 0x00, 0x00}, 5}, + 0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_FLOAT_EXCEPTION), + 0, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_ERR_CONVERSION_UNDER_OVER_FLOW), + INFINITY, + FLOAT_ERR_CODE_NO_FLOAT_HW(QCBOR_SUCCESS) + }, + }; @@ -6057,7 +6526,7 @@ static int32_t SetUpDecoder(QCBORDecodeContext *DCtx, UsefulBufC CBOR, UsefulBuf } -int32_t IntegerConvertTest() +int32_t IntegerConvertTest(void) { const int nNumTests = C_ARRAY_COUNT(NumberConversions, struct NumberConversion); @@ -6130,7 +6599,7 @@ int32_t IntegerConvertTest() #ifndef QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS -int32_t CBORTestIssue134() +int32_t CBORTestIssue134(void) { QCBORDecodeContext DCtx; QCBORItem Item; @@ -6143,7 +6612,7 @@ int32_t CBORTestIssue134() UsefulBuf_MAKE_STACK_UB(StringBuf, 200); QCBORDecode_SetMemPool(&DCtx, StringBuf, false); - + do { uCBORError = QCBORDecode_GetNext(&DCtx, &Item); } while (QCBOR_SUCCESS == uCBORError); @@ -6155,6 +6624,23 @@ int32_t CBORTestIssue134() #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ + + +static const uint8_t spSequenceTestInput[] = { + /* 1. The valid date string "1985-04-12" */ + 0x6a, '1','9','8','5','-','0','4','-','1','2', // Date string + + /* 2. */ + 0x00, + + /* 3. A valid epoch date, 1400000000; Tue, 13 May 2014 16:53:20 GMT */ + 0x1a, 0x53, 0x72, 0x4E, 0x00, + + /* 4. */ + 0x62, 'h', 'i', +}; + + int32_t CBORSequenceDecodeTests(void) { QCBORDecodeContext DCtx; @@ -6164,46 +6650,43 @@ int32_t CBORSequenceDecodeTests(void) // --- Test a sequence with extra bytes --- - // The input for the date test happens to be a sequence so it - // is reused. It is a sequence because it doesn't start as - // an array or map. QCBORDecode_Init(&DCtx, - UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spDateTestInput), + UsefulBuf_FROM_BYTE_ARRAY_LITERAL(spSequenceTestInput), QCBOR_DECODE_MODE_NORMAL); - // Get the first item + // Get 1. uCBORError = QCBORDecode_GetNext(&DCtx, &Item); if(uCBORError != QCBOR_SUCCESS) { return 1; } - if(Item.uDataType != QCBOR_TYPE_DATE_STRING) { + if(Item.uDataType != QCBOR_TYPE_TEXT_STRING ) { return 2; } uCBORError = QCBORDecode_PartialFinish(&DCtx, &uConsumed); if(uCBORError != QCBOR_ERR_EXTRA_BYTES || - uConsumed != 12) { + uConsumed != 11) { return 102; } - // Get a second item + // Get 2. uCBORError = QCBORDecode_GetNext(&DCtx, &Item); - if(uCBORError != QCBOR_ERR_BAD_OPT_TAG) { + if(uCBORError != QCBOR_SUCCESS) { return 66; } uCBORError = QCBORDecode_PartialFinish(&DCtx, &uConsumed); if(uCBORError != QCBOR_ERR_EXTRA_BYTES || - uConsumed != 14) { + uConsumed != 12) { return 102; } - // Get a third item + // Get 3. uCBORError = QCBORDecode_GetNext(&DCtx, &Item); if(uCBORError != QCBOR_SUCCESS) { return 2; } - if(Item.uDataType != QCBOR_TYPE_DATE_EPOCH) { + if(Item.uDataType != QCBOR_TYPE_INT64) { return 3; } @@ -6220,7 +6703,6 @@ int32_t CBORSequenceDecodeTests(void) return 4; } - // --- Test an empty input ---- uint8_t empty[1]; UsefulBufC Empty = {empty, 0}; @@ -6318,7 +6800,7 @@ int32_t CBORSequenceDecodeTests(void) -int32_t IntToTests() +int32_t IntToTests(void) { int nErrCode; int32_t n32; @@ -6572,7 +7054,7 @@ static const uint8_t spBreakInByteString[] = { }; -int32_t EnterBstrTest() +int32_t EnterBstrTest(void) { UsefulBuf_MAKE_STACK_UB(OutputBuffer, 100); @@ -6734,7 +7216,7 @@ static const uint8_t spTaggedTypes[] = { 0x54, 0x43, 0x46, 0x49, 0x43, 0x41, 0x32 }; -int32_t DecodeTaggedTypeTests() +int32_t DecodeTaggedTypeTests(void) { QCBORDecodeContext DC; QCBORError uErr; @@ -7034,6 +7516,12 @@ int32_t TooLargeInputTest(void) #ifndef QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS +/* + An array of three map entries + 1) Indefinite length string label for indefinite lenght byte string + 2) Indefinite length string label for an integer + 3) Indefinite length string label for an indefinite-length negative big num + */ static const uint8_t spMapWithIndefLenStrings[] = { 0xa3, 0x7f, 0x61, 'l', 0x64, 'a', 'b', 'e', 'l' , 0x61, '1', 0xff, @@ -7045,7 +7533,7 @@ static const uint8_t spMapWithIndefLenStrings[] = { 0x5f, 0x42, 0x00, 0x01, 0x42, 0x00, 0x01, 0x41, 0x01, 0xff, }; -int32_t SpiffyIndefiniteLengthStringsTests() +int32_t SpiffyIndefiniteLengthStringsTests(void) { QCBORDecodeContext DCtx; @@ -7059,6 +7547,8 @@ int32_t SpiffyIndefiniteLengthStringsTests() UsefulBufC ByteString; QCBORDecode_EnterMap(&DCtx, NULL); QCBORDecode_GetByteStringInMapSZ(&DCtx, "label1", &ByteString); + +#ifndef QCBOR_DISABLE_TAGS if(QCBORDecode_GetAndResetError(&DCtx)) { return 1; } @@ -7083,6 +7573,7 @@ int32_t SpiffyIndefiniteLengthStringsTests() "label2", 0xff, &uDouble); + #ifndef QCBOR_DISABLE_FLOAT_HW_USE if(QCBORDecode_GetAndResetError(&DCtx)) { return 5; @@ -7097,13 +7588,23 @@ int32_t SpiffyIndefiniteLengthStringsTests() #endif /* QCBOR_DISABLE_FLOAT_HW_USE */ #endif /* USEFULBUF_DISABLE_ALL_FLOAT */ - QCBORDecode_ExitMap(&DCtx); if(QCBORDecode_Finish(&DCtx)) { return 99; } +#else /* QCBOR_DISABLE_TAGS */ + /* The big num in the input is a CBOR tag and you can't do + * map lookups in a map with a tag so this test does very little + * when tags are disabled. That is OK, the test coverage is still + * good when they are not. + */ + if(QCBORDecode_GetAndResetError(&DCtx) != QCBOR_ERR_TAGS_DISABLED) { + return 1002; + } +#endif /*QCBOR_DISABLE_TAGS */ + return 0; } #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ @@ -7179,6 +7680,20 @@ static const uint8_t pWithEmptyMapInDef[] = {0x9f, 0x18, 0x64, 0xbf, 0xff, 0xff} #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_ARRAYS */ #ifndef QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS + +/* +An array of one that contains + a byte string that is tagged 24 which means CBOR-encoded data + the byte string is an indefinite length string + the wrapped byte string is an array of three numbers + [42, 43, 44] + +[ + 24( + (_ h'83', h'18', h'2A182B', h'182C') + ) +] + */ static const uint8_t pWrappedByIndefiniteLength[] = { 0x81, 0xd8, 0x18, @@ -7192,7 +7707,7 @@ static const uint8_t pWrappedByIndefiniteLength[] = { #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ -int32_t PeekAndRewindTest() +int32_t PeekAndRewindTest(void) { QCBORItem Item; QCBORError nCBORError; @@ -7779,6 +8294,7 @@ int32_t PeekAndRewindTest() // Rewind an indefnite length byte-string wrapped sequence #ifndef QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS + // TODO: rewrite this test to not use tags QCBORDecode_Init(&DCtx, UsefulBuf_FROM_BYTE_ARRAY_LITERAL(pWrappedByIndefiniteLength), 0); @@ -7787,11 +8303,13 @@ int32_t PeekAndRewindTest() QCBORDecode_EnterArray(&DCtx, NULL); QCBORDecode_EnterBstrWrapped(&DCtx, 2, NULL); +#ifndef QCBOR_DISABLE_TAGS if(QCBORDecode_GetError(&DCtx) != QCBOR_ERR_INPUT_TOO_LARGE) { - /* this is what happens when trying to enter - indefinite-length byte string - wrapped CBOR. Tolerate for now. Eventually it needs - to be fixed so this works, but that is not simple. */ + /* TODO: This is what happens when trying to enter + * indefinite-length byte string wrapped CBOR. Tolerate for + * now. Eventually it needs to be fixed so this works, but that + * is not simple. + */ return 7300; } @@ -7805,6 +8323,13 @@ int32_t PeekAndRewindTest() if(i != 42) { return 7220; }*/ + +#else /* QCBOR_DISABLE_TAGS */ + if(QCBORDecode_GetError(&DCtx) != QCBOR_ERR_TAGS_DISABLED) { + return 7301; + } +#endif /* QCBOR_DISABLE_TAGS */ + #endif /* QCBOR_DISABLE_INDEFINITE_LENGTH_STRINGS */ diff --git a/3rdparty/exported/QCBOR/test/qcbor_encode_tests.c b/3rdparty/exported/QCBOR/test/qcbor_encode_tests.c index bfb4a8375ed0..6e569cac82fb 100644 --- a/3rdparty/exported/QCBOR/test/qcbor_encode_tests.c +++ b/3rdparty/exported/QCBOR/test/qcbor_encode_tests.c @@ -1,7 +1,7 @@ /*============================================================================== Copyright (c) 2016-2018, The Linux Foundation. Copyright (c) 2018-2021, Laurence Lundblade. - All rights reserved. + Copyright (c) 2022, Arm Limited. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -143,7 +143,7 @@ static uint8_t spBigBuf[2200]; /* Some very minimal tests. */ -int32_t BasicEncodeTest() +int32_t BasicEncodeTest(void) { // Very simple CBOR, a map with one boolean that is true in it QCBOREncodeContext EC; @@ -281,7 +281,7 @@ int32_t BasicEncodeTest() static const uint8_t spExpectedEncodedAll[] = { - 0x98, 0x22, 0x66, 0x55, 0x49, 0x4e, 0x54, 0x36, 0x32, 0xd8, + 0x98, 0x23, 0x66, 0x55, 0x49, 0x4e, 0x54, 0x36, 0x32, 0xd8, 0x64, 0x1a, 0x05, 0x5d, 0x23, 0x15, 0x65, 0x49, 0x4e, 0x54, 0x36, 0x34, 0xd8, 0x4c, 0x1b, 0x00, 0x00, 0x00, 0x12, 0x16, 0xaf, 0x2b, 0x15, 0x00, 0x38, 0x2b, 0xa4, 0x63, 0x4c, 0x42, @@ -296,12 +296,14 @@ static const uint8_t spExpectedEncodedAll[] = { 0x69, 0x65, 0xc1, 0x1a, 0x53, 0x72, 0x4e, 0x00, 0x66, 0x74, 0x69, 0x6d, 0x65, 0x28, 0x29, 0xc1, 0x1a, 0x58, 0x0d, 0x41, 0x72, 0x39, 0x07, 0xb0, 0xc1, 0x1a, 0x58, 0x0d, 0x3f, 0x76, - 0x42, 0xff, 0x00, 0xa3, 0x66, 0x62, 0x69, 0x6e, 0x62, 0x69, - 0x6e, 0xda, 0x00, 0x01, 0x86, 0xa0, 0x41, 0x00, 0x66, 0x62, + 0x42, 0xff, 0x00, 0xa4, 0x66, 0x62, 0x69, 0x6e, 0x62, 0x69, + 0x6e, 0xda, 0x00, 0x01, 0x86, 0xa0, 0x41, 0x00, + 0x65, 0x65, 0x6D, 0x70, 0x74, 0x79, 0x40, + 0x66, 0x62, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x43, 0x01, 0x02, 0x03, 0x00, 0x44, 0x04, 0x02, 0x03, 0xfe, 0x6f, 0x62, 0x61, 0x72, 0x20, 0x62, 0x61, 0x72, 0x20, 0x66, 0x6f, 0x6f, 0x20, 0x62, 0x61, - 0x72, 0x64, 0x6f, 0x6f, 0x66, 0x0a, 0xd8, 0x20, 0x78, 0x6b, + 0x72, 0x64, 0x6f, 0x6f, 0x66, 0x0a, 0x60, 0xd8, 0x20, 0x78, 0x6b, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6f, 0x76, 0x65, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x71, 0x75, 0x65, 0x73, 0x74, @@ -553,6 +555,7 @@ static void AddAll(QCBOREncodeContext *pECtx) QCBOREncode_AddSZString(pECtx, "binbin"); QCBOREncode_AddTag(pECtx, 100000); QCBOREncode_AddBytes(pECtx, ((UsefulBufC) {(uint8_t []){0x00}, 1})); + QCBOREncode_AddBytesToMap(pECtx, "empty", NULLUsefulBufC); // Empty string QCBOREncode_AddBytesToMap(pECtx, "blabel", ((UsefulBufC) {(uint8_t []){0x01, 0x02, 0x03}, 3})); QCBOREncode_AddBytesToMapN(pECtx, 0, ((UsefulBufC){(uint8_t []){0x04, 0x02, 0x03, 0xfe}, 4})); QCBOREncode_CloseMap(pECtx); @@ -560,6 +563,8 @@ static void AddAll(QCBOREncodeContext *pECtx) /* text blobs */ QCBOREncode_AddText(pECtx, UsefulBuf_FROM_SZ_LITERAL("bar bar foo bar")); QCBOREncode_AddSZString(pECtx, "oof\n"); + QCBOREncode_AddText(pECtx, NULLUsefulBufC); // Empty string + const char *szURL = "http://stackoverflow.com/questions/28059697/how-do-i-toggle-between-debug-and-release-builds-in-xcode-6-7-8"; QCBOREncode_AddURI(pECtx, UsefulBuf_FromSZ(szURL)); @@ -686,7 +691,7 @@ static void AddAll(QCBOREncodeContext *pECtx) } -int32_t AllAddMethodsTest() +int32_t AllAddMethodsTest(void) { /* Improvement: this test should be broken down into several so it is more * managable. Tags and labels could be more sensible */ @@ -814,7 +819,7 @@ static const uint8_t spExpectedEncodedInts[] = { to expected values generated from http://cbor.me. */ -int32_t IntegerValuesTest1() +int32_t IntegerValuesTest1(void) { QCBOREncodeContext ECtx; int nReturn = 0; @@ -899,7 +904,7 @@ int32_t IntegerValuesTest1() static const uint8_t spExpectedEncodedSimple[] = { 0x85, 0xf5, 0xf4, 0xf6, 0xf7, 0xa1, 0x65, 0x55, 0x4e, 0x44, 0x65, 0x66, 0xf7}; -int32_t SimpleValuesTest1() +int32_t SimpleValuesTest1(void) { QCBOREncodeContext ECtx; int nReturn = 0; @@ -946,7 +951,7 @@ int32_t SimpleValuesTest1() static const uint8_t spExpectedEncodedSimpleIndefiniteLength[] = { 0x9f, 0xf5, 0xf4, 0xf6, 0xf7, 0xbf, 0x65, 0x55, 0x4e, 0x44, 0x65, 0x66, 0xf7, 0xff, 0xff}; -int32_t SimpleValuesIndefiniteLengthTest1() +int32_t SimpleValuesIndefiniteLengthTest1(void) { QCBOREncodeContext ECtx; int nReturn = 0; @@ -1141,7 +1146,7 @@ static const uint8_t EncodeLengthThirtyone[] = { 0x31 }; -int32_t EncodeLengthThirtyoneTest() +int32_t EncodeLengthThirtyoneTest(void) { QCBOREncodeContext ECtx; int nReturn = 0; @@ -1234,7 +1239,7 @@ static const uint8_t spExpectedEncodedDates[] = { 0x30, 0x2E, 0x35, 0x32, 0x5A, 0x62, 0x53, 0x59, 0xD8, 0x64, 0x39, 0x29, 0xB3, 0x18, 0x2D, 0x19, 0x0F, 0x9A}; -int32_t EncodeDateTest() +int32_t EncodeDateTest(void) { QCBOREncodeContext ECtx; @@ -1291,7 +1296,7 @@ int32_t EncodeDateTest() } -int32_t ArrayNestingTest1() +int32_t ArrayNestingTest1(void) { QCBOREncodeContext ECtx; int i; @@ -1314,7 +1319,7 @@ int32_t ArrayNestingTest1() -int32_t ArrayNestingTest2() +int32_t ArrayNestingTest2(void) { QCBOREncodeContext ECtx; int i; @@ -1338,7 +1343,7 @@ int32_t ArrayNestingTest2() -int32_t ArrayNestingTest3() +int32_t ArrayNestingTest3(void) { QCBOREncodeContext ECtx; int i; @@ -1455,7 +1460,7 @@ static const uint8_t spEncodeRawExpected[] = { 0xff, 0xff}; -int32_t EncodeRawTest() +int32_t EncodeRawTest(void) { QCBOREncodeContext ECtx; @@ -1578,7 +1583,7 @@ static const uint8_t spValidMapEncoded[] = { 0x73 } ; -int32_t MapEncodeTest() +int32_t MapEncodeTest(void) { uint8_t *pEncodedMaps; size_t nEncodedMapLen; @@ -1734,7 +1739,7 @@ static const uint8_t spExpectedRTIC[] = { 0xaa, 0xbb, 0x01, 0x01}; -int32_t RTICResultsTest() +int32_t RTICResultsTest(void) { const UsefulBufC Encoded = FormatRTICResults(CBOR_SIMPLEV_FALSE, 1477263730, "recent", "0xA1eC5001", @@ -1772,7 +1777,7 @@ static const uint8_t spExpectedForBstrWrapCancel[] = {0x82, 0x19, 0x01, 0xC3, 0x /* * bstr wrapping test */ -int BstrWrapTest() +int32_t BstrWrapTest(void) { QCBOREncodeContext EC; @@ -1896,7 +1901,7 @@ int BstrWrapTest() -int32_t BstrWrapErrorTest() +int32_t BstrWrapErrorTest(void) { QCBOREncodeContext EC; UsefulBufC Wrapped; @@ -1963,7 +1968,7 @@ int32_t BstrWrapErrorTest() if(uError != QCBOR_ERR_ARRAY_NESTING_TOO_DEEP) { return (int32_t)(300 + uError); } - + return 0; } @@ -2207,7 +2212,7 @@ static int32_t DecodeNextNested2(UsefulBufC Wrapped) } -int32_t BstrWrapNestTest() +int32_t BstrWrapNestTest(void) { QCBOREncodeContext EC; QCBOREncode_Init(&EC, UsefulBuf_FROM_BYTE_ARRAY(spBigBuf)); @@ -2350,7 +2355,7 @@ static const uint8_t pProtectedHeaders[] = {0xa1, 0x01, 0x26}; C.2.1. This doesn't actually verify the signature (however the t_cose implementation does). */ -int32_t CoseSign1TBSTest() +int32_t CoseSign1TBSTest(void) { // All of this is from RFC 8152 C.2.1 const char *szKid = "11"; @@ -2474,7 +2479,7 @@ int32_t CoseSign1TBSTest() } -int32_t EncodeErrorTests() +int32_t EncodeErrorTests(void) { QCBOREncodeContext EC; QCBORError uErr; @@ -2767,7 +2772,7 @@ static const uint8_t spExpectedExponentAndMantissaMap[] = { }; -int32_t ExponentAndMantissaEncodeTests() +int32_t ExponentAndMantissaEncodeTests(void) { QCBOREncodeContext EC; UsefulBufC EncodedExponentAndMantissa; @@ -2879,7 +2884,7 @@ int32_t ExponentAndMantissaEncodeTests() #endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ -int32_t QCBORHeadTest() +int32_t QCBORHeadTest(void) { /* This test doesn't have to be extensive, because just about every * other test exercises QCBOREncode_EncodeHead(). diff --git a/3rdparty/exported/QCBOR/test/run_tests.c b/3rdparty/exported/QCBOR/test/run_tests.c index f28f526c0bdc..f2baaf114efa 100644 --- a/3rdparty/exported/QCBOR/test/run_tests.c +++ b/3rdparty/exported/QCBOR/test/run_tests.c @@ -99,8 +99,11 @@ static test_entry s_tests[] = { TEST_ENTRY(BasicEncodeTest), TEST_ENTRY(NestedMapTest), TEST_ENTRY(BignumParseTest), +#ifndef QCBOR_DISABLE_TAGS TEST_ENTRY(OptTagParseTest), TEST_ENTRY(DateParseTest), + TEST_ENTRY(DecodeTaggedTypeTests), +#endif /* QCBOR_DISABLE_TAGS */ TEST_ENTRY(SpiffyDateDecodeTest), TEST_ENTRY(ShortBufferParseTest2), TEST_ENTRY(ShortBufferParseTest), @@ -136,12 +139,12 @@ static test_entry s_tests[] = { TEST_ENTRY(EncodeLengthThirtyoneTest), TEST_ENTRY(CBORSequenceDecodeTests), TEST_ENTRY(IntToTests), - TEST_ENTRY(DecodeTaggedTypeTests), TEST_ENTRY(PeekAndRewindTest), -#ifndef QCBOR_DISABLE_EXP_AND_MANTISSA - TEST_ENTRY(EncodeLengthThirtyoneTest), +#ifndef QCBOR_DISABLE_EXP_AND_MANTISSA TEST_ENTRY(ExponentAndMantissaDecodeTests), +#ifndef QCBOR_DISABLE_TAGS TEST_ENTRY(ExponentAndMantissaDecodeFailTests), +#endif /* QCBOR_DISABLE_TAGS */ TEST_ENTRY(ExponentAndMantissaEncodeTests), #endif /* QCBOR_DISABLE_EXP_AND_MANTISSA */ TEST_ENTRY(ParseEmptyMapInMapTest), diff --git a/3rdparty/exported/QCBOR/ub-example.c b/3rdparty/exported/QCBOR/ub-example.c index 4c48d31a935c..996cf3a9071e 100644 --- a/3rdparty/exported/QCBOR/ub-example.c +++ b/3rdparty/exported/QCBOR/ub-example.c @@ -214,7 +214,7 @@ ExpandxUBAdaptor(const UsefulBufC Input, #define INPUT "xyz123xyz" -int32_t RunUsefulBufExample() +int32_t RunUsefulBufExample(void) { /* ------------ UsefulBuf examples ------------- */ UsefulBufC Input = UsefulBuf_FROM_SZ_LITERAL(INPUT); diff --git a/CHANGELOG.md b/CHANGELOG.md index b639662dc544..afd2e02ea4cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [4.0.8]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.8 - Add `/node/ready/app` and `/node/ready/gov` endpoints for the use of load balancers wanting to check if a node is ready to accept application or governance transactions. See [Operator RPC API](https://microsoft.github.io/CCF/main/operations/operator_rpc_api.html) for details. +- Updated QCBOR from `1.1` to `1.2`. - Upgrade `nghttp2` from `1.51.0` to `1.55.1`. ## [4.0.7] diff --git a/cgmanifest.json b/cgmanifest.json index 2ad06131f2f0..f41ed28a48c1 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -141,7 +141,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/laurencelundblade/QCBOR", - "commitHash": "07653df2bbdb2d090d98d0df514fa019ac23dff3" + "commitHash": "92d3f89030baff4af7be8396c563e6c8ef263622" } } }, From 6dc1a32958fe4d8c0903bca3c6b29e1b04f28fa2 Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:36:08 +0100 Subject: [PATCH 070/135] [release/4.x] Cherry pick: Update `fmt` library from `9.1.0` to `10.1.1` (#5605) (#5623) --- 3rdparty/exported/fmt/args.h | 2 +- 3rdparty/exported/fmt/chrono.h | 793 +++++---- 3rdparty/exported/fmt/color.h | 55 +- 3rdparty/exported/fmt/compile.h | 131 +- 3rdparty/exported/fmt/core.h | 1887 ++++++++------------- 3rdparty/exported/fmt/format-inl.h | 243 +-- 3rdparty/exported/fmt/format.h | 1869 +++++++++++--------- 3rdparty/exported/fmt/os.h | 115 +- 3rdparty/exported/fmt/ostream.h | 72 +- 3rdparty/exported/fmt/printf.h | 375 ++-- 3rdparty/exported/fmt/ranges.h | 303 ++-- 3rdparty/exported/fmt/std.h | 368 +++- 3rdparty/exported/fmt/xchar.h | 95 +- CHANGELOG.md | 1 + cgmanifest.json | 2 +- include/ccf/crypto/san.h | 2 +- include/ccf/ds/unit_strings.h | 22 +- include/ccf/service/node_info.h | 2 +- include/ccf/service/node_info_network.h | 2 +- include/ccf/service/tables/proposals.h | 2 +- src/clients/perf/scenario_perf_client.cpp | 2 +- src/clients/perf/timing.h | 2 +- src/host/lfs_file_handler.h | 4 +- src/host/snapshots.h | 2 +- src/kv/kv_types.h | 6 +- src/node/acme_client.h | 33 +- 26 files changed, 3300 insertions(+), 3090 deletions(-) diff --git a/3rdparty/exported/fmt/args.h b/3rdparty/exported/fmt/args.h index a3966d140719..2d684e7cc117 100644 --- a/3rdparty/exported/fmt/args.h +++ b/3rdparty/exported/fmt/args.h @@ -1,4 +1,4 @@ -// Formatting library for C++ - dynamic format arguments +// Formatting library for C++ - dynamic argument lists // // Copyright (c) 2012 - present, Victor Zverovich // All rights reserved. diff --git a/3rdparty/exported/fmt/chrono.h b/3rdparty/exported/fmt/chrono.h index b112f76e991c..ff3e1445b963 100644 --- a/3rdparty/exported/fmt/chrono.h +++ b/3rdparty/exported/fmt/chrono.h @@ -22,6 +22,24 @@ FMT_BEGIN_NAMESPACE +// Check if std::chrono::local_t is available. +#ifndef FMT_USE_LOCAL_TIME +# ifdef __cpp_lib_chrono +# define FMT_USE_LOCAL_TIME (__cpp_lib_chrono >= 201907L) +# else +# define FMT_USE_LOCAL_TIME 0 +# endif +#endif + +// Check if std::chrono::utc_timestamp is available. +#ifndef FMT_USE_UTC_TIME +# ifdef __cpp_lib_chrono +# define FMT_USE_UTC_TIME (__cpp_lib_chrono >= 201907L) +# else +# define FMT_USE_UTC_TIME 0 +# endif +#endif + // Enable tzset. #ifndef FMT_USE_TZSET // UWP doesn't provide _tzset. @@ -203,7 +221,8 @@ To safe_duration_cast(std::chrono::duration from, } const auto min1 = (std::numeric_limits::min)() / Factor::num; - if (!std::is_unsigned::value && count < min1) { + if (detail::const_check(!std::is_unsigned::value) && + count < min1) { ec = 1; return {}; } @@ -358,37 +377,11 @@ auto write_encoded_tm_str(OutputIt out, string_view in, const std::locale& loc) unit_t unit; write_codecvt(unit, in, loc); // In UTF-8 is used one to four one-byte code units. - auto&& buf = basic_memory_buffer(); - for (code_unit* p = unit.buf; p != unit.end; ++p) { - uint32_t c = static_cast(*p); - if (sizeof(code_unit) == 2 && c >= 0xd800 && c <= 0xdfff) { - // surrogate pair - ++p; - if (p == unit.end || (c & 0xfc00) != 0xd800 || - (*p & 0xfc00) != 0xdc00) { - FMT_THROW(format_error("failed to format time")); - } - c = (c << 10) + static_cast(*p) - 0x35fdc00; - } - if (c < 0x80) { - buf.push_back(static_cast(c)); - } else if (c < 0x800) { - buf.push_back(static_cast(0xc0 | (c >> 6))); - buf.push_back(static_cast(0x80 | (c & 0x3f))); - } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) { - buf.push_back(static_cast(0xe0 | (c >> 12))); - buf.push_back(static_cast(0x80 | ((c & 0xfff) >> 6))); - buf.push_back(static_cast(0x80 | (c & 0x3f))); - } else if (c >= 0x10000 && c <= 0x10ffff) { - buf.push_back(static_cast(0xf0 | (c >> 18))); - buf.push_back(static_cast(0x80 | ((c & 0x3ffff) >> 12))); - buf.push_back(static_cast(0x80 | ((c & 0xfff) >> 6))); - buf.push_back(static_cast(0x80 | (c & 0x3f))); - } else { - FMT_THROW(format_error("failed to format time")); - } - } - return copy_str(buf.data(), buf.data() + buf.size(), out); + auto u = + to_utf8>(); + if (!u.convert({unit.buf, to_unsigned(unit.end - unit.buf)})) + FMT_THROW(format_error("failed to format time")); + return copy_str(u.c_str(), u.c_str() + u.size(), out); } return copy_str(in.data(), in.data() + in.size(), out); } @@ -427,7 +420,7 @@ auto write(OutputIt out, const std::tm& time, const std::locale& loc, char format, char modifier = 0) -> OutputIt { auto&& buf = get_buffer(out); do_write(buf, time, loc, format, modifier); - return buf.out(); + return get_iterator(buf, out); } template time_point) { - return localtime(std::chrono::system_clock::to_time_t(time_point)); +#if FMT_USE_LOCAL_TIME +template +inline auto localtime(std::chrono::local_time time) -> std::tm { + return localtime(std::chrono::system_clock::to_time_t( + std::chrono::current_zone()->to_sys(time))); } +#endif /** Converts given time since epoch as ``std::time_t`` value into calendar time, @@ -523,7 +519,7 @@ inline std::tm gmtime(std::time_t time) { } #endif }; - dispatcher gt(time); + auto gt = dispatcher(time); // Too big time values may be unsupported. if (!gt.run()) FMT_THROW(format_error("time_t value out of range")); return gt.tm_; @@ -534,7 +530,7 @@ inline std::tm gmtime( return gmtime(std::chrono::system_clock::to_time_t(time_point)); } -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { // Writes two-digit numbers a, b and c separated by sep to buf. // The method by Pavel Novikov based on @@ -599,12 +595,39 @@ enum class numeric_system { alternative }; +// Glibc extensions for formatting numeric values. +enum class pad_type { + unspecified, + // Do not pad a numeric result string. + none, + // Pad a numeric result string with zeros even if the conversion specifier + // character uses space-padding by default. + zero, + // Pad a numeric result string with spaces. + space, +}; + +template +auto write_padding(OutputIt out, pad_type pad, int width) -> OutputIt { + if (pad == pad_type::none) return out; + return std::fill_n(out, width, pad == pad_type::space ? ' ' : '0'); +} + +template +auto write_padding(OutputIt out, pad_type pad) -> OutputIt { + if (pad != pad_type::none) *out++ = pad == pad_type::space ? ' ' : '0'; + return out; +} + // Parses a put_time-like format string and invokes handler actions. template FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, const Char* end, Handler&& handler) { + if (begin == end || *begin == '}') return begin; + if (*begin != '%') FMT_THROW(format_error("invalid format")); auto ptr = begin; + pad_type pad = pad_type::unspecified; while (ptr != end) { auto c = *ptr; if (c == '}') break; @@ -615,6 +638,22 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, if (begin != ptr) handler.on_text(begin, ptr); ++ptr; // consume '%' if (ptr == end) FMT_THROW(format_error("invalid format")); + c = *ptr; + switch (c) { + case '_': + pad = pad_type::space; + ++ptr; + break; + case '-': + pad = pad_type::none; + ++ptr; + break; + case '0': + pad = pad_type::zero; + ++ptr; + break; + } + if (ptr == end) FMT_THROW(format_error("invalid format")); c = *ptr++; switch (c) { case '%': @@ -691,16 +730,16 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, break; // Hour, minute, second: case 'H': - handler.on_24_hour(numeric_system::standard); + handler.on_24_hour(numeric_system::standard, pad); break; case 'I': - handler.on_12_hour(numeric_system::standard); + handler.on_12_hour(numeric_system::standard, pad); break; case 'M': - handler.on_minute(numeric_system::standard); + handler.on_minute(numeric_system::standard, pad); break; case 'S': - handler.on_second(numeric_system::standard); + handler.on_second(numeric_system::standard, pad); break; // Other: case 'c': @@ -737,7 +776,7 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, handler.on_duration_unit(); break; case 'z': - handler.on_utc_offset(); + handler.on_utc_offset(numeric_system::standard); break; case 'Z': handler.on_tz_name(); @@ -765,6 +804,9 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, case 'X': handler.on_loc_time(numeric_system::alternative); break; + case 'z': + handler.on_utc_offset(numeric_system::alternative); + break; default: FMT_THROW(format_error("invalid format")); } @@ -802,16 +844,19 @@ FMT_CONSTEXPR const Char* parse_chrono_format(const Char* begin, handler.on_dec1_weekday(numeric_system::alternative); break; case 'H': - handler.on_24_hour(numeric_system::alternative); + handler.on_24_hour(numeric_system::alternative, pad); break; case 'I': - handler.on_12_hour(numeric_system::alternative); + handler.on_12_hour(numeric_system::alternative, pad); break; case 'M': - handler.on_minute(numeric_system::alternative); + handler.on_minute(numeric_system::alternative, pad); break; case 'S': - handler.on_second(numeric_system::alternative); + handler.on_second(numeric_system::alternative, pad); + break; + case 'z': + handler.on_utc_offset(numeric_system::alternative); break; default: FMT_THROW(format_error("invalid format")); @@ -864,7 +909,7 @@ template struct null_chrono_spec_handler { FMT_CONSTEXPR void on_am_pm() { unsupported(); } FMT_CONSTEXPR void on_duration_value() { unsupported(); } FMT_CONSTEXPR void on_duration_unit() { unsupported(); } - FMT_CONSTEXPR void on_utc_offset() { unsupported(); } + FMT_CONSTEXPR void on_utc_offset(numeric_system) { unsupported(); } FMT_CONSTEXPR void on_tz_name() { unsupported(); } }; @@ -892,10 +937,10 @@ struct tm_format_checker : null_chrono_spec_handler { FMT_CONSTEXPR void on_day_of_year() {} FMT_CONSTEXPR void on_day_of_month(numeric_system) {} FMT_CONSTEXPR void on_day_of_month_space(numeric_system) {} - FMT_CONSTEXPR void on_24_hour(numeric_system) {} - FMT_CONSTEXPR void on_12_hour(numeric_system) {} - FMT_CONSTEXPR void on_minute(numeric_system) {} - FMT_CONSTEXPR void on_second(numeric_system) {} + FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_second(numeric_system, pad_type) {} FMT_CONSTEXPR void on_datetime(numeric_system) {} FMT_CONSTEXPR void on_loc_date(numeric_system) {} FMT_CONSTEXPR void on_loc_time(numeric_system) {} @@ -905,7 +950,7 @@ struct tm_format_checker : null_chrono_spec_handler { FMT_CONSTEXPR void on_24_hour_time() {} FMT_CONSTEXPR void on_iso_time() {} FMT_CONSTEXPR void on_am_pm() {} - FMT_CONSTEXPR void on_utc_offset() {} + FMT_CONSTEXPR void on_utc_offset(numeric_system) {} FMT_CONSTEXPR void on_tz_name() {} }; @@ -957,13 +1002,130 @@ inline void tzset_once() { } #endif -template class tm_writer { +// Converts value to Int and checks that it's in the range [0, upper). +template ::value)> +inline Int to_nonnegative_int(T value, Int upper) { + FMT_ASSERT(std::is_unsigned::value || + (value >= 0 && to_unsigned(value) <= to_unsigned(upper)), + "invalid value"); + (void)upper; + return static_cast(value); +} +template ::value)> +inline Int to_nonnegative_int(T value, Int upper) { + if (value < 0 || value > static_cast(upper)) + FMT_THROW(format_error("invalid value")); + return static_cast(value); +} + +constexpr long long pow10(std::uint32_t n) { + return n == 0 ? 1 : 10 * pow10(n - 1); +} + +// Counts the number of fractional digits in the range [0, 18] according to the +// C++20 spec. If more than 18 fractional digits are required then returns 6 for +// microseconds precision. +template () / 10)> +struct count_fractional_digits { + static constexpr int value = + Num % Den == 0 ? N : count_fractional_digits::value; +}; + +// Base case that doesn't instantiate any more templates +// in order to avoid overflow. +template +struct count_fractional_digits { + static constexpr int value = (Num % Den == 0) ? N : 6; +}; + +// Format subseconds which are given as an integer type with an appropriate +// number of digits. +template +void write_fractional_seconds(OutputIt& out, Duration d, int precision = -1) { + constexpr auto num_fractional_digits = + count_fractional_digits::value; + + using subsecond_precision = std::chrono::duration< + typename std::common_type::type, + std::ratio<1, detail::pow10(num_fractional_digits)>>; + + const auto fractional = + d - std::chrono::duration_cast(d); + const auto subseconds = + std::chrono::treat_as_floating_point< + typename subsecond_precision::rep>::value + ? fractional.count() + : std::chrono::duration_cast(fractional).count(); + auto n = static_cast>(subseconds); + const int num_digits = detail::count_digits(n); + + int leading_zeroes = (std::max)(0, num_fractional_digits - num_digits); + if (precision < 0) { + FMT_ASSERT(!std::is_floating_point::value, ""); + if (std::ratio_less::value) { + *out++ = '.'; + out = std::fill_n(out, leading_zeroes, '0'); + out = format_decimal(out, n, num_digits).end; + } + } else { + *out++ = '.'; + leading_zeroes = (std::min)(leading_zeroes, precision); + out = std::fill_n(out, leading_zeroes, '0'); + int remaining = precision - leading_zeroes; + if (remaining != 0 && remaining < num_digits) { + n /= to_unsigned(detail::pow10(to_unsigned(num_digits - remaining))); + out = format_decimal(out, n, remaining).end; + return; + } + out = format_decimal(out, n, num_digits).end; + remaining -= num_digits; + out = std::fill_n(out, remaining, '0'); + } +} + +// Format subseconds which are given as a floating point type with an +// appropriate number of digits. We cannot pass the Duration here, as we +// explicitly need to pass the Rep value in the chrono_formatter. +template +void write_floating_seconds(memory_buffer& buf, Duration duration, + int num_fractional_digits = -1) { + using rep = typename Duration::rep; + FMT_ASSERT(std::is_floating_point::value, ""); + + auto val = duration.count(); + + if (num_fractional_digits < 0) { + // For `std::round` with fallback to `round`: + // On some toolchains `std::round` is not available (e.g. GCC 6). + using namespace std; + num_fractional_digits = + count_fractional_digits::value; + if (num_fractional_digits < 6 && static_cast(round(val)) != val) + num_fractional_digits = 6; + } + + format_to(std::back_inserter(buf), FMT_STRING("{:.{}f}"), + std::fmod(val * static_cast(Duration::period::num) / + static_cast(Duration::period::den), + static_cast(60)), + num_fractional_digits); +} + +template +class tm_writer { private: static constexpr int days_per_week = 7; const std::locale& loc_; const bool is_classic_; OutputIt out_; + const Duration* subsecs_; const std::tm& tm_; auto tm_sec() const noexcept -> int { @@ -1051,6 +1213,17 @@ template class tm_writer { *out_++ = *d++; *out_++ = *d; } + void write2(int value, pad_type pad) { + unsigned int v = to_unsigned(value) % 100; + if (v >= 10) { + const char* d = digits2(v); + *out_++ = *d++; + *out_++ = *d; + } else { + out_ = detail::write_padding(out_, pad); + *out_++ = static_cast('0' + v); + } + } void write_year_extended(long long year) { // At least 4 characters. @@ -1074,7 +1247,7 @@ template class tm_writer { } } - void write_utc_offset(long offset) { + void write_utc_offset(long offset, numeric_system ns) { if (offset < 0) { *out_++ = '-'; offset = -offset; @@ -1083,14 +1256,15 @@ template class tm_writer { } offset /= 60; write2(static_cast(offset / 60)); + if (ns != numeric_system::standard) *out_++ = ':'; write2(static_cast(offset % 60)); } template ::value)> - void format_utc_offset_impl(const T& tm) { - write_utc_offset(tm.tm_gmtoff); + void format_utc_offset_impl(const T& tm, numeric_system ns) { + write_utc_offset(tm.tm_gmtoff, ns); } template ::value)> - void format_utc_offset_impl(const T& tm) { + void format_utc_offset_impl(const T& tm, numeric_system ns) { #if defined(_WIN32) && defined(_UCRT) # if FMT_USE_TZSET tzset_once(); @@ -1102,10 +1276,17 @@ template class tm_writer { _get_dstbias(&dstbias); offset += dstbias; } - write_utc_offset(-offset); + write_utc_offset(-offset, ns); #else - ignore_unused(tm); - format_localized('z'); + if (ns == numeric_system::standard) return format_localized('z'); + + // Extract timezone offset from timezone conversion functions. + std::tm gtm = tm; + std::time_t gt = std::mktime(>m); + std::tm ltm = gmtime(gt); + std::time_t lt = std::mktime(<m); + long offset = gt - lt; + write_utc_offset(offset, ns); #endif } @@ -1126,10 +1307,12 @@ template class tm_writer { } public: - tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm) + tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm, + const Duration* subsecs = nullptr) : loc_(loc), is_classic_(loc_ == get_classic_locale()), out_(out), + subsecs_(subsecs), tm_(tm) {} OutputIt out() const { return out_; } @@ -1227,7 +1410,7 @@ template class tm_writer { out_ = copy_str(std::begin(buf) + offset, std::end(buf), out_); } - void on_utc_offset() { format_utc_offset_impl(tm_); } + void on_utc_offset(numeric_system ns) { format_utc_offset_impl(tm_, ns); } void on_tz_name() { format_tz_name_impl(tm_); } void on_year(numeric_system ns) { @@ -1315,22 +1498,41 @@ template class tm_writer { } } - void on_24_hour(numeric_system ns) { - if (is_classic_ || ns == numeric_system::standard) return write2(tm_hour()); + void on_24_hour(numeric_system ns, pad_type pad) { + if (is_classic_ || ns == numeric_system::standard) + return write2(tm_hour(), pad); format_localized('H', 'O'); } - void on_12_hour(numeric_system ns) { + void on_12_hour(numeric_system ns, pad_type pad) { if (is_classic_ || ns == numeric_system::standard) - return write2(tm_hour12()); + return write2(tm_hour12(), pad); format_localized('I', 'O'); } - void on_minute(numeric_system ns) { - if (is_classic_ || ns == numeric_system::standard) return write2(tm_min()); + void on_minute(numeric_system ns, pad_type pad) { + if (is_classic_ || ns == numeric_system::standard) + return write2(tm_min(), pad); format_localized('M', 'O'); } - void on_second(numeric_system ns) { - if (is_classic_ || ns == numeric_system::standard) return write2(tm_sec()); - format_localized('S', 'O'); + + void on_second(numeric_system ns, pad_type pad) { + if (is_classic_ || ns == numeric_system::standard) { + write2(tm_sec(), pad); + if (subsecs_) { + if (std::is_floating_point::value) { + auto buf = memory_buffer(); + write_floating_seconds(buf, *subsecs_); + if (buf.size() > 1) { + // Remove the leading "0", write something like ".123". + out_ = std::copy(buf.begin() + 1, buf.end(), out_); + } + } else { + write_fractional_seconds(out_, *subsecs_); + } + } + } else { + // Currently no formatting of subseconds when a locale is set. + format_localized('S', 'O'); + } } void on_12_hour_time() { @@ -1351,10 +1553,9 @@ template class tm_writer { write2(tm_min()); } void on_iso_time() { - char buf[8]; - write_digit2_separated(buf, to_unsigned(tm_hour()), to_unsigned(tm_min()), - to_unsigned(tm_sec()), ':'); - out_ = copy_str(std::begin(buf), std::end(buf), out_); + on_24_hour_time(); + *out_++ = ':'; + on_second(numeric_system::standard, pad_type::unspecified); } void on_am_pm() { @@ -1372,43 +1573,34 @@ template class tm_writer { }; struct chrono_format_checker : null_chrono_spec_handler { + bool has_precision_integral = false; + FMT_NORETURN void unsupported() { FMT_THROW(format_error("no date")); } template FMT_CONSTEXPR void on_text(const Char*, const Char*) {} - FMT_CONSTEXPR void on_24_hour(numeric_system) {} - FMT_CONSTEXPR void on_12_hour(numeric_system) {} - FMT_CONSTEXPR void on_minute(numeric_system) {} - FMT_CONSTEXPR void on_second(numeric_system) {} + FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {} + FMT_CONSTEXPR void on_second(numeric_system, pad_type) {} FMT_CONSTEXPR void on_12_hour_time() {} FMT_CONSTEXPR void on_24_hour_time() {} FMT_CONSTEXPR void on_iso_time() {} FMT_CONSTEXPR void on_am_pm() {} - FMT_CONSTEXPR void on_duration_value() {} + FMT_CONSTEXPR void on_duration_value() const { + if (has_precision_integral) { + FMT_THROW(format_error("precision not allowed for this argument type")); + } + } FMT_CONSTEXPR void on_duration_unit() {} }; -template ::value)> +template ::value&& has_isfinite::value)> inline bool isfinite(T) { return true; } -// Converts value to Int and checks that it's in the range [0, upper). -template ::value)> -inline Int to_nonnegative_int(T value, Int upper) { - FMT_ASSERT(std::is_unsigned::value || - (value >= 0 && to_unsigned(value) <= to_unsigned(upper)), - "invalid value"); - (void)upper; - return static_cast(value); -} -template ::value)> -inline Int to_nonnegative_int(T value, Int upper) { - if (value < 0 || value > static_cast(upper)) - FMT_THROW(format_error("invalid value")); - return static_cast(value); -} - template ::value)> inline T mod(T x, int y) { return x % static_cast(y); @@ -1463,47 +1655,6 @@ inline std::chrono::duration get_milliseconds( #endif } -// Counts the number of fractional digits in the range [0, 18] according to the -// C++20 spec. If more than 18 fractional digits are required then returns 6 for -// microseconds precision. -template () / 10)> -struct count_fractional_digits { - static constexpr int value = - Num % Den == 0 ? N : count_fractional_digits::value; -}; - -// Base case that doesn't instantiate any more templates -// in order to avoid overflow. -template -struct count_fractional_digits { - static constexpr int value = (Num % Den == 0) ? N : 6; -}; - -constexpr long long pow10(std::uint32_t n) { - return n == 0 ? 1 : 10 * pow10(n - 1); -} - -template ::is_signed)> -constexpr std::chrono::duration abs( - std::chrono::duration d) { - // We need to compare the duration using the count() method directly - // due to a compiler bug in clang-11 regarding the spaceship operator, - // when -Wzero-as-null-pointer-constant is enabled. - // In clang-12 the bug has been fixed. See - // https://bugs.llvm.org/show_bug.cgi?id=46235 and the reproducible example: - // https://www.godbolt.org/z/Knbb5joYx. - return d.count() >= d.zero().count() ? d : -d; -} - -template ::is_signed)> -constexpr std::chrono::duration abs( - std::chrono::duration d) { - return d; -} - template ::value)> OutputIt format_duration_value(OutputIt out, Rep val, int) { @@ -1513,7 +1664,7 @@ OutputIt format_duration_value(OutputIt out, Rep val, int) { template ::value)> OutputIt format_duration_value(OutputIt out, Rep val, int precision) { - auto specs = basic_format_specs(); + auto specs = format_specs(); specs.precision = precision; specs.type = precision >= 0 ? presentation_type::fixed_lower : presentation_type::general_lower; @@ -1654,44 +1805,16 @@ struct chrono_formatter { } } - void write(Rep value, int width) { + void write(Rep value, int width, pad_type pad = pad_type::unspecified) { write_sign(); if (isnan(value)) return write_nan(); uint32_or_64_or_128_t n = to_unsigned(to_nonnegative_int(value, max_value())); int num_digits = detail::count_digits(n); - if (width > num_digits) out = std::fill_n(out, width - num_digits, '0'); - out = format_decimal(out, n, num_digits).end; - } - - template void write_fractional_seconds(Duration d) { - FMT_ASSERT(!std::is_floating_point::value, ""); - constexpr auto num_fractional_digits = - count_fractional_digits::value; - - using subsecond_precision = std::chrono::duration< - typename std::common_type::type, - std::ratio<1, detail::pow10(num_fractional_digits)>>; - if (std::ratio_less::value) { - *out++ = '.'; - auto fractional = - detail::abs(d) - std::chrono::duration_cast(d); - auto subseconds = - std::chrono::treat_as_floating_point< - typename subsecond_precision::rep>::value - ? fractional.count() - : std::chrono::duration_cast(fractional) - .count(); - uint32_or_64_or_128_t n = - to_unsigned(to_nonnegative_int(subseconds, max_value())); - int num_digits = detail::count_digits(n); - if (num_fractional_digits > num_digits) - out = std::fill_n(out, num_fractional_digits - num_digits, '0'); - out = format_decimal(out, n, num_digits).end; + if (width > num_digits) { + out = detail::write_padding(out, pad, width - num_digits); } + out = format_decimal(out, n, num_digits).end; } void write_nan() { std::copy_n("nan", 3, out); } @@ -1723,7 +1846,7 @@ struct chrono_formatter { void on_loc_time(numeric_system) {} void on_us_date() {} void on_iso_date() {} - void on_utc_offset() {} + void on_utc_offset(numeric_system) {} void on_tz_name() {} void on_year(numeric_system) {} void on_short_year(numeric_system) {} @@ -1739,58 +1862,56 @@ struct chrono_formatter { void on_day_of_month(numeric_system) {} void on_day_of_month_space(numeric_system) {} - void on_24_hour(numeric_system ns) { + void on_24_hour(numeric_system ns, pad_type pad) { if (handle_nan_inf()) return; - if (ns == numeric_system::standard) return write(hour(), 2); + if (ns == numeric_system::standard) return write(hour(), 2, pad); auto time = tm(); time.tm_hour = to_nonnegative_int(hour(), 24); - format_tm(time, &tm_writer_type::on_24_hour, ns); + format_tm(time, &tm_writer_type::on_24_hour, ns, pad); } - void on_12_hour(numeric_system ns) { + void on_12_hour(numeric_system ns, pad_type pad) { if (handle_nan_inf()) return; - if (ns == numeric_system::standard) return write(hour12(), 2); + if (ns == numeric_system::standard) return write(hour12(), 2, pad); auto time = tm(); time.tm_hour = to_nonnegative_int(hour12(), 12); - format_tm(time, &tm_writer_type::on_12_hour, ns); + format_tm(time, &tm_writer_type::on_12_hour, ns, pad); } - void on_minute(numeric_system ns) { + void on_minute(numeric_system ns, pad_type pad) { if (handle_nan_inf()) return; - if (ns == numeric_system::standard) return write(minute(), 2); + if (ns == numeric_system::standard) return write(minute(), 2, pad); auto time = tm(); time.tm_min = to_nonnegative_int(minute(), 60); - format_tm(time, &tm_writer_type::on_minute, ns); + format_tm(time, &tm_writer_type::on_minute, ns, pad); } - void on_second(numeric_system ns) { + void on_second(numeric_system ns, pad_type pad) { if (handle_nan_inf()) return; if (ns == numeric_system::standard) { if (std::is_floating_point::value) { - constexpr auto num_fractional_digits = - count_fractional_digits::value; auto buf = memory_buffer(); - format_to(std::back_inserter(buf), runtime("{:.{}f}"), - std::fmod(val * static_cast(Period::num) / - static_cast(Period::den), - static_cast(60)), - num_fractional_digits); + write_floating_seconds(buf, std::chrono::duration(val), + precision); if (negative) *out++ = '-'; - if (buf.size() < 2 || buf[1] == '.') *out++ = '0'; + if (buf.size() < 2 || buf[1] == '.') { + out = detail::write_padding(out, pad); + } out = std::copy(buf.begin(), buf.end(), out); } else { - write(second(), 2); - write_fractional_seconds(std::chrono::duration(val)); + write(second(), 2, pad); + write_fractional_seconds( + out, std::chrono::duration(val), precision); } return; } auto time = tm(); time.tm_sec = to_nonnegative_int(second(), 60); - format_tm(time, &tm_writer_type::on_second, ns); + format_tm(time, &tm_writer_type::on_second, ns, pad); } void on_12_hour_time() { @@ -1814,7 +1935,7 @@ struct chrono_formatter { on_24_hour_time(); *out++ = ':'; if (handle_nan_inf()) return; - on_second(numeric_system::standard); + on_second(numeric_system::standard, pad_type::unspecified); } void on_am_pm() { @@ -1833,7 +1954,7 @@ struct chrono_formatter { } }; -FMT_END_DETAIL_NAMESPACE +} // namespace detail #if defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907 using weekday = std::chrono::weekday; @@ -1883,118 +2004,67 @@ template struct formatter { template struct formatter, Char> { private: - basic_format_specs specs; - int precision = -1; - using arg_ref_type = detail::arg_ref; - arg_ref_type width_ref; - arg_ref_type precision_ref; - bool localized = false; - basic_string_view format_str; - using duration = std::chrono::duration; - - struct spec_handler { - formatter& f; - basic_format_parse_context& context; - basic_string_view format_str; - - template FMT_CONSTEXPR arg_ref_type make_arg_ref(Id arg_id) { - context.check_arg_id(arg_id); - return arg_ref_type(arg_id); - } - - FMT_CONSTEXPR arg_ref_type make_arg_ref(basic_string_view arg_id) { - context.check_arg_id(arg_id); - return arg_ref_type(arg_id); - } - - FMT_CONSTEXPR arg_ref_type make_arg_ref(detail::auto_id) { - return arg_ref_type(context.next_arg_id()); - } + format_specs specs_; + detail::arg_ref width_ref_; + detail::arg_ref precision_ref_; + bool localized_ = false; + basic_string_view format_str_; - void on_error(const char* msg) { FMT_THROW(format_error(msg)); } - FMT_CONSTEXPR void on_fill(basic_string_view fill) { - f.specs.fill = fill; - } - FMT_CONSTEXPR void on_align(align_t align) { f.specs.align = align; } - FMT_CONSTEXPR void on_width(int width) { f.specs.width = width; } - FMT_CONSTEXPR void on_precision(int _precision) { - f.precision = _precision; - } - FMT_CONSTEXPR void end_precision() {} + public: + FMT_CONSTEXPR auto parse(basic_format_parse_context& ctx) + -> decltype(ctx.begin()) { + auto it = ctx.begin(), end = ctx.end(); + if (it == end || *it == '}') return it; - template FMT_CONSTEXPR void on_dynamic_width(Id arg_id) { - f.width_ref = make_arg_ref(arg_id); - } + it = detail::parse_align(it, end, specs_); + if (it == end) return it; - template FMT_CONSTEXPR void on_dynamic_precision(Id arg_id) { - f.precision_ref = make_arg_ref(arg_id); - } - }; + it = detail::parse_dynamic_spec(it, end, specs_.width, width_ref_, ctx); + if (it == end) return it; - using iterator = typename basic_format_parse_context::iterator; - struct parse_range { - iterator begin; - iterator end; - }; - - FMT_CONSTEXPR parse_range do_parse(basic_format_parse_context& ctx) { - auto begin = ctx.begin(), end = ctx.end(); - if (begin == end || *begin == '}') return {begin, begin}; - spec_handler handler{*this, ctx, format_str}; - begin = detail::parse_align(begin, end, handler); - if (begin == end) return {begin, begin}; - begin = detail::parse_width(begin, end, handler); - if (begin == end) return {begin, begin}; - if (*begin == '.') { - if (std::is_floating_point::value) - begin = detail::parse_precision(begin, end, handler); - else - handler.on_error("precision not allowed for this argument type"); + auto checker = detail::chrono_format_checker(); + if (*it == '.') { + checker.has_precision_integral = !std::is_floating_point::value; + it = detail::parse_precision(it, end, specs_.precision, precision_ref_, + ctx); } - if (begin != end && *begin == 'L') { - ++begin; - localized = true; + if (it != end && *it == 'L') { + localized_ = true; + ++it; } - end = detail::parse_chrono_format(begin, end, - detail::chrono_format_checker()); - return {begin, end}; - } - - public: - FMT_CONSTEXPR auto parse(basic_format_parse_context& ctx) - -> decltype(ctx.begin()) { - auto range = do_parse(ctx); - format_str = basic_string_view( - &*range.begin, detail::to_unsigned(range.end - range.begin)); - return range.end; + end = detail::parse_chrono_format(it, end, checker); + format_str_ = {it, detail::to_unsigned(end - it)}; + return end; } template - auto format(const duration& d, FormatContext& ctx) const + auto format(std::chrono::duration d, FormatContext& ctx) const -> decltype(ctx.out()) { - auto specs_copy = specs; - auto precision_copy = precision; - auto begin = format_str.begin(), end = format_str.end(); + auto specs = specs_; + auto precision = specs.precision; + specs.precision = -1; + auto begin = format_str_.begin(), end = format_str_.end(); // As a possible future optimization, we could avoid extra copying if width // is not specified. - basic_memory_buffer buf; + auto buf = basic_memory_buffer(); auto out = std::back_inserter(buf); - detail::handle_dynamic_spec(specs_copy.width, - width_ref, ctx); - detail::handle_dynamic_spec(precision_copy, - precision_ref, ctx); + detail::handle_dynamic_spec(specs.width, width_ref_, + ctx); + detail::handle_dynamic_spec(precision, + precision_ref_, ctx); if (begin == end || *begin == '}') { - out = detail::format_duration_value(out, d.count(), precision_copy); + out = detail::format_duration_value(out, d.count(), precision); detail::format_duration_unit(out); } else { - detail::chrono_formatter f( - ctx, out, d); - f.precision = precision_copy; - f.localized = localized; + using chrono_formatter = + detail::chrono_formatter; + auto f = chrono_formatter(ctx, out, d); + f.precision = precision; + f.localized = localized_; detail::parse_chrono_format(begin, end, f); } return detail::write( - ctx.out(), basic_string_view(buf.data(), buf.size()), specs_copy); + ctx.out(), basic_string_view(buf.data(), buf.size()), specs); } }; @@ -2002,68 +2072,137 @@ template struct formatter, Char> : formatter { FMT_CONSTEXPR formatter() { - basic_string_view default_specs = - detail::string_literal{}; - this->do_parse(default_specs.begin(), default_specs.end()); + this->format_str_ = detail::string_literal{}; } template - auto format(std::chrono::time_point val, + auto format(std::chrono::time_point val, FormatContext& ctx) const -> decltype(ctx.out()) { - return formatter::format(localtime(val), ctx); + using period = typename Duration::period; + if (detail::const_check( + period::num != 1 || period::den != 1 || + std::is_floating_point::value)) { + const auto epoch = val.time_since_epoch(); + auto subsecs = std::chrono::duration_cast( + epoch - std::chrono::duration_cast(epoch)); + + if (subsecs.count() < 0) { + auto second = + std::chrono::duration_cast(std::chrono::seconds(1)); + if (epoch.count() < ((Duration::min)() + second).count()) + FMT_THROW(format_error("duration is too small")); + subsecs += second; + val -= second; + } + + return formatter::do_format( + gmtime(std::chrono::time_point_cast(val)), ctx, + &subsecs); + } + + return formatter::format( + gmtime(std::chrono::time_point_cast(val)), ctx); } }; +#if FMT_USE_LOCAL_TIME +template +struct formatter, Char> + : formatter { + FMT_CONSTEXPR formatter() { + this->format_str_ = detail::string_literal{}; + } + + template + auto format(std::chrono::local_time val, FormatContext& ctx) const + -> decltype(ctx.out()) { + using period = typename Duration::period; + if (period::num != 1 || period::den != 1 || + std::is_floating_point::value) { + const auto epoch = val.time_since_epoch(); + const auto subsecs = std::chrono::duration_cast( + epoch - std::chrono::duration_cast(epoch)); + + return formatter::do_format( + localtime(std::chrono::time_point_cast(val)), + ctx, &subsecs); + } + + return formatter::format( + localtime(std::chrono::time_point_cast(val)), + ctx); + } +}; +#endif + +#if FMT_USE_UTC_TIME +template +struct formatter, + Char> + : formatter, + Char> { + template + auto format(std::chrono::time_point val, + FormatContext& ctx) const -> decltype(ctx.out()) { + return formatter< + std::chrono::time_point, + Char>::format(std::chrono::utc_clock::to_sys(val), ctx); + } +}; +#endif + template struct formatter { private: - enum class spec { - unknown, - year_month_day, - hh_mm_ss, - }; - spec spec_ = spec::unknown; - basic_string_view specs; + format_specs specs_; + detail::arg_ref width_ref_; protected: - template FMT_CONSTEXPR auto do_parse(It begin, It end) -> It { - if (begin != end && *begin == ':') ++begin; - end = detail::parse_chrono_format(begin, end, detail::tm_format_checker()); - // Replace default spec only if the new spec is not empty. - if (end != begin) specs = {begin, detail::to_unsigned(end - begin)}; - return end; + basic_string_view format_str_; + + template + auto do_format(const std::tm& tm, FormatContext& ctx, + const Duration* subsecs) const -> decltype(ctx.out()) { + auto specs = specs_; + auto buf = basic_memory_buffer(); + auto out = std::back_inserter(buf); + detail::handle_dynamic_spec(specs.width, width_ref_, + ctx); + + auto loc_ref = ctx.locale(); + detail::get_locale loc(static_cast(loc_ref), loc_ref); + auto w = + detail::tm_writer(loc, out, tm, subsecs); + detail::parse_chrono_format(format_str_.begin(), format_str_.end(), w); + return detail::write( + ctx.out(), basic_string_view(buf.data(), buf.size()), specs); } public: FMT_CONSTEXPR auto parse(basic_format_parse_context& ctx) -> decltype(ctx.begin()) { - auto end = this->do_parse(ctx.begin(), ctx.end()); - // basic_string_view<>::compare isn't constexpr before C++17. - if (specs.size() == 2 && specs[0] == Char('%')) { - if (specs[1] == Char('F')) - spec_ = spec::year_month_day; - else if (specs[1] == Char('T')) - spec_ = spec::hh_mm_ss; - } + auto it = ctx.begin(), end = ctx.end(); + if (it == end || *it == '}') return it; + + it = detail::parse_align(it, end, specs_); + if (it == end) return it; + + it = detail::parse_dynamic_spec(it, end, specs_.width, width_ref_, ctx); + if (it == end) return it; + + end = detail::parse_chrono_format(it, end, detail::tm_format_checker()); + // Replace the default format_str only if the new spec is not empty. + if (end != it) format_str_ = {it, detail::to_unsigned(end - it)}; return end; } template auto format(const std::tm& tm, FormatContext& ctx) const -> decltype(ctx.out()) { - const auto loc_ref = ctx.locale(); - detail::get_locale loc(static_cast(loc_ref), loc_ref); - auto w = detail::tm_writer(loc, ctx.out(), tm); - if (spec_ == spec::year_month_day) - w.on_iso_date(); - else if (spec_ == spec::hh_mm_ss) - w.on_iso_time(); - else - detail::parse_chrono_format(specs.begin(), specs.end(), w); - return w.out(); + return do_format(tm, ctx, nullptr); } }; -FMT_MODULE_EXPORT_END +FMT_END_EXPORT FMT_END_NAMESPACE #endif // FMT_CHRONO_H_ diff --git a/3rdparty/exported/fmt/color.h b/3rdparty/exported/fmt/color.h index 4c163277ef18..8697e1ca0bad 100644 --- a/3rdparty/exported/fmt/color.h +++ b/3rdparty/exported/fmt/color.h @@ -11,7 +11,7 @@ #include "format.h" FMT_BEGIN_NAMESPACE -FMT_MODULE_EXPORT_BEGIN +FMT_BEGIN_EXPORT enum class color : uint32_t { alice_blue = 0xF0F8FF, // rgb(240,248,255) @@ -203,7 +203,7 @@ struct rgb { uint8_t b; }; -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { // color is a struct of either a rgb color or a terminal color. struct color_type { @@ -225,8 +225,7 @@ struct color_type { uint32_t rgb_color; } value; }; - -FMT_END_DETAIL_NAMESPACE +} // namespace detail /** A text style consisting of foreground and background colors and emphasis. */ class text_style { @@ -323,7 +322,7 @@ FMT_CONSTEXPR inline text_style operator|(emphasis lhs, emphasis rhs) noexcept { return text_style(lhs) | rhs; } -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { template struct ansi_color_escape { FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color, @@ -423,26 +422,6 @@ FMT_CONSTEXPR ansi_color_escape make_emphasis(emphasis em) noexcept { return ansi_color_escape(em); } -template inline void fputs(const Char* chars, FILE* stream) { - int result = std::fputs(chars, stream); - if (result < 0) - FMT_THROW(system_error(errno, FMT_STRING("cannot write to file"))); -} - -template <> inline void fputs(const wchar_t* chars, FILE* stream) { - int result = std::fputws(chars, stream); - if (result < 0) - FMT_THROW(system_error(errno, FMT_STRING("cannot write to file"))); -} - -template inline void reset_color(FILE* stream) { - fputs("\x1b[0m", stream); -} - -template <> inline void reset_color(FILE* stream) { - fputs(L"\x1b[0m", stream); -} - template inline void reset_color(buffer& buffer) { auto reset_color = string_view("\x1b[0m"); buffer.append(reset_color.begin(), reset_color.end()); @@ -477,19 +456,21 @@ void vformat_to(buffer& buf, const text_style& ts, if (has_style) detail::reset_color(buf); } -FMT_END_DETAIL_NAMESPACE +} // namespace detail -template > -void vprint(std::FILE* f, const text_style& ts, const S& format, - basic_format_args>> args) { - basic_memory_buffer buf; - detail::vformat_to(buf, ts, detail::to_string_view(format), args); +inline void vprint(std::FILE* f, const text_style& ts, string_view fmt, + format_args args) { + // Legacy wide streams are not supported. + auto buf = memory_buffer(); + detail::vformat_to(buf, ts, fmt, args); if (detail::is_utf8()) { - detail::print(f, basic_string_view(buf.begin(), buf.size())); - } else { - buf.push_back(Char(0)); - detail::fputs(buf.data(), f); + detail::print(f, string_view(buf.begin(), buf.size())); + return; } + buf.push_back('\0'); + int result = std::fputs(buf.data(), f); + if (result < 0) + FMT_THROW(system_error(errno, FMT_STRING("cannot write to file"))); } /** @@ -566,7 +547,7 @@ OutputIt vformat_to( basic_format_args>> args) { auto&& buf = detail::get_buffer(out); detail::vformat_to(buf, ts, format_str, args); - return detail::get_iterator(buf); + return detail::get_iterator(buf, out); } /** @@ -645,7 +626,7 @@ FMT_CONSTEXPR auto styled(const T& value, text_style ts) return detail::styled_arg>{value, ts}; } -FMT_MODULE_EXPORT_END +FMT_END_EXPORT FMT_END_NAMESPACE #endif // FMT_COLOR_H_ diff --git a/3rdparty/exported/fmt/compile.h b/3rdparty/exported/fmt/compile.h index 933668c41c3e..a4c7e49563dc 100644 --- a/3rdparty/exported/fmt/compile.h +++ b/3rdparty/exported/fmt/compile.h @@ -19,84 +19,6 @@ FMT_CONSTEXPR inline counting_iterator copy_str(InputIt begin, InputIt end, return it + (end - begin); } -template class truncating_iterator_base { - protected: - OutputIt out_; - size_t limit_; - size_t count_ = 0; - - truncating_iterator_base() : out_(), limit_(0) {} - - truncating_iterator_base(OutputIt out, size_t limit) - : out_(out), limit_(limit) {} - - public: - using iterator_category = std::output_iterator_tag; - using value_type = typename std::iterator_traits::value_type; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = void; - FMT_UNCHECKED_ITERATOR(truncating_iterator_base); - - OutputIt base() const { return out_; } - size_t count() const { return count_; } -}; - -// An output iterator that truncates the output and counts the number of objects -// written to it. -template ::value_type>::type> -class truncating_iterator; - -template -class truncating_iterator - : public truncating_iterator_base { - mutable typename truncating_iterator_base::value_type blackhole_; - - public: - using value_type = typename truncating_iterator_base::value_type; - - truncating_iterator() = default; - - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} - - truncating_iterator& operator++() { - if (this->count_++ < this->limit_) ++this->out_; - return *this; - } - - truncating_iterator operator++(int) { - auto it = *this; - ++*this; - return it; - } - - value_type& operator*() const { - return this->count_ < this->limit_ ? *this->out_ : blackhole_; - } -}; - -template -class truncating_iterator - : public truncating_iterator_base { - public: - truncating_iterator() = default; - - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} - - template truncating_iterator& operator=(T val) { - if (this->count_++ < this->limit_) *this->out_++ = val; - return *this; - } - - truncating_iterator& operator++() { return *this; } - truncating_iterator& operator++(int) { return *this; } - truncating_iterator& operator*() { return *this; } -}; - // A compile-time string which is compiled into fast formatting code. class compiled_string {}; @@ -196,7 +118,8 @@ template struct code_unit { template constexpr OutputIt format(OutputIt out, const Args&...) const { - return write(out, value); + *out++ = value; + return out; } }; @@ -220,7 +143,12 @@ template struct field { template constexpr OutputIt format(OutputIt out, const Args&... args) const { - return write(out, get_arg_checked(args...)); + const T& arg = get_arg_checked(args...); + if constexpr (std::is_convertible_v>) { + auto s = basic_string_view(arg); + return copy_str(s.begin(), s.end(), out); + } + return write(out, arg); } }; @@ -331,14 +259,14 @@ template struct parse_specs_result { int next_arg_id; }; -constexpr int manual_indexing_id = -1; +enum { manual_indexing_id = -1 }; template constexpr parse_specs_result parse_specs(basic_string_view str, size_t pos, int next_arg_id) { str.remove_prefix(pos); - auto ctx = compile_parse_context(str, max_value(), nullptr, {}, - next_arg_id); + auto ctx = + compile_parse_context(str, max_value(), nullptr, next_arg_id); auto f = formatter(); auto end = f.parse(ctx); return {f, pos + fmt::detail::to_unsigned(end - str.data()), @@ -348,22 +276,18 @@ constexpr parse_specs_result parse_specs(basic_string_view str, template struct arg_id_handler { arg_ref arg_id; - constexpr int operator()() { + constexpr int on_auto() { FMT_ASSERT(false, "handler cannot be used with automatic indexing"); return 0; } - constexpr int operator()(int id) { + constexpr int on_index(int id) { arg_id = arg_ref(id); return 0; } - constexpr int operator()(basic_string_view id) { + constexpr int on_name(basic_string_view id) { arg_id = arg_ref(id); return 0; } - - constexpr void on_error(const char* message) { - FMT_THROW(format_error(message)); - } }; template struct parse_arg_id_result { @@ -452,20 +376,18 @@ constexpr auto compile_format_string(S format_str) { } else if constexpr (arg_id_result.arg_id.kind == arg_id_kind::name) { constexpr auto arg_index = get_arg_index_by_name(arg_id_result.arg_id.val.name, Args{}); - if constexpr (arg_index != invalid_arg_index) { + if constexpr (arg_index >= 0) { constexpr auto next_id = ID != manual_indexing_id ? ID + 1 : manual_indexing_id; return parse_replacement_field_then_tail< decltype(get_type::value), Args, arg_id_end_pos, arg_index, next_id>(format_str); - } else { - if constexpr (c == '}') { - return parse_tail( - runtime_named_field{arg_id_result.arg_id.val.name}, - format_str); - } else if constexpr (c == ':') { - return unknown_format(); // no type info for specs parsing - } + } else if constexpr (c == '}') { + return parse_tail( + runtime_named_field{arg_id_result.arg_id.val.name}, + format_str); + } else if constexpr (c == ':') { + return unknown_format(); // no type info for specs parsing } } } @@ -501,7 +423,7 @@ constexpr auto compile(S format_str) { #endif // defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction) } // namespace detail -FMT_MODULE_EXPORT_BEGIN +FMT_BEGIN_EXPORT #if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction) @@ -568,9 +490,10 @@ template ::value)> format_to_n_result format_to_n(OutputIt out, size_t n, const S& format_str, Args&&... args) { - auto it = fmt::format_to(detail::truncating_iterator(out, n), - format_str, std::forward(args)...); - return {it.base(), it.count()}; + using traits = detail::fixed_buffer_traits; + auto buf = detail::iterator_buffer(out, n); + format_to(std::back_inserter(buf), format_str, std::forward(args)...); + return {buf.out(), buf.count()}; } template constexpr auto operator""_cf() { } // namespace literals #endif -FMT_MODULE_EXPORT_END +FMT_END_EXPORT FMT_END_NAMESPACE #endif // FMT_COMPILE_H_ diff --git a/3rdparty/exported/fmt/core.h b/3rdparty/exported/fmt/core.h index f6a37af9e35d..f9e3b7d6dc16 100644 --- a/3rdparty/exported/fmt/core.h +++ b/3rdparty/exported/fmt/core.h @@ -13,11 +13,12 @@ #include // std::strlen #include #include +#include // std::addressof #include #include // The fmt library version in the form major * 10000 + minor * 100 + patch. -#define FMT_VERSION 90100 +#define FMT_VERSION 100101 #if defined(__clang__) && !defined(__ibmxl__) # define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) @@ -69,9 +70,7 @@ # define FMT_HAS_FEATURE(x) 0 #endif -#if (defined(__has_include) || FMT_ICC_VERSION >= 1600 || \ - FMT_MSC_VERSION > 1900) && \ - !defined(__INTELLISENSE__) +#if defined(__has_include) || FMT_ICC_VERSION >= 1600 || FMT_MSC_VERSION > 1900 # define FMT_HAS_INCLUDE(x) __has_include(x) #else # define FMT_HAS_INCLUDE(x) 0 @@ -94,7 +93,7 @@ #ifndef FMT_USE_CONSTEXPR # if (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VERSION >= 1912 || \ (FMT_GCC_VERSION >= 600 && FMT_CPLUSPLUS >= 201402L)) && \ - !FMT_ICC_VERSION && !defined(__NVCC__) + !FMT_ICC_VERSION && (!defined(__NVCC__) || FMT_CPLUSPLUS >= 202002L) # define FMT_USE_CONSTEXPR 1 # else # define FMT_USE_CONSTEXPR 0 @@ -140,22 +139,7 @@ # endif #endif -#ifndef FMT_DEPRECATED -# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VERSION >= 1900 -# define FMT_DEPRECATED [[deprecated]] -# else -# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) -# define FMT_DEPRECATED __attribute__((deprecated)) -# elif FMT_MSC_VERSION -# define FMT_DEPRECATED __declspec(deprecated) -# else -# define FMT_DEPRECATED /* deprecated */ -# endif -# endif -#endif - -// [[noreturn]] is disabled on MSVC and NVCC because of bogus unreachable code -// warnings. +// Disable [[noreturn]] on MSVC/NVCC because of bogus unreachable code warnings. #if FMT_EXCEPTIONS && FMT_HAS_CPP_ATTRIBUTE(noreturn) && !FMT_MSC_VERSION && \ !defined(__NVCC__) # define FMT_NORETURN [[noreturn]] @@ -163,17 +147,6 @@ # define FMT_NORETURN #endif -#if FMT_HAS_CPP17_ATTRIBUTE(fallthrough) -# define FMT_FALLTHROUGH [[fallthrough]] -#elif defined(__clang__) -# define FMT_FALLTHROUGH [[clang::fallthrough]] -#elif FMT_GCC_VERSION >= 700 && \ - (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) -# define FMT_FALLTHROUGH [[gnu::fallthrough]] -#else -# define FMT_FALLTHROUGH -#endif - #ifndef FMT_NODISCARD # if FMT_HAS_CPP17_ATTRIBUTE(nodiscard) # define FMT_NODISCARD [[nodiscard]] @@ -182,16 +155,6 @@ # endif #endif -#ifndef FMT_USE_FLOAT -# define FMT_USE_FLOAT 1 -#endif -#ifndef FMT_USE_DOUBLE -# define FMT_USE_DOUBLE 1 -#endif -#ifndef FMT_USE_LONG_DOUBLE -# define FMT_USE_LONG_DOUBLE 1 -#endif - #ifndef FMT_INLINE # if FMT_GCC_VERSION || FMT_CLANG_VERSION # define FMT_INLINE inline __attribute__((always_inline)) @@ -200,9 +163,6 @@ # endif #endif -// An inline std::forward replacement. -#define FMT_FORWARD(...) static_cast(__VA_ARGS__) - #ifdef _MSC_VER # define FMT_UNCHECKED_ITERATOR(It) \ using _Unchecked_type = It // Mark iterator as checked. @@ -213,30 +173,26 @@ #ifndef FMT_BEGIN_NAMESPACE # define FMT_BEGIN_NAMESPACE \ namespace fmt { \ - inline namespace v9 { + inline namespace v10 { # define FMT_END_NAMESPACE \ } \ } #endif -#ifndef FMT_MODULE_EXPORT -# define FMT_MODULE_EXPORT -# define FMT_MODULE_EXPORT_BEGIN -# define FMT_MODULE_EXPORT_END -# define FMT_BEGIN_DETAIL_NAMESPACE namespace detail { -# define FMT_END_DETAIL_NAMESPACE } +#ifndef FMT_EXPORT +# define FMT_EXPORT +# define FMT_BEGIN_EXPORT +# define FMT_END_EXPORT #endif #if !defined(FMT_HEADER_ONLY) && defined(_WIN32) -# define FMT_CLASS_API FMT_MSC_WARNING(suppress : 4275) -# ifdef FMT_EXPORT +# ifdef FMT_LIB_EXPORT # define FMT_API __declspec(dllexport) # elif defined(FMT_SHARED) # define FMT_API __declspec(dllimport) # endif #else -# define FMT_CLASS_API -# if defined(FMT_EXPORT) || defined(FMT_SHARED) +# if defined(FMT_LIB_EXPORT) || defined(FMT_SHARED) # if defined(__GNUC__) || defined(__clang__) # define FMT_API __attribute__((visibility("default"))) # endif @@ -261,11 +217,13 @@ #endif #ifndef FMT_CONSTEVAL -# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \ - FMT_CPLUSPLUS >= 202002L && !defined(__apple_build_version__)) || \ - (defined(__cpp_consteval) && \ +# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \ + (!defined(__apple_build_version__) || \ + __apple_build_version__ >= 14000029L) && \ + FMT_CPLUSPLUS >= 202002L) || \ + (defined(__cpp_consteval) && \ (!FMT_MSC_VERSION || _MSC_FULL_VER >= 193030704)) -// consteval is broken in MSVC before VS2022 and Apple clang 13. +// consteval is broken in MSVC before VS2022 and Apple clang before 14. # define FMT_CONSTEVAL consteval # define FMT_HAS_CONSTEVAL # else @@ -277,7 +235,7 @@ # if defined(__cpp_nontype_template_args) && \ ((FMT_GCC_VERSION >= 903 && FMT_CPLUSPLUS >= 201709L) || \ __cpp_nontype_template_args >= 201911L) && \ - !defined(__NVCOMPILER) + !defined(__NVCOMPILER) && !defined(__LCC__) # define FMT_USE_NONTYPE_TEMPLATE_ARGS 1 # else # define FMT_USE_NONTYPE_TEMPLATE_ARGS 0 @@ -286,12 +244,12 @@ // Enable minimal optimizations for more compact code in debug mode. FMT_GCC_PRAGMA("GCC push_options") -#if !defined(__OPTIMIZE__) && !defined(__NVCOMPILER) +#if !defined(__OPTIMIZE__) && !defined(__NVCOMPILER) && !defined(__LCC__) && \ + !defined(__CUDACC__) FMT_GCC_PRAGMA("GCC optimize(\"Og\")") #endif FMT_BEGIN_NAMESPACE -FMT_MODULE_EXPORT_BEGIN // Implementations of enable_if_t and other metafunctions for older systems. template @@ -310,17 +268,10 @@ template using type_identity_t = typename type_identity::type; template using underlying_t = typename std::underlying_type::type; -template struct disjunction : std::false_type {}; -template struct disjunction

: P {}; -template -struct disjunction - : conditional_t> {}; - -template struct conjunction : std::true_type {}; -template struct conjunction

: P {}; -template -struct conjunction - : conditional_t, P1> {}; +// Checks whether T is a container with contiguous storage. +template struct is_contiguous : std::false_type {}; +template +struct is_contiguous> : std::true_type {}; struct monostate { constexpr monostate() {} @@ -332,11 +283,19 @@ struct monostate { #ifdef FMT_DOC # define FMT_ENABLE_IF(...) #else -# define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 +# define FMT_ENABLE_IF(...) fmt::enable_if_t<(__VA_ARGS__), int> = 0 #endif -FMT_BEGIN_DETAIL_NAMESPACE +// This is defined in core.h instead of format.h to avoid injecting in std. +// It is a template to avoid undesirable implicit conversions to std::byte. +#ifdef __cpp_lib_byte +template ::value)> +inline auto format_as(T b) -> unsigned char { + return static_cast(b); +} +#endif +namespace detail { // Suppresses "unused variable" warnings with the method described in // https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/. // (void)var does not work on many Intel compilers. @@ -344,7 +303,15 @@ template FMT_CONSTEXPR void ignore_unused(const T&...) {} constexpr FMT_INLINE auto is_constant_evaluated( bool default_value = false) noexcept -> bool { -#ifdef __cpp_lib_is_constant_evaluated +// Workaround for incompatibility between libstdc++ consteval-based +// std::is_constant_evaluated() implementation and clang-14. +// https://github.com/fmtlib/fmt/issues/3247 +#if FMT_CPLUSPLUS >= 202002L && defined(_GLIBCXX_RELEASE) && \ + _GLIBCXX_RELEASE >= 12 && \ + (FMT_CLANG_VERSION >= 1400 && FMT_CLANG_VERSION < 1500) + ignore_unused(default_value); + return __builtin_is_constant_evaluated(); +#elif defined(__cpp_lib_is_constant_evaluated) ignore_unused(default_value); return std::is_constant_evaluated(); #else @@ -364,12 +331,12 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line, # ifdef NDEBUG // FMT_ASSERT is not empty to avoid -Wempty-body. # define FMT_ASSERT(condition, message) \ - ::fmt::detail::ignore_unused((condition), (message)) + fmt::detail::ignore_unused((condition), (message)) # else # define FMT_ASSERT(condition, message) \ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \ ? (void)0 \ - : ::fmt::detail::assert_fail(__FILE__, __LINE__, (message))) + : fmt::detail::assert_fail(__FILE__, __LINE__, (message))) # endif #endif @@ -410,15 +377,15 @@ FMT_CONSTEXPR auto to_unsigned(Int value) -> return static_cast::type>(value); } -FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char micro[] = "\u00B5"; +FMT_CONSTEXPR inline auto is_utf8() -> bool { + FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char section[] = "\u00A7"; -constexpr auto is_utf8() -> bool { // Avoid buggy sign extensions in MSVC's constant evaluation mode (#2297). using uchar = unsigned char; - return FMT_UNICODE || (sizeof(micro) == 3 && uchar(micro[0]) == 0xC2 && - uchar(micro[1]) == 0xB5); + return FMT_UNICODE || (sizeof(section) == 3 && uchar(section[0]) == 0xC2 && + uchar(section[1]) == 0xA7); } -FMT_END_DETAIL_NAMESPACE +} // namespace detail /** An implementation of ``std::basic_string_view`` for pre-C++17. It provides a @@ -427,6 +394,7 @@ FMT_END_DETAIL_NAMESPACE compiled with a different ``-std`` option than the client code (which is not recommended). */ +FMT_EXPORT template class basic_string_view { private: const Char* data_; @@ -486,6 +454,18 @@ template class basic_string_view { size_ -= n; } + FMT_CONSTEXPR_CHAR_TRAITS bool starts_with( + basic_string_view sv) const noexcept { + return size_ >= sv.size_ && + std::char_traits::compare(data_, sv.data_, sv.size_) == 0; + } + FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(Char c) const noexcept { + return size_ >= 1 && std::char_traits::eq(*data_, c); + } + FMT_CONSTEXPR_CHAR_TRAITS bool starts_with(const Char* s) const { + return starts_with(basic_string_view(s)); + } + // Lexicographically compare this string reference to other. FMT_CONSTEXPR_CHAR_TRAITS auto compare(basic_string_view other) const -> int { size_t str_size = size_ < other.size_ ? size_ : other.size_; @@ -517,13 +497,15 @@ template class basic_string_view { } }; +FMT_EXPORT using string_view = basic_string_view; /** Specifies if ``T`` is a character type. Can be specialized by users. */ +FMT_EXPORT template struct is_char : std::false_type {}; template <> struct is_char : std::true_type {}; -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { // A base class for compile-time strings. struct compile_string {}; @@ -531,7 +513,6 @@ struct compile_string {}; template struct is_compile_string : std::is_base_of {}; -// Returns a string view of `s`. template ::value)> FMT_INLINE auto to_string_view(const Char* s) -> basic_string_view { return s; @@ -561,10 +542,10 @@ void to_string_view(...); // Specifies whether S is a string type convertible to fmt::basic_string_view. // It should be a constexpr function but MSVC 2017 fails to compile it in // enable_if and MSVC 2015 fails to compile it as an alias template. -// ADL invocation of to_string_view is DEPRECATED! +// ADL is intentionally disabled as to_string_view is not an extension point. template -struct is_string : std::is_class()))> { -}; +struct is_string + : std::is_class()))> {}; template struct char_t_impl {}; template struct char_t_impl::value>> { @@ -622,23 +603,44 @@ FMT_TYPE_CONSTANT(const void*, pointer_type); constexpr bool is_integral_type(type t) { return t > type::none_type && t <= type::last_integer_type; } - constexpr bool is_arithmetic_type(type t) { return t > type::none_type && t <= type::last_numeric_type; } +constexpr auto set(type rhs) -> int { return 1 << static_cast(rhs); } +constexpr auto in(type t, int set) -> bool { + return ((set >> static_cast(t)) & 1) != 0; +} + +// Bitsets of types. +enum { + sint_set = + set(type::int_type) | set(type::long_long_type) | set(type::int128_type), + uint_set = set(type::uint_type) | set(type::ulong_long_type) | + set(type::uint128_type), + bool_set = set(type::bool_type), + char_set = set(type::char_type), + float_set = set(type::float_type) | set(type::double_type) | + set(type::long_double_type), + string_set = set(type::string_type), + cstring_set = set(type::cstring_type), + pointer_set = set(type::pointer_type) +}; + FMT_NORETURN FMT_API void throw_format_error(const char* message); struct error_handler { constexpr error_handler() = default; - constexpr error_handler(const error_handler&) = default; // This function is intentionally not constexpr to give a compile-time error. FMT_NORETURN void on_error(const char* message) { throw_format_error(message); } }; -FMT_END_DETAIL_NAMESPACE +} // namespace detail + +/** Throws ``format_error`` with a given message. */ +using detail::throw_format_error; /** String's character type. */ template using char_t = typename detail::char_t_impl::type; @@ -650,8 +652,8 @@ template using char_t = typename detail::char_t_impl::type; You can use the ``format_parse_context`` type alias for ``char`` instead. \endrst */ -template -class basic_format_parse_context : private ErrorHandler { +FMT_EXPORT +template class basic_format_parse_context { private: basic_string_view format_str_; int next_arg_id_; @@ -660,12 +662,11 @@ class basic_format_parse_context : private ErrorHandler { public: using char_type = Char; - using iterator = typename basic_string_view::iterator; + using iterator = const Char*; explicit constexpr basic_format_parse_context( - basic_string_view format_str, ErrorHandler eh = {}, - int next_arg_id = 0) - : ErrorHandler(eh), format_str_(format_str), next_arg_id_(next_arg_id) {} + basic_string_view format_str, int next_arg_id = 0) + : format_str_(format_str), next_arg_id_(next_arg_id) {} /** Returns an iterator to the beginning of the format string range being @@ -691,7 +692,8 @@ class basic_format_parse_context : private ErrorHandler { */ FMT_CONSTEXPR auto next_arg_id() -> int { if (next_arg_id_ < 0) { - on_error("cannot switch from manual to automatic argument indexing"); + detail::throw_format_error( + "cannot switch from manual to automatic argument indexing"); return 0; } int id = next_arg_id_++; @@ -705,7 +707,8 @@ class basic_format_parse_context : private ErrorHandler { */ FMT_CONSTEXPR void check_arg_id(int id) { if (next_arg_id_ > 0) { - on_error("cannot switch from automatic to manual argument indexing"); + detail::throw_format_error( + "cannot switch from automatic to manual argument indexing"); return; } next_arg_id_ = -1; @@ -713,117 +716,50 @@ class basic_format_parse_context : private ErrorHandler { } FMT_CONSTEXPR void check_arg_id(basic_string_view) {} FMT_CONSTEXPR void check_dynamic_spec(int arg_id); - - FMT_CONSTEXPR void on_error(const char* message) { - ErrorHandler::on_error(message); - } - - constexpr auto error_handler() const -> ErrorHandler { return *this; } }; +FMT_EXPORT using format_parse_context = basic_format_parse_context; -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { // A parse context with extra data used only in compile-time checks. -template -class compile_parse_context - : public basic_format_parse_context { +template +class compile_parse_context : public basic_format_parse_context { private: int num_args_; const type* types_; - using base = basic_format_parse_context; + using base = basic_format_parse_context; public: explicit FMT_CONSTEXPR compile_parse_context( basic_string_view format_str, int num_args, const type* types, - ErrorHandler eh = {}, int next_arg_id = 0) - : base(format_str, eh, next_arg_id), num_args_(num_args), types_(types) {} + int next_arg_id = 0) + : base(format_str, next_arg_id), num_args_(num_args), types_(types) {} constexpr auto num_args() const -> int { return num_args_; } constexpr auto arg_type(int id) const -> type { return types_[id]; } FMT_CONSTEXPR auto next_arg_id() -> int { int id = base::next_arg_id(); - if (id >= num_args_) this->on_error("argument not found"); + if (id >= num_args_) throw_format_error("argument not found"); return id; } FMT_CONSTEXPR void check_arg_id(int id) { base::check_arg_id(id); - if (id >= num_args_) this->on_error("argument not found"); + if (id >= num_args_) throw_format_error("argument not found"); } using base::check_arg_id; FMT_CONSTEXPR void check_dynamic_spec(int arg_id) { + detail::ignore_unused(arg_id); +#if !defined(__LCC__) if (arg_id < num_args_ && types_ && !is_integral_type(types_[arg_id])) - this->on_error("width/precision is not integer"); - } -}; -FMT_END_DETAIL_NAMESPACE - -template -FMT_CONSTEXPR void -basic_format_parse_context::do_check_arg_id(int id) { - // Argument id is only checked at compile-time during parsing because - // formatting has its own validation. - if (detail::is_constant_evaluated() && FMT_GCC_VERSION >= 1200) { - using context = detail::compile_parse_context; - if (id >= static_cast(this)->num_args()) - on_error("argument not found"); - } -} - -template -FMT_CONSTEXPR void -basic_format_parse_context::check_dynamic_spec(int arg_id) { - if (detail::is_constant_evaluated()) { - using context = detail::compile_parse_context; - static_cast(this)->check_dynamic_spec(arg_id); + throw_format_error("width/precision is not integer"); +#endif } -} - -template class basic_format_arg; -template class basic_format_args; -template class dynamic_format_arg_store; - -// A formatter for objects of type T. -template -struct formatter { - // A deleted default constructor indicates a disabled formatter. - formatter() = delete; }; -// Specifies if T has an enabled formatter specialization. A type can be -// formattable even if it doesn't have a formatter e.g. via a conversion. -template -using has_formatter = - std::is_constructible>; - -// Checks whether T is a container with contiguous storage. -template struct is_contiguous : std::false_type {}; -template -struct is_contiguous> : std::true_type {}; - -class appender; - -FMT_BEGIN_DETAIL_NAMESPACE - -template -constexpr auto has_const_formatter_impl(T*) - -> decltype(typename Context::template formatter_type().format( - std::declval(), std::declval()), - true) { - return true; -} -template -constexpr auto has_const_formatter_impl(...) -> bool { - return false; -} -template -constexpr auto has_const_formatter() -> bool { - return has_const_formatter_impl(static_cast(nullptr)); -} - // Extracts a reference to the container from back_insert_iterator. template inline auto get_container(std::back_insert_iterator it) @@ -849,7 +785,7 @@ template U* { if (is_constant_evaluated()) return copy_str(begin, end, out); auto size = to_unsigned(end - begin); - memcpy(out, begin, size * sizeof(U)); + if (size > 0) memcpy(out, begin, size * sizeof(U)); return out + size; } @@ -892,11 +828,11 @@ template class buffer { buffer(const buffer&) = delete; void operator=(const buffer&) = delete; - auto begin() noexcept -> T* { return ptr_; } - auto end() noexcept -> T* { return ptr_ + size_; } + FMT_INLINE auto begin() noexcept -> T* { return ptr_; } + FMT_INLINE auto end() noexcept -> T* { return ptr_ + size_; } - auto begin() const noexcept -> const T* { return ptr_; } - auto end() const noexcept -> const T* { return ptr_ + size_; } + FMT_INLINE auto begin() const noexcept -> const T* { return ptr_; } + FMT_INLINE auto end() const noexcept -> const T* { return ptr_ + size_; } /** Returns the size of this buffer. */ constexpr auto size() const noexcept -> size_t { return size_; } @@ -904,10 +840,8 @@ template class buffer { /** Returns the capacity of this buffer. */ constexpr auto capacity() const noexcept -> size_t { return capacity_; } - /** Returns a pointer to the buffer data. */ + /** Returns a pointer to the buffer data (not null-terminated). */ FMT_CONSTEXPR auto data() noexcept -> T* { return ptr_; } - - /** Returns a pointer to the buffer data. */ FMT_CONSTEXPR auto data() const noexcept -> const T* { return ptr_; } /** Clears this buffer. */ @@ -1100,6 +1034,79 @@ template class counting_buffer final : public buffer { auto count() -> size_t { return count_ + this->size(); } }; +} // namespace detail + +template +FMT_CONSTEXPR void basic_format_parse_context::do_check_arg_id(int id) { + // Argument id is only checked at compile-time during parsing because + // formatting has its own validation. + if (detail::is_constant_evaluated() && + (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) { + using context = detail::compile_parse_context; + if (id >= static_cast(this)->num_args()) + detail::throw_format_error("argument not found"); + } +} + +template +FMT_CONSTEXPR void basic_format_parse_context::check_dynamic_spec( + int arg_id) { + if (detail::is_constant_evaluated() && + (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) { + using context = detail::compile_parse_context; + static_cast(this)->check_dynamic_spec(arg_id); + } +} + +FMT_EXPORT template class basic_format_arg; +FMT_EXPORT template class basic_format_args; +FMT_EXPORT template class dynamic_format_arg_store; + +// A formatter for objects of type T. +FMT_EXPORT +template +struct formatter { + // A deleted default constructor indicates a disabled formatter. + formatter() = delete; +}; + +// Specifies if T has an enabled formatter specialization. A type can be +// formattable even if it doesn't have a formatter e.g. via a conversion. +template +using has_formatter = + std::is_constructible>; + +// An output iterator that appends to a buffer. +// It is used to reduce symbol sizes for the common case. +class appender : public std::back_insert_iterator> { + using base = std::back_insert_iterator>; + + public: + using std::back_insert_iterator>::back_insert_iterator; + appender(base it) noexcept : base(it) {} + FMT_UNCHECKED_ITERATOR(appender); + + auto operator++() noexcept -> appender& { return *this; } + auto operator++(int) noexcept -> appender { return *this; } +}; + +namespace detail { + +template +constexpr auto has_const_formatter_impl(T*) + -> decltype(typename Context::template formatter_type().format( + std::declval(), std::declval()), + true) { + return true; +} +template +constexpr auto has_const_formatter_impl(...) -> bool { + return false; +} +template +constexpr auto has_const_formatter() -> bool { + return has_const_formatter_impl(static_cast(nullptr)); +} template using buffer_appender = conditional_t::value, appender, @@ -1110,29 +1117,21 @@ template auto get_buffer(OutputIt out) -> iterator_buffer { return iterator_buffer(out); } +template , Buf>::value)> +auto get_buffer(std::back_insert_iterator out) -> buffer& { + return get_container(out); +} -template -auto get_iterator(Buffer& buf) -> decltype(buf.out()) { +template +FMT_INLINE auto get_iterator(Buf& buf, OutputIt) -> decltype(buf.out()) { return buf.out(); } -template auto get_iterator(buffer& buf) -> buffer_appender { - return buffer_appender(buf); +template +auto get_iterator(buffer&, OutputIt out) -> OutputIt { + return out; } -template -struct fallback_formatter { - fallback_formatter() = delete; -}; - -// Specifies if T has an enabled fallback_formatter specialization. -template -using has_fallback_formatter = -#ifdef FMT_DEPRECATED_OSTREAM - std::is_constructible>; -#else - std::false_type; -#endif - struct view {}; template struct named_arg : view { @@ -1217,7 +1216,6 @@ constexpr auto count_statically_named_args() -> size_t { struct unformattable {}; struct unformattable_char : unformattable {}; -struct unformattable_const : unformattable {}; struct unformattable_pointer : unformattable {}; template struct string_value { @@ -1284,21 +1282,17 @@ template class value { FMT_INLINE value(const named_arg_info* args, size_t size) : named_args{args, size} {} - template FMT_CONSTEXPR FMT_INLINE value(T& val) { - using value_type = remove_cvref_t; - custom.value = const_cast(&val); + template FMT_CONSTEXPR20 FMT_INLINE value(T& val) { + using value_type = remove_const_t; + custom.value = const_cast(std::addressof(val)); // Get the formatter type through the context to allow different contexts // have different extension points, e.g. `formatter` for `format` and // `printf_formatter` for `printf`. custom.format = format_custom_arg< - value_type, - conditional_t::value, - typename Context::template formatter_type, - fallback_formatter>>; + value_type, typename Context::template formatter_type>; } value(unformattable); value(unformattable_char); - value(unformattable_const); value(unformattable_pointer); private: @@ -1315,29 +1309,25 @@ template class value { } }; -template -FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg; - // To minimize the number of types we need to deal with, long is translated // either to int or to long long depending on its size. enum { long_short = sizeof(long) == sizeof(int) }; using long_type = conditional_t; using ulong_type = conditional_t; -#ifdef __cpp_lib_byte -inline auto format_as(std::byte b) -> unsigned char { - return static_cast(b); -} -#endif - -template struct has_format_as { - template ::value&& std::is_integral::value)> - static auto check(U*) -> std::true_type; - static auto check(...) -> std::false_type; +template struct format_as_result { + template ::value || std::is_class::value)> + static auto map(U*) -> decltype(format_as(std::declval())); + static auto map(...) -> void; - enum { value = decltype(check(static_cast(nullptr)))::value }; + using type = decltype(map(static_cast(nullptr))); }; +template using format_as_t = typename format_as_result::type; + +template +struct has_format_as + : bool_constant, void>::value> {}; // Maps formatting arguments to core types. // arg_mapper reports errors by returning unformattable instead of using @@ -1414,25 +1404,6 @@ template struct arg_mapper { FMT_CONSTEXPR FMT_INLINE auto map(const T&) -> unformattable_char { return {}; } - template >::value && - !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR FMT_INLINE auto map(const T& val) - -> basic_string_view { - return basic_string_view(val); - } - template >::value && - !std::is_convertible>::value && - !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR FMT_INLINE auto map(const T& val) - -> basic_string_view { - return std_string_view(val); - } FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; } FMT_CONSTEXPR FMT_INLINE auto map(const void* val) -> const void* { @@ -1442,16 +1413,15 @@ template struct arg_mapper { return val; } - // We use SFINAE instead of a const T* parameter to avoid conflicting with - // the C array overload. + // Use SFINAE instead of a const T* parameter to avoid a conflict with the + // array overload. template < typename T, FMT_ENABLE_IF( std::is_pointer::value || std::is_member_pointer::value || std::is_function::type>::value || - (std::is_convertible::value && - !std::is_convertible::value && - !has_formatter::value))> + (std::is_array::value && + !std::is_convertible::value))> FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer { return {}; } @@ -1462,62 +1432,40 @@ template struct arg_mapper { return values; } - template ::value&& std::is_convertible::value && - !has_format_as::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(const T& val) - -> decltype(std::declval().map( - static_cast>(val))) { - return map(static_cast>(val)); - } - - template ::value && - !has_formatter::value)> - FMT_CONSTEXPR FMT_INLINE auto map(const T& val) - -> decltype(std::declval().map(format_as(T()))) { + // Only map owning types because mapping views can be unsafe. + template , + FMT_ENABLE_IF(std::is_arithmetic::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> decltype(this->map(U())) { return map(format_as(val)); } - template > - struct formattable - : bool_constant() || - !std::is_const>::value || - has_fallback_formatter::value> {}; + template > + struct formattable : bool_constant() || + (has_formatter::value && + !std::is_const::value)> {}; -#if (FMT_MSC_VERSION != 0 && FMT_MSC_VERSION < 1910) || \ - FMT_ICC_VERSION != 0 || defined(__NVCC__) - // Workaround a bug in MSVC and Intel (Issue 2746). - template FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& { - return val; - } -#else template ::value)> - FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& { + FMT_CONSTEXPR FMT_INLINE auto do_map(T& val) -> T& { return val; } template ::value)> - FMT_CONSTEXPR FMT_INLINE auto do_map(T&&) -> unformattable_const { + FMT_CONSTEXPR FMT_INLINE auto do_map(T&) -> unformattable { return {}; } -#endif - template , - FMT_ENABLE_IF(!is_string::value && !is_char::value && - !std::is_array::value && - !std::is_pointer::value && - !has_format_as::value && - (has_formatter::value || - has_fallback_formatter::value))> - FMT_CONSTEXPR FMT_INLINE auto map(T&& val) - -> decltype(this->do_map(std::forward(val))) { - return do_map(std::forward(val)); + template , + FMT_ENABLE_IF((std::is_class::value || std::is_enum::value || + std::is_union::value) && + !is_string::value && !is_char::value && + !is_named_arg::value && + !std::is_arithmetic>::value)> + FMT_CONSTEXPR FMT_INLINE auto map(T& val) -> decltype(this->do_map(val)) { + return do_map(val); } template ::value)> FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg) - -> decltype(std::declval().map(named_arg.value)) { + -> decltype(this->map(named_arg.value)) { return map(named_arg.value); } @@ -1536,53 +1484,146 @@ enum { max_packed_args = 62 / packed_arg_bits }; enum : unsigned long long { is_unpacked_bit = 1ULL << 63 }; enum : unsigned long long { has_named_args_bit = 1ULL << 62 }; -FMT_END_DETAIL_NAMESPACE +template +auto copy_str(InputIt begin, InputIt end, appender out) -> appender { + get_container(out).append(begin, end); + return out; +} +template +auto copy_str(InputIt begin, InputIt end, + std::back_insert_iterator out) + -> std::back_insert_iterator { + get_container(out).append(begin, end); + return out; +} + +template +FMT_CONSTEXPR auto copy_str(R&& rng, OutputIt out) -> OutputIt { + return detail::copy_str(rng.begin(), rng.end(), out); +} -// An output iterator that appends to a buffer. -// It is used to reduce symbol sizes for the common case. -class appender : public std::back_insert_iterator> { - using base = std::back_insert_iterator>; +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500 +// A workaround for gcc 4.8 to make void_t work in a SFINAE context. +template struct void_t_impl { using type = void; }; +template using void_t = typename void_t_impl::type; +#else +template using void_t = void; +#endif - template - friend auto get_buffer(appender out) -> detail::buffer& { - return detail::get_container(out); - } +template +struct is_output_iterator : std::false_type {}; - public: - using std::back_insert_iterator>::back_insert_iterator; - appender(base it) noexcept : base(it) {} - FMT_UNCHECKED_ITERATOR(appender); +template +struct is_output_iterator< + It, T, + void_t::iterator_category, + decltype(*std::declval() = std::declval())>> + : std::true_type {}; - auto operator++() noexcept -> appender& { return *this; } - auto operator++(int) noexcept -> appender { return *this; } -}; +template struct is_back_insert_iterator : std::false_type {}; +template +struct is_back_insert_iterator> + : std::true_type {}; -// A formatting argument. It is a trivially copyable/constructible type to -// allow storage in basic_memory_buffer. -template class basic_format_arg { +// A type-erased reference to an std::locale to avoid a heavy include. +class locale_ref { private: - detail::value value_; - detail::type type_; - - template - friend FMT_CONSTEXPR auto detail::make_arg(T&& value) - -> basic_format_arg; + const void* locale_; // A type-erased pointer to std::locale. - template - friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis, - const basic_format_arg& arg) - -> decltype(vis(0)); + public: + constexpr FMT_INLINE locale_ref() : locale_(nullptr) {} + template explicit locale_ref(const Locale& loc); - friend class basic_format_args; - friend class dynamic_format_arg_store; + explicit operator bool() const noexcept { return locale_ != nullptr; } - using char_type = typename Context::char_type; + template auto get() const -> Locale; +}; - template - friend struct detail::arg_data; +template constexpr auto encode_types() -> unsigned long long { + return 0; +} - basic_format_arg(const detail::named_arg_info* args, size_t size) - : value_(args, size) {} +template +constexpr auto encode_types() -> unsigned long long { + return static_cast(mapped_type_constant::value) | + (encode_types() << packed_arg_bits); +} + +#if defined(__cpp_if_constexpr) +// This type is intentionally undefined, only used for errors +template struct type_is_unformattable_for; +#endif + +template +FMT_CONSTEXPR FMT_INLINE auto make_arg(T& val) -> value { + using arg_type = remove_cvref_t().map(val))>; + + constexpr bool formattable_char = + !std::is_same::value; + static_assert(formattable_char, "Mixing character types is disallowed."); + + // Formatting of arbitrary pointers is disallowed. If you want to format a + // pointer cast it to `void*` or `const void*`. In particular, this forbids + // formatting of `[const] volatile char*` printed as bool by iostreams. + constexpr bool formattable_pointer = + !std::is_same::value; + static_assert(formattable_pointer, + "Formatting of non-void pointers is disallowed."); + + constexpr bool formattable = !std::is_same::value; +#if defined(__cpp_if_constexpr) + if constexpr (!formattable) { + type_is_unformattable_for _; + } +#endif + static_assert( + formattable, + "Cannot format an argument. To make type T formattable provide a " + "formatter specialization: https://fmt.dev/latest/api.html#udt"); + return {arg_mapper().map(val)}; +} + +template +FMT_CONSTEXPR auto make_arg(T& val) -> basic_format_arg { + auto arg = basic_format_arg(); + arg.type_ = mapped_type_constant::value; + arg.value_ = make_arg(val); + return arg; +} + +template +FMT_CONSTEXPR inline auto make_arg(T& val) -> basic_format_arg { + return make_arg(val); +} +} // namespace detail +FMT_BEGIN_EXPORT + +// A formatting argument. It is a trivially copyable/constructible type to +// allow storage in basic_memory_buffer. +template class basic_format_arg { + private: + detail::value value_; + detail::type type_; + + template + friend FMT_CONSTEXPR auto detail::make_arg(T& value) + -> basic_format_arg; + + template + friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis, + const basic_format_arg& arg) + -> decltype(vis(0)); + + friend class basic_format_args; + friend class dynamic_format_arg_store; + + using char_type = typename Context::char_type; + + template + friend struct detail::arg_data; + + basic_format_arg(const detail::named_arg_info* args, size_t size) + : value_(args, size) {} public: class handle { @@ -1619,6 +1660,7 @@ template class basic_format_arg { ``vis(value)`` will be called with the value of type ``double``. \endrst */ +// DEPRECATED! template FMT_CONSTEXPR FMT_INLINE auto visit_format_arg( Visitor&& vis, const basic_format_arg& arg) -> decltype(vis(0)) { @@ -1660,136 +1702,8 @@ FMT_CONSTEXPR FMT_INLINE auto visit_format_arg( return vis(monostate()); } -FMT_BEGIN_DETAIL_NAMESPACE - -template -auto copy_str(InputIt begin, InputIt end, appender out) -> appender { - get_container(out).append(begin, end); - return out; -} - -template -FMT_CONSTEXPR auto copy_str(R&& rng, OutputIt out) -> OutputIt { - return detail::copy_str(rng.begin(), rng.end(), out); -} - -#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500 -// A workaround for gcc 4.8 to make void_t work in a SFINAE context. -template struct void_t_impl { using type = void; }; -template -using void_t = typename detail::void_t_impl::type; -#else -template using void_t = void; -#endif - -template -struct is_output_iterator : std::false_type {}; - -template -struct is_output_iterator< - It, T, - void_t::iterator_category, - decltype(*std::declval() = std::declval())>> - : std::true_type {}; - -template -struct is_back_insert_iterator : std::false_type {}; -template -struct is_back_insert_iterator> - : std::true_type {}; - -template -struct is_contiguous_back_insert_iterator : std::false_type {}; -template -struct is_contiguous_back_insert_iterator> - : is_contiguous {}; -template <> -struct is_contiguous_back_insert_iterator : std::true_type {}; - -// A type-erased reference to an std::locale to avoid a heavy include. -class locale_ref { - private: - const void* locale_; // A type-erased pointer to std::locale. - - public: - constexpr locale_ref() : locale_(nullptr) {} - template explicit locale_ref(const Locale& loc); - - explicit operator bool() const noexcept { return locale_ != nullptr; } - - template auto get() const -> Locale; -}; - -template constexpr auto encode_types() -> unsigned long long { - return 0; -} - -template -constexpr auto encode_types() -> unsigned long long { - return static_cast(mapped_type_constant::value) | - (encode_types() << packed_arg_bits); -} - -template -FMT_CONSTEXPR FMT_INLINE auto make_value(T&& val) -> value { - const auto& arg = arg_mapper().map(FMT_FORWARD(val)); - - constexpr bool formattable_char = - !std::is_same::value; - static_assert(formattable_char, "Mixing character types is disallowed."); - - constexpr bool formattable_const = - !std::is_same::value; - static_assert(formattable_const, "Cannot format a const argument."); - - // Formatting of arbitrary pointers is disallowed. If you want to output - // a pointer cast it to "void *" or "const void *". In particular, this - // forbids formatting of "[const] volatile char *" which is printed as bool - // by iostreams. - constexpr bool formattable_pointer = - !std::is_same::value; - static_assert(formattable_pointer, - "Formatting of non-void pointers is disallowed."); - - constexpr bool formattable = - !std::is_same::value; - static_assert( - formattable, - "Cannot format an argument. To make type T formattable provide a " - "formatter specialization: https://fmt.dev/latest/api.html#udt"); - return {arg}; -} - -template -FMT_CONSTEXPR auto make_arg(T&& value) -> basic_format_arg { - basic_format_arg arg; - arg.type_ = mapped_type_constant::value; - arg.value_ = make_value(value); - return arg; -} - -// The type template parameter is there to avoid an ODR violation when using -// a fallback formatter in one translation unit and an implicit conversion in -// another (not recommended). -template -FMT_CONSTEXPR FMT_INLINE auto make_arg(T&& val) -> value { - return make_value(val); -} - -template -FMT_CONSTEXPR inline auto make_arg(T&& value) -> basic_format_arg { - return make_arg(value); -} -FMT_END_DETAIL_NAMESPACE - // Formatting context. template class basic_format_context { - public: - /** The character type for the output. */ - using char_type = Char; - private: OutputIt out_; basic_format_args args_; @@ -1798,31 +1712,32 @@ template class basic_format_context { public: using iterator = OutputIt; using format_arg = basic_format_arg; + using format_args = basic_format_args; using parse_context_type = basic_format_parse_context; - template using formatter_type = formatter; + template using formatter_type = formatter; + + /** The character type for the output. */ + using char_type = Char; basic_format_context(basic_format_context&&) = default; basic_format_context(const basic_format_context&) = delete; void operator=(const basic_format_context&) = delete; /** - Constructs a ``basic_format_context`` object. References to the arguments are - stored in the object so make sure they have appropriate lifetimes. + Constructs a ``basic_format_context`` object. References to the arguments + are stored in the object so make sure they have appropriate lifetimes. */ - constexpr basic_format_context( - OutputIt out, basic_format_args ctx_args, - detail::locale_ref loc = detail::locale_ref()) + constexpr basic_format_context(OutputIt out, format_args ctx_args, + detail::locale_ref loc = {}) : out_(out), args_(ctx_args), loc_(loc) {} constexpr auto arg(int id) const -> format_arg { return args_.get(id); } - FMT_CONSTEXPR auto arg(basic_string_view name) -> format_arg { + FMT_CONSTEXPR auto arg(basic_string_view name) -> format_arg { return args_.get(name); } - FMT_CONSTEXPR auto arg_id(basic_string_view name) -> int { + FMT_CONSTEXPR auto arg_id(basic_string_view name) -> int { return args_.get_id(name); } - auto args() const -> const basic_format_args& { - return args_; - } + auto args() const -> const format_args& { return args_; } FMT_CONSTEXPR auto error_handler() -> detail::error_handler { return {}; } void on_error(const char* message) { error_handler().on_error(message); } @@ -1843,16 +1758,10 @@ using buffer_context = basic_format_context, Char>; using format_context = buffer_context; -// Workaround an alias issue: https://stackoverflow.com/q/62767544/471164. -#define FMT_BUFFER_CONTEXT(Char) \ - basic_format_context, Char> - template -using is_formattable = bool_constant< - !std::is_base_of>().map( - std::declval()))>::value && - !detail::has_fallback_formatter::value>; +using is_formattable = bool_constant>() + .map(std::declval()))>::value>; /** \rst @@ -1870,7 +1779,7 @@ class format_arg_store { private: static const size_t num_args = sizeof...(Args); - static const size_t num_named_args = detail::count_named_args(); + static constexpr size_t num_named_args = detail::count_named_args(); static const bool is_packed = num_args <= detail::max_packed_args; using value_type = conditional_t, @@ -1891,16 +1800,14 @@ class format_arg_store public: template - FMT_CONSTEXPR FMT_INLINE format_arg_store(T&&... args) + FMT_CONSTEXPR FMT_INLINE format_arg_store(T&... args) : #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 basic_format_args(*this), #endif - data_{detail::make_arg< - is_packed, Context, - detail::mapped_type_constant, Context>::value>( - FMT_FORWARD(args))...} { - detail::init_named_args(data_.named_args(), 0, 0, args...); + data_{detail::make_arg(args)...} { + if (detail::const_check(num_named_args != 0)) + detail::init_named_args(data_.named_args(), 0, 0, args...); } }; @@ -1908,14 +1815,15 @@ class format_arg_store \rst Constructs a `~fmt::format_arg_store` object that contains references to arguments and can be implicitly converted to `~fmt::format_args`. `Context` - can be omitted in which case it defaults to `~fmt::context`. + can be omitted in which case it defaults to `~fmt::format_context`. See `~fmt::arg` for lifetime considerations. \endrst */ -template -constexpr auto make_format_args(Args&&... args) - -> format_arg_store...> { - return {FMT_FORWARD(args)...}; +// Arguments are taken by lvalue references to avoid some lifetime issues. +template +constexpr auto make_format_args(T&... args) + -> format_arg_store...> { + return {args...}; } /** @@ -1934,6 +1842,7 @@ inline auto arg(const Char* name, const T& arg) -> detail::named_arg { static_assert(!detail::is_named_arg(), "nested named arguments"); return {name, arg}; } +FMT_END_EXPORT /** \rst @@ -1942,7 +1851,7 @@ inline auto arg(const Char* name, const T& arg) -> detail::named_arg { ``vformat``:: void vlog(string_view format_str, format_args args); // OK - format_args args = make_format_args(42); // Error: dangling reference + format_args args = make_format_args(); // Error: dangling reference \endrst */ template class basic_format_args { @@ -2059,7 +1968,7 @@ template class basic_format_args { /** An alias to ``basic_format_args``. */ // A separate type would result in shorter symbols but break ABI compatibility // between clang and gcc on ARM (#1919). -using format_args = basic_format_args; +FMT_EXPORT using format_args = basic_format_args; // We cannot use enum classes as bit fields because of a gcc bug, so we put them // in namespaces instead (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414). @@ -2080,7 +1989,7 @@ enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, minus, plus, space}; } using sign_t = sign::type; -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { // Workaround an array initialization issue in gcc 4.8. template struct fill_t { @@ -2092,7 +2001,7 @@ template struct fill_t { public: FMT_CONSTEXPR void operator=(basic_string_view s) { auto size = s.size(); - if (size > max_size) return throw_format_error("invalid fill"); + FMT_ASSERT(size <= max_size, "invalid fill"); for (size_t i = 0; i < size; ++i) data_[i] = s[i]; size_ = static_cast(size); } @@ -2105,11 +2014,10 @@ template struct fill_t { return data_[index]; } }; -FMT_END_DETAIL_NAMESPACE +} // namespace detail enum class presentation_type : unsigned char { none, - // Integer types should go first, dec, // 'd' oct, // 'o' hex_lower, // 'x' @@ -2131,7 +2039,7 @@ enum class presentation_type : unsigned char { }; // Format specifiers for built-in and string types. -template struct basic_format_specs { +template struct format_specs { int width; int precision; presentation_type type; @@ -2141,7 +2049,7 @@ template struct basic_format_specs { bool localized : 1; detail::fill_t fill; - constexpr basic_format_specs() + constexpr format_specs() : width(0), precision(-1), type(presentation_type::none), @@ -2151,9 +2059,7 @@ template struct basic_format_specs { localized(false) {} }; -using format_specs = basic_format_specs; - -FMT_BEGIN_DETAIL_NAMESPACE +namespace detail { enum class arg_id_kind { none, index, name }; @@ -2174,7 +2080,7 @@ template struct arg_ref { arg_id_kind kind; union value { - FMT_CONSTEXPR value(int id = 0) : index{id} {} + FMT_CONSTEXPR value(int idx = 0) : index(idx) {} FMT_CONSTEXPR value(basic_string_view n) : name(n) {} int index; @@ -2183,134 +2089,30 @@ template struct arg_ref { }; // Format specifiers with width and precision resolved at formatting rather -// than parsing time to allow re-using the same parsed specifiers with +// than parsing time to allow reusing the same parsed specifiers with // different sets of arguments (precompilation of format strings). -template -struct dynamic_format_specs : basic_format_specs { +template +struct dynamic_format_specs : format_specs { arg_ref width_ref; arg_ref precision_ref; }; -struct auto_id {}; - -// A format specifier handler that sets fields in basic_format_specs. -template class specs_setter { - protected: - basic_format_specs& specs_; - - public: - explicit FMT_CONSTEXPR specs_setter(basic_format_specs& specs) - : specs_(specs) {} - - FMT_CONSTEXPR specs_setter(const specs_setter& other) - : specs_(other.specs_) {} - - FMT_CONSTEXPR void on_align(align_t align) { specs_.align = align; } - FMT_CONSTEXPR void on_fill(basic_string_view fill) { - specs_.fill = fill; - } - FMT_CONSTEXPR void on_sign(sign_t s) { specs_.sign = s; } - FMT_CONSTEXPR void on_hash() { specs_.alt = true; } - FMT_CONSTEXPR void on_localized() { specs_.localized = true; } - - FMT_CONSTEXPR void on_zero() { - if (specs_.align == align::none) specs_.align = align::numeric; - specs_.fill[0] = Char('0'); - } - - FMT_CONSTEXPR void on_width(int width) { specs_.width = width; } - FMT_CONSTEXPR void on_precision(int precision) { - specs_.precision = precision; - } - FMT_CONSTEXPR void end_precision() {} - - FMT_CONSTEXPR void on_type(presentation_type type) { specs_.type = type; } -}; - -// Format spec handler that saves references to arguments representing dynamic -// width and precision to be resolved at formatting time. -template -class dynamic_specs_handler - : public specs_setter { - public: - using char_type = typename ParseContext::char_type; - - FMT_CONSTEXPR dynamic_specs_handler(dynamic_format_specs& specs, - ParseContext& ctx) - : specs_setter(specs), specs_(specs), context_(ctx) {} - - FMT_CONSTEXPR dynamic_specs_handler(const dynamic_specs_handler& other) - : specs_setter(other), - specs_(other.specs_), - context_(other.context_) {} - - template FMT_CONSTEXPR void on_dynamic_width(Id arg_id) { - specs_.width_ref = make_arg_ref(arg_id); - } - - template FMT_CONSTEXPR void on_dynamic_precision(Id arg_id) { - specs_.precision_ref = make_arg_ref(arg_id); - } - - FMT_CONSTEXPR void on_error(const char* message) { - context_.on_error(message); - } - - private: - dynamic_format_specs& specs_; - ParseContext& context_; - - using arg_ref_type = arg_ref; - - FMT_CONSTEXPR auto make_arg_ref(int arg_id) -> arg_ref_type { - context_.check_arg_id(arg_id); - context_.check_dynamic_spec(arg_id); - return arg_ref_type(arg_id); - } - - FMT_CONSTEXPR auto make_arg_ref(auto_id) -> arg_ref_type { - int arg_id = context_.next_arg_id(); - context_.check_dynamic_spec(arg_id); - return arg_ref_type(arg_id); - } - - FMT_CONSTEXPR auto make_arg_ref(basic_string_view arg_id) - -> arg_ref_type { - context_.check_arg_id(arg_id); - basic_string_view format_str( - context_.begin(), to_unsigned(context_.end() - context_.begin())); - return arg_ref_type(arg_id); - } -}; - -template constexpr bool is_ascii_letter(Char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); -} - -// Converts a character to ASCII. Returns a number > 127 on conversion failure. +// Converts a character to ASCII. Returns '\0' on conversion failure. template ::value)> -constexpr auto to_ascii(Char c) -> Char { - return c; +constexpr auto to_ascii(Char c) -> char { + return c <= 0xff ? static_cast(c) : '\0'; } template ::value)> -constexpr auto to_ascii(Char c) -> underlying_t { - return c; -} - -FMT_CONSTEXPR inline auto code_point_length_impl(char c) -> int { - return "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4" - [static_cast(c) >> 3]; +constexpr auto to_ascii(Char c) -> char { + return c <= 0xff ? static_cast(c) : '\0'; } +// Returns the number of code units in a code point or 1 on error. template FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int { if (const_check(sizeof(Char) != 1)) return 1; - int len = code_point_length_impl(static_cast(*begin)); - - // Compute the pointer to the next character early so that the next - // iteration can start working on the next character. Neither Clang - // nor GCC figure out this reordering on their own. - return len + !len; + auto c = static_cast(*begin); + return static_cast((0x3a55000000000000ull >> (2 * (c >> 3))) & 0x3) + 1; } // Return the result via the out param to workaround gcc bug 77539. @@ -2355,279 +2157,284 @@ FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end, : error_value; } -// Parses fill and alignment. -template -FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end, - Handler&& handler) -> const Char* { - FMT_ASSERT(begin != end, ""); - auto align = align::none; - auto p = begin + code_point_length(begin); - if (end - p <= 0) p = begin; - for (;;) { - switch (to_ascii(*p)) { - case '<': - align = align::left; - break; - case '>': - align = align::right; - break; - case '^': - align = align::center; - break; - default: - break; - } - if (align != align::none) { - if (p != begin) { - auto c = *begin; - if (c == '{') - return handler.on_error("invalid fill character '{'"), begin; - handler.on_fill(basic_string_view(begin, to_unsigned(p - begin))); - begin = p + 1; - } else - ++begin; - handler.on_align(align); - break; - } else if (p == begin) { - break; - } - p = begin; +FMT_CONSTEXPR inline auto parse_align(char c) -> align_t { + switch (c) { + case '<': + return align::left; + case '>': + return align::right; + case '^': + return align::center; } - return begin; + return align::none; } -template FMT_CONSTEXPR bool is_name_start(Char c) { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; +template constexpr auto is_name_start(Char c) -> bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_'; } -template +template FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end, - IDHandler&& handler) -> const Char* { - FMT_ASSERT(begin != end, ""); + Handler&& handler) -> const Char* { Char c = *begin; if (c >= '0' && c <= '9') { int index = 0; + constexpr int max = (std::numeric_limits::max)(); if (c != '0') - index = - parse_nonnegative_int(begin, end, (std::numeric_limits::max)()); + index = parse_nonnegative_int(begin, end, max); else ++begin; if (begin == end || (*begin != '}' && *begin != ':')) - handler.on_error("invalid format string"); + throw_format_error("invalid format string"); else - handler(index); + handler.on_index(index); return begin; } if (!is_name_start(c)) { - handler.on_error("invalid format string"); + throw_format_error("invalid format string"); return begin; } auto it = begin; do { ++it; - } while (it != end && (is_name_start(c = *it) || ('0' <= c && c <= '9'))); - handler(basic_string_view(begin, to_unsigned(it - begin))); + } while (it != end && (is_name_start(*it) || ('0' <= *it && *it <= '9'))); + handler.on_name({begin, to_unsigned(it - begin)}); return it; } -template +template FMT_CONSTEXPR FMT_INLINE auto parse_arg_id(const Char* begin, const Char* end, - IDHandler&& handler) -> const Char* { + Handler&& handler) -> const Char* { + FMT_ASSERT(begin != end, ""); Char c = *begin; if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler); - handler(); + handler.on_auto(); return begin; } -template -FMT_CONSTEXPR auto parse_width(const Char* begin, const Char* end, - Handler&& handler) -> const Char* { - using detail::auto_id; - struct width_adapter { - Handler& handler; +template struct dynamic_spec_id_handler { + basic_format_parse_context& ctx; + arg_ref& ref; - FMT_CONSTEXPR void operator()() { handler.on_dynamic_width(auto_id()); } - FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_width(id); } - FMT_CONSTEXPR void operator()(basic_string_view id) { - handler.on_dynamic_width(id); - } - FMT_CONSTEXPR void on_error(const char* message) { - if (message) handler.on_error(message); - } - }; + FMT_CONSTEXPR void on_auto() { + int id = ctx.next_arg_id(); + ref = arg_ref(id); + ctx.check_dynamic_spec(id); + } + FMT_CONSTEXPR void on_index(int id) { + ref = arg_ref(id); + ctx.check_arg_id(id); + ctx.check_dynamic_spec(id); + } + FMT_CONSTEXPR void on_name(basic_string_view id) { + ref = arg_ref(id); + ctx.check_arg_id(id); + } +}; +// Parses [integer | "{" [arg_id] "}"]. +template +FMT_CONSTEXPR auto parse_dynamic_spec(const Char* begin, const Char* end, + int& value, arg_ref& ref, + basic_format_parse_context& ctx) + -> const Char* { FMT_ASSERT(begin != end, ""); if ('0' <= *begin && *begin <= '9') { - int width = parse_nonnegative_int(begin, end, -1); - if (width != -1) - handler.on_width(width); + int val = parse_nonnegative_int(begin, end, -1); + if (val != -1) + value = val; else - handler.on_error("number is too big"); + throw_format_error("number is too big"); } else if (*begin == '{') { ++begin; - if (begin != end) begin = parse_arg_id(begin, end, width_adapter{handler}); - if (begin == end || *begin != '}') - return handler.on_error("invalid format string"), begin; - ++begin; - } - return begin; -} - -template -FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end, - Handler&& handler) -> const Char* { - using detail::auto_id; - struct precision_adapter { - Handler& handler; - - FMT_CONSTEXPR void operator()() { handler.on_dynamic_precision(auto_id()); } - FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_precision(id); } - FMT_CONSTEXPR void operator()(basic_string_view id) { - handler.on_dynamic_precision(id); - } - FMT_CONSTEXPR void on_error(const char* message) { - if (message) handler.on_error(message); - } - }; - - ++begin; - auto c = begin != end ? *begin : Char(); - if ('0' <= c && c <= '9') { - auto precision = parse_nonnegative_int(begin, end, -1); - if (precision != -1) - handler.on_precision(precision); - else - handler.on_error("number is too big"); - } else if (c == '{') { - ++begin; - if (begin != end) - begin = parse_arg_id(begin, end, precision_adapter{handler}); - if (begin == end || *begin++ != '}') - return handler.on_error("invalid format string"), begin; - } else { - return handler.on_error("missing precision specifier"), begin; + auto handler = dynamic_spec_id_handler{ctx, ref}; + if (begin != end) begin = parse_arg_id(begin, end, handler); + if (begin != end && *begin == '}') return ++begin; + throw_format_error("invalid format string"); } - handler.end_precision(); return begin; } template -FMT_CONSTEXPR auto parse_presentation_type(Char type) -> presentation_type { - switch (to_ascii(type)) { - case 'd': - return presentation_type::dec; - case 'o': - return presentation_type::oct; - case 'x': - return presentation_type::hex_lower; - case 'X': - return presentation_type::hex_upper; - case 'b': - return presentation_type::bin_lower; - case 'B': - return presentation_type::bin_upper; - case 'a': - return presentation_type::hexfloat_lower; - case 'A': - return presentation_type::hexfloat_upper; - case 'e': - return presentation_type::exp_lower; - case 'E': - return presentation_type::exp_upper; - case 'f': - return presentation_type::fixed_lower; - case 'F': - return presentation_type::fixed_upper; - case 'g': - return presentation_type::general_lower; - case 'G': - return presentation_type::general_upper; - case 'c': - return presentation_type::chr; - case 's': - return presentation_type::string; - case 'p': - return presentation_type::pointer; - case '?': - return presentation_type::debug; - default: - return presentation_type::none; - } -} - -// Parses standard format specifiers and sends notifications about parsed -// components to handler. -template -FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(const Char* begin, - const Char* end, - SpecHandler&& handler) +FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end, + int& value, arg_ref& ref, + basic_format_parse_context& ctx) -> const Char* { - if (1 < end - begin && begin[1] == '}' && is_ascii_letter(*begin) && - *begin != 'L') { - presentation_type type = parse_presentation_type(*begin++); - if (type == presentation_type::none) - handler.on_error("invalid type specifier"); - handler.on_type(type); + ++begin; + if (begin == end || *begin == '}') { + throw_format_error("invalid precision"); return begin; } + return parse_dynamic_spec(begin, end, value, ref, ctx); +} - if (begin == end) return begin; - - begin = parse_align(begin, end, handler); - if (begin == end) return begin; - - // Parse sign. - switch (to_ascii(*begin)) { - case '+': - handler.on_sign(sign::plus); - ++begin; - break; - case '-': - handler.on_sign(sign::minus); - ++begin; - break; - case ' ': - handler.on_sign(sign::space); - ++begin; - break; - default: - break; - } - if (begin == end) return begin; - - if (*begin == '#') { - handler.on_hash(); - if (++begin == end) return begin; - } - - // Parse zero flag. - if (*begin == '0') { - handler.on_zero(); - if (++begin == end) return begin; - } - - begin = parse_width(begin, end, handler); - if (begin == end) return begin; +enum class state { start, align, sign, hash, zero, width, precision, locale }; - // Parse precision. - if (*begin == '.') { - begin = parse_precision(begin, end, handler); +// Parses standard format specifiers. +template +FMT_CONSTEXPR FMT_INLINE auto parse_format_specs( + const Char* begin, const Char* end, dynamic_format_specs& specs, + basic_format_parse_context& ctx, type arg_type) -> const Char* { + auto c = '\0'; + if (end - begin > 1) { + auto next = to_ascii(begin[1]); + c = parse_align(next) == align::none ? to_ascii(*begin) : '\0'; + } else { if (begin == end) return begin; + c = to_ascii(*begin); } - if (*begin == 'L') { - handler.on_localized(); - ++begin; - } + struct { + state current_state = state::start; + FMT_CONSTEXPR void operator()(state s, bool valid = true) { + if (current_state >= s || !valid) + throw_format_error("invalid format specifier"); + current_state = s; + } + } enter_state; + + using pres = presentation_type; + constexpr auto integral_set = sint_set | uint_set | bool_set | char_set; + struct { + const Char*& begin; + dynamic_format_specs& specs; + type arg_type; + + FMT_CONSTEXPR auto operator()(pres type, int set) -> const Char* { + if (!in(arg_type, set)) throw_format_error("invalid format specifier"); + specs.type = type; + return begin + 1; + } + } parse_presentation_type{begin, specs, arg_type}; - // Parse type. - if (begin != end && *begin != '}') { - presentation_type type = parse_presentation_type(*begin++); - if (type == presentation_type::none) - handler.on_error("invalid type specifier"); - handler.on_type(type); + for (;;) { + switch (c) { + case '<': + case '>': + case '^': + enter_state(state::align); + specs.align = parse_align(c); + ++begin; + break; + case '+': + case '-': + case ' ': + enter_state(state::sign, in(arg_type, sint_set | float_set)); + switch (c) { + case '+': + specs.sign = sign::plus; + break; + case '-': + specs.sign = sign::minus; + break; + case ' ': + specs.sign = sign::space; + break; + } + ++begin; + break; + case '#': + enter_state(state::hash, is_arithmetic_type(arg_type)); + specs.alt = true; + ++begin; + break; + case '0': + enter_state(state::zero); + if (!is_arithmetic_type(arg_type)) + throw_format_error("format specifier requires numeric argument"); + if (specs.align == align::none) { + // Ignore 0 if align is specified for compatibility with std::format. + specs.align = align::numeric; + specs.fill[0] = Char('0'); + } + ++begin; + break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '{': + enter_state(state::width); + begin = parse_dynamic_spec(begin, end, specs.width, specs.width_ref, ctx); + break; + case '.': + enter_state(state::precision, + in(arg_type, float_set | string_set | cstring_set)); + begin = parse_precision(begin, end, specs.precision, specs.precision_ref, + ctx); + break; + case 'L': + enter_state(state::locale, is_arithmetic_type(arg_type)); + specs.localized = true; + ++begin; + break; + case 'd': + return parse_presentation_type(pres::dec, integral_set); + case 'o': + return parse_presentation_type(pres::oct, integral_set); + case 'x': + return parse_presentation_type(pres::hex_lower, integral_set); + case 'X': + return parse_presentation_type(pres::hex_upper, integral_set); + case 'b': + return parse_presentation_type(pres::bin_lower, integral_set); + case 'B': + return parse_presentation_type(pres::bin_upper, integral_set); + case 'a': + return parse_presentation_type(pres::hexfloat_lower, float_set); + case 'A': + return parse_presentation_type(pres::hexfloat_upper, float_set); + case 'e': + return parse_presentation_type(pres::exp_lower, float_set); + case 'E': + return parse_presentation_type(pres::exp_upper, float_set); + case 'f': + return parse_presentation_type(pres::fixed_lower, float_set); + case 'F': + return parse_presentation_type(pres::fixed_upper, float_set); + case 'g': + return parse_presentation_type(pres::general_lower, float_set); + case 'G': + return parse_presentation_type(pres::general_upper, float_set); + case 'c': + return parse_presentation_type(pres::chr, integral_set); + case 's': + return parse_presentation_type(pres::string, + bool_set | string_set | cstring_set); + case 'p': + return parse_presentation_type(pres::pointer, pointer_set | cstring_set); + case '?': + return parse_presentation_type(pres::debug, + char_set | string_set | cstring_set); + case '}': + return begin; + default: { + if (*begin == '}') return begin; + // Parse fill and alignment. + auto fill_end = begin + code_point_length(begin); + if (end - fill_end <= 0) { + throw_format_error("invalid format specifier"); + return begin; + } + if (*begin == '{') { + throw_format_error("invalid fill character '{'"); + return begin; + } + auto align = parse_align(to_ascii(*fill_end)); + enter_state(state::align, align != align::none); + specs.fill = {begin, to_unsigned(fill_end - begin)}; + specs.align = align; + begin = fill_end + 1; + } + } + if (begin == end) return begin; + c = to_ascii(*begin); } - return begin; } template @@ -2637,14 +2444,11 @@ FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end, Handler& handler; int arg_id; - FMT_CONSTEXPR void operator()() { arg_id = handler.on_arg_id(); } - FMT_CONSTEXPR void operator()(int id) { arg_id = handler.on_arg_id(id); } - FMT_CONSTEXPR void operator()(basic_string_view id) { + FMT_CONSTEXPR void on_auto() { arg_id = handler.on_arg_id(); } + FMT_CONSTEXPR void on_index(int id) { arg_id = handler.on_arg_id(id); } + FMT_CONSTEXPR void on_name(basic_string_view id) { arg_id = handler.on_arg_id(id); } - FMT_CONSTEXPR void on_error(const char* message) { - if (message) handler.on_error(message); - } }; ++begin; @@ -2673,9 +2477,6 @@ FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end, template FMT_CONSTEXPR FMT_INLINE void parse_format_string( basic_string_view format_str, Handler&& handler) { - // Workaround a name-lookup bug in MSVC's modules implementation. - using detail::find; - auto begin = format_str.data(); auto end = begin + format_str.size(); if (end - begin < 32) { @@ -2735,189 +2536,46 @@ FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx) -> decltype(ctx.begin()) { using char_type = typename ParseContext::char_type; using context = buffer_context; - using stripped_type = typename strip_named_arg::type; using mapped_type = conditional_t< mapped_type_constant::value != type::custom_type, decltype(arg_mapper().map(std::declval())), - stripped_type>; - auto f = conditional_t::value, - formatter, - fallback_formatter>(); - return f.parse(ctx); -} - -template -FMT_CONSTEXPR void check_int_type_spec(presentation_type type, - ErrorHandler&& eh) { - if (type > presentation_type::bin_upper && type != presentation_type::chr) - eh.on_error("invalid type specifier"); + typename strip_named_arg::type>; +#if defined(__cpp_if_constexpr) + if constexpr (std::is_default_constructible_v< + formatter>) { + return formatter().parse(ctx); + } else { + type_is_unformattable_for _; + return ctx.begin(); + } +#else + return formatter().parse(ctx); +#endif } -// Checks char specs and returns true if the type spec is char (and not int). -template -FMT_CONSTEXPR auto check_char_specs(const basic_format_specs& specs, - ErrorHandler&& eh = {}) -> bool { +// Checks char specs and returns true iff the presentation type is char-like. +template +FMT_CONSTEXPR auto check_char_specs(const format_specs& specs) -> bool { if (specs.type != presentation_type::none && specs.type != presentation_type::chr && specs.type != presentation_type::debug) { - check_int_type_spec(specs.type, eh); return false; } if (specs.align == align::numeric || specs.sign != sign::none || specs.alt) - eh.on_error("invalid format specifier for char"); + throw_format_error("invalid format specifier for char"); return true; } -// A floating-point presentation format. -enum class float_format : unsigned char { - general, // General: exponent notation or fixed point based on magnitude. - exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. - fixed, // Fixed point with the default precision of 6, e.g. 0.0012. - hex -}; - -struct float_specs { - int precision; - float_format format : 8; - sign_t sign : 8; - bool upper : 1; - bool locale : 1; - bool binary32 : 1; - bool showpoint : 1; -}; - -template -FMT_CONSTEXPR auto parse_float_type_spec(const basic_format_specs& specs, - ErrorHandler&& eh = {}) - -> float_specs { - auto result = float_specs(); - result.showpoint = specs.alt; - result.locale = specs.localized; - switch (specs.type) { - case presentation_type::none: - result.format = float_format::general; - break; - case presentation_type::general_upper: - result.upper = true; - FMT_FALLTHROUGH; - case presentation_type::general_lower: - result.format = float_format::general; - break; - case presentation_type::exp_upper: - result.upper = true; - FMT_FALLTHROUGH; - case presentation_type::exp_lower: - result.format = float_format::exp; - result.showpoint |= specs.precision != 0; - break; - case presentation_type::fixed_upper: - result.upper = true; - FMT_FALLTHROUGH; - case presentation_type::fixed_lower: - result.format = float_format::fixed; - result.showpoint |= specs.precision != 0; - break; - case presentation_type::hexfloat_upper: - result.upper = true; - FMT_FALLTHROUGH; - case presentation_type::hexfloat_lower: - result.format = float_format::hex; - break; - default: - eh.on_error("invalid type specifier"); - break; - } - return result; -} - -template -FMT_CONSTEXPR auto check_cstring_type_spec(presentation_type type, - ErrorHandler&& eh = {}) -> bool { - if (type == presentation_type::none || type == presentation_type::string || - type == presentation_type::debug) - return true; - if (type != presentation_type::pointer) eh.on_error("invalid type specifier"); - return false; -} - -template -FMT_CONSTEXPR void check_string_type_spec(presentation_type type, - ErrorHandler&& eh = {}) { - if (type != presentation_type::none && type != presentation_type::string && - type != presentation_type::debug) - eh.on_error("invalid type specifier"); -} - -template -FMT_CONSTEXPR void check_pointer_type_spec(presentation_type type, - ErrorHandler&& eh) { - if (type != presentation_type::none && type != presentation_type::pointer) - eh.on_error("invalid type specifier"); -} - -// A parse_format_specs handler that checks if specifiers are consistent with -// the argument type. -template class specs_checker : public Handler { - private: - detail::type arg_type_; - - FMT_CONSTEXPR void require_numeric_argument() { - if (!is_arithmetic_type(arg_type_)) - this->on_error("format specifier requires numeric argument"); - } - - public: - FMT_CONSTEXPR specs_checker(const Handler& handler, detail::type arg_type) - : Handler(handler), arg_type_(arg_type) {} - - FMT_CONSTEXPR void on_align(align_t align) { - if (align == align::numeric) require_numeric_argument(); - Handler::on_align(align); - } - - FMT_CONSTEXPR void on_sign(sign_t s) { - require_numeric_argument(); - if (is_integral_type(arg_type_) && arg_type_ != type::int_type && - arg_type_ != type::long_long_type && arg_type_ != type::int128_type && - arg_type_ != type::char_type) { - this->on_error("format specifier requires signed argument"); - } - Handler::on_sign(s); - } - - FMT_CONSTEXPR void on_hash() { - require_numeric_argument(); - Handler::on_hash(); - } - - FMT_CONSTEXPR void on_localized() { - require_numeric_argument(); - Handler::on_localized(); - } - - FMT_CONSTEXPR void on_zero() { - require_numeric_argument(); - Handler::on_zero(); - } - - FMT_CONSTEXPR void end_precision() { - if (is_integral_type(arg_type_) || arg_type_ == type::pointer_type) - this->on_error("precision not allowed for this argument type"); - } -}; - -constexpr int invalid_arg_index = -1; - #if FMT_USE_NONTYPE_TEMPLATE_ARGS template constexpr auto get_arg_index_by_name(basic_string_view name) -> int { - if constexpr (detail::is_statically_named_arg()) { + if constexpr (is_statically_named_arg()) { if (name == T::name) return N; } if constexpr (sizeof...(Args) > 0) return get_arg_index_by_name(name); (void)name; // Workaround an MSVC bug about "unused" parameter. - return invalid_arg_index; + return -1; } #endif @@ -2928,34 +2586,29 @@ FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view name) -> int { return get_arg_index_by_name<0, Args...>(name); #endif (void)name; - return invalid_arg_index; + return -1; } -template -class format_string_checker { +template class format_string_checker { private: - // In the future basic_format_parse_context will replace compile_parse_context - // here and will use is_constant_evaluated and downcasting to access the data - // needed for compile-time checks: https://godbolt.org/z/GvWzcTjh1. - using parse_context_type = compile_parse_context; + using parse_context_type = compile_parse_context; static constexpr int num_args = sizeof...(Args); // Format specifier parsing function. + // In the future basic_format_parse_context will replace compile_parse_context + // here and will use is_constant_evaluated and downcasting to access the data + // needed for compile-time checks: https://godbolt.org/z/GvWzcTjh1. using parse_func = const Char* (*)(parse_context_type&); + type types_[num_args > 0 ? static_cast(num_args) : 1]; parse_context_type context_; parse_func parse_funcs_[num_args > 0 ? static_cast(num_args) : 1]; - type types_[num_args > 0 ? static_cast(num_args) : 1]; public: - explicit FMT_CONSTEXPR format_string_checker( - basic_string_view format_str, ErrorHandler eh) - : context_(format_str, num_args, types_, eh), - parse_funcs_{&parse_format_specs...}, - types_{ - mapped_type_constant>::value...} { - } + explicit FMT_CONSTEXPR format_string_checker(basic_string_view fmt) + : types_{mapped_type_constant>::value...}, + context_(fmt, num_args, types_), + parse_funcs_{&parse_format_specs...} {} FMT_CONSTEXPR void on_text(const Char*, const Char*) {} @@ -2966,8 +2619,8 @@ class format_string_checker { FMT_CONSTEXPR auto on_arg_id(basic_string_view id) -> int { #if FMT_USE_NONTYPE_TEMPLATE_ARGS auto index = get_arg_index_by_name(id); - if (index == invalid_arg_index) on_error("named argument is not found"); - return context_.check_arg_id(index), index; + if (index < 0) on_error("named argument is not found"); + return index; #else (void)id; on_error("compile-time checks for named arguments require C++20 support"); @@ -2975,17 +2628,19 @@ class format_string_checker { #endif } - FMT_CONSTEXPR void on_replacement_field(int, const Char*) {} + FMT_CONSTEXPR void on_replacement_field(int id, const Char* begin) { + on_format_specs(id, begin, begin); // Call parse() on empty specs. + } FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*) -> const Char* { - context_.advance_to(context_.begin() + (begin - &*context_.begin())); + context_.advance_to(begin); // id >= 0 check is a workaround for gcc 10 bug (#2065). return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin; } FMT_CONSTEXPR void on_error(const char* message) { - context_.on_error(message); + throw_format_error(message); } }; @@ -3001,28 +2656,33 @@ FMT_INLINE void check_format_string(const S&) { template ::value)> void check_format_string(S format_str) { - FMT_CONSTEXPR auto s = basic_string_view(format_str); - using checker = format_string_checker...>; - FMT_CONSTEXPR bool invalid_format = - (parse_format_string(s, checker(s, {})), true); - ignore_unused(invalid_format); + using char_t = typename S::char_type; + FMT_CONSTEXPR auto s = basic_string_view(format_str); + using checker = format_string_checker...>; + FMT_CONSTEXPR bool error = (parse_format_string(s, checker(s)), true); + ignore_unused(error); } +template struct vformat_args { + using type = basic_format_args< + basic_format_context>, Char>>; +}; +template <> struct vformat_args { using type = format_args; }; + +// Use vformat_args and avoid type_identity to keep symbols short. template -void vformat_to( - buffer& buf, basic_string_view fmt, - basic_format_args)> args, - locale_ref loc = {}); +void vformat_to(buffer& buf, basic_string_view fmt, + typename vformat_args::type args, locale_ref loc = {}); FMT_API void vprint_mojibake(std::FILE*, string_view, format_args); #ifndef _WIN32 inline void vprint_mojibake(std::FILE*, string_view, format_args) {} #endif -FMT_END_DETAIL_NAMESPACE +} // namespace detail + +FMT_BEGIN_EXPORT -// A formatter specialization for the core types corresponding to detail::type -// constants. +// A formatter specialization for natively supported types. template struct formatter::value != @@ -3031,81 +2691,21 @@ struct formatter specs_; public: - // Parses format specifiers stopping either at the end of the range or at the - // terminating '}'. template - FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) { - auto begin = ctx.begin(), end = ctx.end(); - if (begin == end) return begin; - using handler_type = detail::dynamic_specs_handler; + FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* { auto type = detail::type_constant::value; - auto checker = - detail::specs_checker(handler_type(specs_, ctx), type); - auto it = detail::parse_format_specs(begin, end, checker); - auto eh = ctx.error_handler(); - switch (type) { - case detail::type::none_type: - FMT_ASSERT(false, "invalid argument type"); - break; - case detail::type::bool_type: - if (specs_.type == presentation_type::none || - specs_.type == presentation_type::string) { - break; - } - FMT_FALLTHROUGH; - case detail::type::int_type: - case detail::type::uint_type: - case detail::type::long_long_type: - case detail::type::ulong_long_type: - case detail::type::int128_type: - case detail::type::uint128_type: - detail::check_int_type_spec(specs_.type, eh); - break; - case detail::type::char_type: - detail::check_char_specs(specs_, eh); - break; - case detail::type::float_type: - if (detail::const_check(FMT_USE_FLOAT)) - detail::parse_float_type_spec(specs_, eh); - else - FMT_ASSERT(false, "float support disabled"); - break; - case detail::type::double_type: - if (detail::const_check(FMT_USE_DOUBLE)) - detail::parse_float_type_spec(specs_, eh); - else - FMT_ASSERT(false, "double support disabled"); - break; - case detail::type::long_double_type: - if (detail::const_check(FMT_USE_LONG_DOUBLE)) - detail::parse_float_type_spec(specs_, eh); - else - FMT_ASSERT(false, "long double support disabled"); - break; - case detail::type::cstring_type: - detail::check_cstring_type_spec(specs_.type, eh); - break; - case detail::type::string_type: - detail::check_string_type_spec(specs_.type, eh); - break; - case detail::type::pointer_type: - detail::check_pointer_type_spec(specs_.type, eh); - break; - case detail::type::custom_type: - // Custom format specifiers are checked in parse functions of - // formatter specializations. - break; - } - return it; + auto end = + detail::parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx, type); + if (type == detail::type::char_type) detail::check_char_specs(specs_); + return end; } template ::value, - enable_if_t<(U == detail::type::string_type || - U == detail::type::cstring_type || - U == detail::type::char_type), - int> = 0> - FMT_CONSTEXPR void set_debug_format() { - specs_.type = presentation_type::debug; + FMT_ENABLE_IF(U == detail::type::string_type || + U == detail::type::cstring_type || + U == detail::type::char_type)> + FMT_CONSTEXPR void set_debug_format(bool set = true) { + specs_.type = set ? presentation_type::debug : presentation_type::none; } template @@ -3113,28 +2713,9 @@ struct formatter decltype(ctx.out()); }; -#define FMT_FORMAT_AS(Type, Base) \ - template \ - struct formatter : formatter { \ - template \ - auto format(Type const& val, FormatContext& ctx) const \ - -> decltype(ctx.out()) { \ - return formatter::format(static_cast(val), ctx); \ - } \ - } - -FMT_FORMAT_AS(signed char, int); -FMT_FORMAT_AS(unsigned char, unsigned); -FMT_FORMAT_AS(short, int); -FMT_FORMAT_AS(unsigned short, unsigned); -FMT_FORMAT_AS(long, long long); -FMT_FORMAT_AS(unsigned long, unsigned long long); -FMT_FORMAT_AS(Char*, const Char*); -FMT_FORMAT_AS(std::basic_string, basic_string_view); -FMT_FORMAT_AS(std::nullptr_t, const void*); -FMT_FORMAT_AS(detail::std_string_view, basic_string_view); - -template struct basic_runtime { basic_string_view str; }; +template struct runtime_format_string { + basic_string_view str; +}; /** A compile-time format string. */ template class basic_format_string { @@ -3154,17 +2735,18 @@ template class basic_format_string { #ifdef FMT_HAS_CONSTEVAL if constexpr (detail::count_named_args() == detail::count_statically_named_args()) { - using checker = detail::format_string_checker...>; - detail::parse_format_string(str_, checker(s, {})); + using checker = + detail::format_string_checker...>; + detail::parse_format_string(str_, checker(s)); } #else detail::check_format_string(s); #endif } - basic_format_string(basic_runtime r) : str_(r.str) {} + basic_format_string(runtime_format_string fmt) : str_(fmt.str) {} FMT_INLINE operator basic_string_view() const { return str_; } + FMT_INLINE auto get() const -> basic_string_view { return str_; } }; #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 @@ -3184,7 +2766,7 @@ using format_string = basic_format_string...>; fmt::print(fmt::runtime("{:d}"), "I am not a number"); \endrst */ -inline auto runtime(string_view s) -> basic_runtime { return {{s}}; } +inline auto runtime(string_view s) -> runtime_format_string<> { return {{s}}; } #endif FMT_API auto vformat(string_view fmt, format_args args) -> std::string; @@ -3210,10 +2792,9 @@ FMT_NODISCARD FMT_INLINE auto format(format_string fmt, T&&... args) template ::value)> auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt { - using detail::get_buffer; - auto&& buf = get_buffer(out); + auto&& buf = detail::get_buffer(out); detail::vformat_to(buf, fmt, args, {}); - return detail::get_iterator(buf); + return detail::get_iterator(buf, out); } /** @@ -3272,7 +2853,7 @@ template FMT_NODISCARD FMT_INLINE auto formatted_size(format_string fmt, T&&... args) -> size_t { auto buf = detail::counting_buffer<>(); - detail::vformat_to(buf, string_view(fmt), fmt::make_format_args(args...), {}); + detail::vformat_to(buf, fmt, fmt::make_format_args(args...), {}); return buf.count(); } @@ -3313,7 +2894,25 @@ FMT_INLINE void print(std::FILE* f, format_string fmt, T&&... args) { : detail::vprint_mojibake(f, fmt, vargs); } -FMT_MODULE_EXPORT_END +/** + Formats ``args`` according to specifications in ``fmt`` and writes the + output to the file ``f`` followed by a newline. + */ +template +FMT_INLINE void println(std::FILE* f, format_string fmt, T&&... args) { + return fmt::print(f, "{}\n", fmt::format(fmt, std::forward(args)...)); +} + +/** + Formats ``args`` according to specifications in ``fmt`` and writes the output + to ``stdout`` followed by a newline. + */ +template +FMT_INLINE void println(format_string fmt, T&&... args) { + return fmt::println(stdout, fmt, std::forward(args)...); +} + +FMT_END_EXPORT FMT_GCC_PRAGMA("GCC pop_options") FMT_END_NAMESPACE diff --git a/3rdparty/exported/fmt/format-inl.h b/3rdparty/exported/fmt/format-inl.h index 22b1ec8df0eb..dac2d437a41a 100644 --- a/3rdparty/exported/fmt/format-inl.h +++ b/3rdparty/exported/fmt/format-inl.h @@ -9,13 +9,9 @@ #define FMT_FORMAT_INL_H_ #include -#include #include // errno #include #include -#include -#include // std::memmove -#include #include #ifndef FMT_STATIC_THOUSANDS_SEPARATOR @@ -115,16 +111,43 @@ template FMT_FUNC Char decimal_point_impl(locale_ref) { return '.'; } #endif + +FMT_FUNC auto write_loc(appender out, loc_value value, + const format_specs<>& specs, locale_ref loc) -> bool { +#ifndef FMT_STATIC_THOUSANDS_SEPARATOR + auto locale = loc.get(); + // We cannot use the num_put facet because it may produce output in + // a wrong encoding. + using facet = format_facet; + if (std::has_facet(locale)) + return std::use_facet(locale).put(out, value, specs); + return facet(locale).put(out, value, specs); +#endif + return false; +} } // namespace detail -#if !FMT_MSC_VERSION -FMT_API FMT_FUNC format_error::~format_error() noexcept = default; +template typename Locale::id format_facet::id; + +#ifndef FMT_STATIC_THOUSANDS_SEPARATOR +template format_facet::format_facet(Locale& loc) { + auto& numpunct = std::use_facet>(loc); + grouping_ = numpunct.grouping(); + if (!grouping_.empty()) separator_ = std::string(1, numpunct.thousands_sep()); +} + +template <> +FMT_API FMT_FUNC auto format_facet::do_put( + appender out, loc_value val, const format_specs<>& specs) const -> bool { + return val.visit( + detail::loc_writer<>{out, specs, separator_, grouping_, decimal_point_}); +} #endif -FMT_FUNC std::system_error vsystem_error(int error_code, string_view format_str, +FMT_FUNC std::system_error vsystem_error(int error_code, string_view fmt, format_args args) { auto ec = std::error_code(error_code, std::generic_category()); - return std::system_error(ec, vformat(format_str, args)); + return std::system_error(ec, vformat(fmt, args)); } namespace detail { @@ -143,58 +166,8 @@ FMT_CONSTEXPR inline uint64_t rotr(uint64_t n, uint32_t r) noexcept { return (n >> r) | (n << (64 - r)); } -// Computes 128-bit result of multiplication of two 64-bit unsigned integers. -inline uint128_fallback umul128(uint64_t x, uint64_t y) noexcept { -#if FMT_USE_INT128 - auto p = static_cast(x) * static_cast(y); - return {static_cast(p >> 64), static_cast(p)}; -#elif defined(_MSC_VER) && defined(_M_X64) - auto result = uint128_fallback(); - result.lo_ = _umul128(x, y, &result.hi_); - return result; -#else - const uint64_t mask = static_cast(max_value()); - - uint64_t a = x >> 32; - uint64_t b = x & mask; - uint64_t c = y >> 32; - uint64_t d = y & mask; - - uint64_t ac = a * c; - uint64_t bc = b * c; - uint64_t ad = a * d; - uint64_t bd = b * d; - - uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask); - - return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32), - (intermediate << 32) + (bd & mask)}; -#endif -} - // Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox. namespace dragonbox { -// Computes upper 64 bits of multiplication of two 64-bit unsigned integers. -inline uint64_t umul128_upper64(uint64_t x, uint64_t y) noexcept { -#if FMT_USE_INT128 - auto p = static_cast(x) * static_cast(y); - return static_cast(p >> 64); -#elif defined(_MSC_VER) && defined(_M_X64) - return __umulh(x, y); -#else - return umul128(x, y).high(); -#endif -} - -// Computes upper 128 bits of multiplication of a 64-bit unsigned integer and a -// 128-bit unsigned integer. -inline uint128_fallback umul192_upper128(uint64_t x, - uint128_fallback y) noexcept { - uint128_fallback r = umul128(x, y.high()); - r += umul128_upper64(x, y.low()); - return r; -} - // Computes upper 64 bits of multiplication of a 32-bit unsigned integer and a // 64-bit unsigned integer. inline uint64_t umul96_upper64(uint32_t x, uint64_t y) noexcept { @@ -216,25 +189,13 @@ inline uint64_t umul96_lower64(uint32_t x, uint64_t y) noexcept { return x * y; } -// Computes floor(log10(pow(2, e))) for e in [-2620, 2620] using the method from -// https://fmt.dev/papers/Dragonbox.pdf#page=28, section 6.1. -inline int floor_log10_pow2(int e) noexcept { - FMT_ASSERT(e <= 2620 && e >= -2620, "too large exponent"); - static_assert((-1 >> 1) == -1, "right shift is not arithmetic"); - return (e * 315653) >> 20; -} - // Various fast log computations. -inline int floor_log2_pow10(int e) noexcept { - FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent"); - return (e * 1741647) >> 19; -} inline int floor_log10_pow2_minus_log10_4_over_3(int e) noexcept { FMT_ASSERT(e <= 2936 && e >= -2985, "too large exponent"); return (e * 631305 - 261663) >> 21; } -static constexpr struct { +FMT_INLINE_VARIABLE constexpr struct { uint32_t divisor; int shift_amount; } div_small_pow10_infos[] = {{10, 16}, {100, 16}}; @@ -288,7 +249,7 @@ inline uint64_t divide_by_10_to_kappa_plus_1(uint64_t n) noexcept { } // Various subroutines using pow10 cache -template struct cache_accessor; +template struct cache_accessor; template <> struct cache_accessor { using carrier_uint = float_info::carrier_uint; @@ -1009,8 +970,23 @@ template <> struct cache_accessor { {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6}, {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2}, {0xc5a05277621be293, 0xc7098b7305241886}, - { 0xf70867153aa2db38, - 0xb8cbee4fc66d1ea8 } + {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8}, + {0x9a65406d44a5c903, 0x737f74f1dc043329}, + {0xc0fe908895cf3b44, 0x505f522e53053ff3}, + {0xf13e34aabb430a15, 0x647726b9e7c68ff0}, + {0x96c6e0eab509e64d, 0x5eca783430dc19f6}, + {0xbc789925624c5fe0, 0xb67d16413d132073}, + {0xeb96bf6ebadf77d8, 0xe41c5bd18c57e890}, + {0x933e37a534cbaae7, 0x8e91b962f7b6f15a}, + {0xb80dc58e81fe95a1, 0x723627bbb5a4adb1}, + {0xe61136f2227e3b09, 0xcec3b1aaa30dd91d}, + {0x8fcac257558ee4e6, 0x213a4f0aa5e8a7b2}, + {0xb3bd72ed2af29e1f, 0xa988e2cd4f62d19e}, + {0xe0accfa875af45a7, 0x93eb1b80a33b8606}, + {0x8c6c01c9498d8b88, 0xbc72f130660533c4}, + {0xaf87023b9bf0ee6a, 0xeb8fad7c7f8680b5}, + { 0xdb68c2ca82ed2a05, + 0xa67398db9f6820e2 } #else {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, @@ -1034,8 +1010,8 @@ template <> struct cache_accessor { {0x8da471a9de737e24, 0x5ceaecfed289e5d3}, {0xe4d5e82392a40515, 0x0fabaf3feaa5334b}, {0xb8da1662e7b00a17, 0x3d6a751f3b936244}, - { 0x95527a5202df0ccb, - 0x0f37801e0c43ebc9 } + {0x95527a5202df0ccb, 0x0f37801e0c43ebc9}, + {0xf13e34aabb430a15, 0x647726b9e7c68ff0} #endif }; @@ -1138,8 +1114,12 @@ template <> struct cache_accessor { } }; +FMT_FUNC uint128_fallback get_cached_power(int k) noexcept { + return cache_accessor::get_cached_power(k); +} + // Various integer checks -template +template bool is_left_endpoint_integer_shorter_interval(int exponent) noexcept { const int case_shorter_interval_left_endpoint_lower_threshold = 2; const int case_shorter_interval_left_endpoint_upper_threshold = 3; @@ -1148,12 +1128,12 @@ bool is_left_endpoint_integer_shorter_interval(int exponent) noexcept { } // Remove trailing zeros from n and return the number of zeros removed (float) -FMT_INLINE int remove_trailing_zeros(uint32_t& n) noexcept { +FMT_INLINE int remove_trailing_zeros(uint32_t& n, int s = 0) noexcept { FMT_ASSERT(n != 0, ""); - const uint32_t mod_inv_5 = 0xcccccccd; - const uint32_t mod_inv_25 = mod_inv_5 * mod_inv_5; + // Modular inverse of 5 (mod 2^32): (mod_inv_5 * 5) mod 2^32 = 1. + constexpr uint32_t mod_inv_5 = 0xcccccccd; + constexpr uint32_t mod_inv_25 = 0xc28f5c29; // = mod_inv_5 * mod_inv_5 - int s = 0; while (true) { auto q = rotr(n * mod_inv_25, 2); if (q > max_value() / 100) break; @@ -1165,7 +1145,6 @@ FMT_INLINE int remove_trailing_zeros(uint32_t& n) noexcept { n = q; s |= 1; } - return s; } @@ -1179,32 +1158,17 @@ FMT_INLINE int remove_trailing_zeros(uint64_t& n) noexcept { // Is n is divisible by 10^8? if ((nm.high() & ((1ull << (90 - 64)) - 1)) == 0 && nm.low() < magic_number) { - // If yes, work with the quotient. + // If yes, work with the quotient... auto n32 = static_cast(nm.high() >> (90 - 64)); - - const uint32_t mod_inv_5 = 0xcccccccd; - const uint32_t mod_inv_25 = mod_inv_5 * mod_inv_5; - - int s = 8; - while (true) { - auto q = rotr(n32 * mod_inv_25, 2); - if (q > max_value() / 100) break; - n32 = q; - s += 2; - } - auto q = rotr(n32 * mod_inv_5, 1); - if (q <= max_value() / 10) { - n32 = q; - s |= 1; - } - + // ... and use the 32 bit variant of the function + int s = remove_trailing_zeros(n32, 8); n = n32; return s; } // If n is not divisible by 10^8, work with n itself. - const uint64_t mod_inv_5 = 0xcccccccccccccccd; - const uint64_t mod_inv_25 = mod_inv_5 * mod_inv_5; + constexpr uint64_t mod_inv_5 = 0xcccccccccccccccd; + constexpr uint64_t mod_inv_25 = 0x8f5c28f5c28f5c29; // = mod_inv_5 * mod_inv_5 int s = 0; while (true) { @@ -1223,7 +1187,7 @@ FMT_INLINE int remove_trailing_zeros(uint64_t& n) noexcept { } // The main algorithm for shorter interval case -template +template FMT_INLINE decimal_fp shorter_interval_case(int exponent) noexcept { decimal_fp ret_value; // Compute k and beta @@ -1394,17 +1358,6 @@ template decimal_fp to_decimal(T x) noexcept { return ret_value; } } // namespace dragonbox - -#ifdef _MSC_VER -FMT_FUNC auto fmt_snprintf(char* buf, size_t size, const char* fmt, ...) - -> int { - auto args = va_list(); - va_start(args, fmt); - int result = vsnprintf_s(buf, size, _TRUNCATE, fmt, args); - va_end(args); - return result; -} -#endif } // namespace detail template <> struct formatter { @@ -1413,9 +1366,8 @@ template <> struct formatter { return ctx.begin(); } - template - auto format(const detail::bigint& n, FormatContext& ctx) const -> - typename FormatContext::iterator { + auto format(const detail::bigint& n, format_context& ctx) const + -> format_context::iterator { auto out = ctx.out(); bool first = true; for (auto i = n.bigits_.size(); i > 0; --i) { @@ -1474,57 +1426,44 @@ FMT_FUNC std::string vformat(string_view fmt, format_args args) { } namespace detail { -#ifdef _WIN32 +#ifndef _WIN32 +FMT_FUNC bool write_console(std::FILE*, string_view) { return false; } +#else using dword = conditional_t; extern "C" __declspec(dllimport) int __stdcall WriteConsoleW( // void*, const void*, dword, dword*, void*); FMT_FUNC bool write_console(std::FILE* f, string_view text) { auto fd = _fileno(f); - if (_isatty(fd)) { - detail::utf8_to_utf16 u16(string_view(text.data(), text.size())); - auto written = detail::dword(); - if (detail::WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), - u16.c_str(), static_cast(u16.size()), - &written, nullptr)) { - return true; - } - } - // We return false if the file descriptor was not TTY, or it was but - // SetConsoleW failed which can happen if the output has been redirected to - // NUL. In both cases when we return false, we should attempt to do regular - // write via fwrite or std::ostream::write. - return false; + if (!_isatty(fd)) return false; + auto u16 = utf8_to_utf16(text); + auto written = dword(); + return WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), u16.c_str(), + static_cast(u16.size()), &written, nullptr) != 0; +} + +// Print assuming legacy (non-Unicode) encoding. +FMT_FUNC void vprint_mojibake(std::FILE* f, string_view fmt, format_args args) { + auto buffer = memory_buffer(); + detail::vformat_to(buffer, fmt, + basic_format_args>(args)); + fwrite_fully(buffer.data(), 1, buffer.size(), f); } #endif FMT_FUNC void print(std::FILE* f, string_view text) { -#ifdef _WIN32 - if (write_console(f, text)) return; -#endif - detail::fwrite_fully(text.data(), 1, text.size(), f); + if (!write_console(f, text)) fwrite_fully(text.data(), 1, text.size(), f); } } // namespace detail -FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { - memory_buffer buffer; - detail::vformat_to(buffer, format_str, args); +FMT_FUNC void vprint(std::FILE* f, string_view fmt, format_args args) { + auto buffer = memory_buffer(); + detail::vformat_to(buffer, fmt, args); detail::print(f, {buffer.data(), buffer.size()}); } -#ifdef _WIN32 -// Print assuming legacy (non-Unicode) encoding. -FMT_FUNC void detail::vprint_mojibake(std::FILE* f, string_view format_str, - format_args args) { - memory_buffer buffer; - detail::vformat_to(buffer, format_str, - basic_format_args>(args)); - fwrite_fully(buffer.data(), 1, buffer.size(), f); -} -#endif - -FMT_FUNC void vprint(string_view format_str, format_args args) { - vprint(stdout, format_str, args); +FMT_FUNC void vprint(string_view fmt, format_args args) { + vprint(stdout, fmt, args); } namespace detail { diff --git a/3rdparty/exported/fmt/format.h b/3rdparty/exported/fmt/format.h index 7c607dbd3042..87a34b972ce6 100644 --- a/3rdparty/exported/fmt/format.h +++ b/3rdparty/exported/fmt/format.h @@ -33,13 +33,14 @@ #ifndef FMT_FORMAT_H_ #define FMT_FORMAT_H_ -#include // std::signbit -#include // uint32_t -#include // std::memcpy -#include // std::numeric_limits -#include // std::uninitialized_copy -#include // std::runtime_error -#include // std::system_error +#include // std::signbit +#include // uint32_t +#include // std::memcpy +#include // std::initializer_list +#include // std::numeric_limits +#include // std::uninitialized_copy +#include // std::runtime_error +#include // std::system_error #ifdef __cpp_lib_bit_cast # include // std::bitcast @@ -47,16 +48,55 @@ #include "core.h" -#if FMT_GCC_VERSION -# define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden"))) +#if defined __cpp_inline_variables && __cpp_inline_variables >= 201606L +# define FMT_INLINE_VARIABLE inline #else -# define FMT_GCC_VISIBILITY_HIDDEN +# define FMT_INLINE_VARIABLE #endif -#ifdef __NVCC__ -# define FMT_CUDA_VERSION (__CUDACC_VER_MAJOR__ * 100 + __CUDACC_VER_MINOR__) +#if FMT_HAS_CPP17_ATTRIBUTE(fallthrough) +# define FMT_FALLTHROUGH [[fallthrough]] +#elif defined(__clang__) +# define FMT_FALLTHROUGH [[clang::fallthrough]] +#elif FMT_GCC_VERSION >= 700 && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) +# define FMT_FALLTHROUGH [[gnu::fallthrough]] #else -# define FMT_CUDA_VERSION 0 +# define FMT_FALLTHROUGH +#endif + +#ifndef FMT_DEPRECATED +# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VERSION >= 1900 +# define FMT_DEPRECATED [[deprecated]] +# else +# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) +# define FMT_DEPRECATED __attribute__((deprecated)) +# elif FMT_MSC_VERSION +# define FMT_DEPRECATED __declspec(deprecated) +# else +# define FMT_DEPRECATED /* deprecated */ +# endif +# endif +#endif + +#ifndef FMT_NO_UNIQUE_ADDRESS +# if FMT_CPLUSPLUS >= 202002L +# if FMT_HAS_CPP_ATTRIBUTE(no_unique_address) +# define FMT_NO_UNIQUE_ADDRESS [[no_unique_address]] +// VS2019 v16.10 and later except clang-cl (https://reviews.llvm.org/D110485) +# elif (FMT_MSC_VERSION >= 1929) && !FMT_CLANG_VERSION +# define FMT_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]] +# endif +# endif +#endif +#ifndef FMT_NO_UNIQUE_ADDRESS +# define FMT_NO_UNIQUE_ADDRESS +#endif + +#if FMT_GCC_VERSION || defined(__clang__) +# define FMT_VISIBILITY(value) __attribute__((visibility(value))) +#else +# define FMT_VISIBILITY(value) #endif #ifdef __has_builtin @@ -71,12 +111,6 @@ # define FMT_NOINLINE #endif -#if FMT_MSC_VERSION -# define FMT_MSC_DEFAULT = default -#else -# define FMT_MSC_DEFAULT -#endif - #ifndef FMT_THROW # if FMT_EXCEPTIONS # if FMT_MSC_VERSION || defined(__NVCC__) @@ -95,10 +129,8 @@ FMT_END_NAMESPACE # define FMT_THROW(x) throw x # endif # else -# define FMT_THROW(x) \ - do { \ - FMT_ASSERT(false, (x).what()); \ - } while (false) +# define FMT_THROW(x) \ + ::fmt::detail::assert_fail(__FILE__, __LINE__, (x).what()) # endif #endif @@ -200,7 +232,8 @@ inline auto clzll(uint64_t x) -> int { _BitScanReverse64(&r, x); # else // Scan the high 32 bits. - if (_BitScanReverse(&r, static_cast(x >> 32))) return 63 ^ (r + 32); + if (_BitScanReverse(&r, static_cast(x >> 32))) + return 63 ^ static_cast(r + 32); // Scan the low 32 bits. _BitScanReverse(&r, static_cast(x)); # endif @@ -240,6 +273,19 @@ FMT_END_NAMESPACE #endif FMT_BEGIN_NAMESPACE + +template struct disjunction : std::false_type {}; +template struct disjunction

: P {}; +template +struct disjunction + : conditional_t> {}; + +template struct conjunction : std::true_type {}; +template struct conjunction

: P {}; +template +struct conjunction + : conditional_t, P1> {}; + namespace detail { FMT_CONSTEXPR inline void abort_fuzzing_if(bool condition) { @@ -323,8 +369,6 @@ class uint128_fallback { private: uint64_t lo_, hi_; - friend uint128_fallback umul128(uint64_t x, uint64_t y) noexcept; - public: constexpr uint128_fallback(uint64_t hi, uint64_t lo) : lo_(lo), hi_(hi) {} constexpr uint128_fallback(uint64_t value = 0) : lo_(value), hi_(0) {} @@ -359,6 +403,10 @@ class uint128_fallback { -> uint128_fallback { return {lhs.hi_ & rhs.hi_, lhs.lo_ & rhs.lo_}; } + friend constexpr auto operator~(const uint128_fallback& n) + -> uint128_fallback { + return {~n.hi_, ~n.lo_}; + } friend auto operator+(const uint128_fallback& lhs, const uint128_fallback& rhs) -> uint128_fallback { auto result = uint128_fallback(lhs); @@ -397,6 +445,10 @@ class uint128_fallback { lo_ = new_lo; hi_ = new_hi; } + FMT_CONSTEXPR void operator&=(uint128_fallback n) { + lo_ &= n.lo_; + hi_ &= n.hi_; + } FMT_CONSTEXPR20 uint128_fallback& operator+=(uint64_t n) noexcept { if (is_constant_evaluated()) { @@ -463,10 +515,34 @@ inline auto bit_cast(const From& from) -> To { return result; } +template +FMT_CONSTEXPR20 inline auto countl_zero_fallback(UInt n) -> int { + int lz = 0; + constexpr UInt msb_mask = static_cast(1) << (num_bits() - 1); + for (; (n & msb_mask) == 0; n <<= 1) lz++; + return lz; +} + +FMT_CONSTEXPR20 inline auto countl_zero(uint32_t n) -> int { +#ifdef FMT_BUILTIN_CLZ + if (!is_constant_evaluated()) return FMT_BUILTIN_CLZ(n); +#endif + return countl_zero_fallback(n); +} + +FMT_CONSTEXPR20 inline auto countl_zero(uint64_t n) -> int { +#ifdef FMT_BUILTIN_CLZLL + if (!is_constant_evaluated()) return FMT_BUILTIN_CLZLL(n); +#endif + return countl_zero_fallback(n); +} + FMT_INLINE void assume(bool condition) { (void)condition; #if FMT_HAS_BUILTIN(__builtin_assume) && !FMT_ICC_VERSION __builtin_assume(condition); +#elif FMT_GCC_VERSION + if (!condition) __builtin_unreachable(); #endif } @@ -485,20 +561,6 @@ inline auto get_data(Container& c) -> typename Container::value_type* { return c.data(); } -#if defined(_SECURE_SCL) && _SECURE_SCL -// Make a checked iterator to avoid MSVC warnings. -template using checked_ptr = stdext::checked_array_iterator; -template -constexpr auto make_checked(T* p, size_t size) -> checked_ptr { - return {p, size}; -} -#else -template using checked_ptr = T*; -template constexpr auto make_checked(T* p, size_t) -> T* { - return p; -} -#endif - // Attempts to reserve space for n extra characters in the output range. // Returns a pointer to the reserved range or a reference to it. template ::value)> @@ -506,12 +568,12 @@ template ::value)> __attribute__((no_sanitize("undefined"))) #endif inline auto -reserve(std::back_insert_iterator it, size_t n) - -> checked_ptr { +reserve(std::back_insert_iterator it, size_t n) -> + typename Container::value_type* { Container& c = get_container(it); size_t size = c.size(); c.resize(size + n); - return make_checked(get_data(c) + size, n); + return get_data(c) + size; } template @@ -543,8 +605,8 @@ template auto to_pointer(buffer_appender it, size_t n) -> T* { } template ::value)> -inline auto base_iterator(std::back_insert_iterator& it, - checked_ptr) +inline auto base_iterator(std::back_insert_iterator it, + typename Container::value_type*) -> std::back_insert_iterator { return it; } @@ -607,7 +669,8 @@ FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e) constexpr const int shiftc[] = {0, 18, 12, 6, 0}; constexpr const int shifte[] = {0, 6, 4, 2, 0}; - int len = code_point_length_impl(*s); + int len = "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4" + [static_cast(*s) >> 3]; // Compute the pointer to the next character early so that the next // iteration can start working on the next character. Neither Clang // nor GCC figure out this reordering on their own. @@ -636,7 +699,7 @@ FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e) return next; } -constexpr uint32_t invalid_code_point = ~uint32_t(); +constexpr FMT_INLINE_VARIABLE uint32_t invalid_code_point = ~uint32_t(); // Invokes f(cp, sv) for every code point cp in s with sv being the string view // corresponding to the code point. cp is invalid_code_point on error. @@ -706,6 +769,7 @@ FMT_CONSTEXPR inline size_t compute_width(string_view s) { return true; } }; + // We could avoid branches by using utf8_decode directly. for_each_codepoint(s, count_code_points{&num_code_points}); return num_code_points; } @@ -737,13 +801,48 @@ inline auto code_point_index(basic_string_view s, size_t n) string_view(reinterpret_cast(s.data()), s.size()), n); } +template struct is_integral : std::is_integral {}; +template <> struct is_integral : std::true_type {}; +template <> struct is_integral : std::true_type {}; + +template +using is_signed = + std::integral_constant::is_signed || + std::is_same::value>; + +template +using is_integer = + bool_constant::value && !std::is_same::value && + !std::is_same::value && + !std::is_same::value>; + +#ifndef FMT_USE_FLOAT +# define FMT_USE_FLOAT 1 +#endif +#ifndef FMT_USE_DOUBLE +# define FMT_USE_DOUBLE 1 +#endif +#ifndef FMT_USE_LONG_DOUBLE +# define FMT_USE_LONG_DOUBLE 1 +#endif + #ifndef FMT_USE_FLOAT128 -# ifdef __SIZEOF_FLOAT128__ -# define FMT_USE_FLOAT128 1 -# else +# ifdef __clang__ +// Clang emulates GCC, so it has to appear early. +# if FMT_HAS_INCLUDE() +# define FMT_USE_FLOAT128 1 +# endif +# elif defined(__GNUC__) +// GNU C++: +# if defined(_GLIBCXX_USE_FLOAT128) && !defined(__STRICT_ANSI__) +# define FMT_USE_FLOAT128 1 +# endif +# endif +# ifndef FMT_USE_FLOAT128 # define FMT_USE_FLOAT128 0 # endif #endif + #if FMT_USE_FLOAT128 using float128 = __float128; #else @@ -775,7 +874,7 @@ void buffer::append(const U* begin, const U* end) { try_reserve(size_ + count); auto free_cap = capacity_ - size_; if (free_cap < count) count = free_cap; - std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count)); + std::uninitialized_copy_n(begin, count, ptr_ + size_); size_ += count; begin += count; } @@ -787,7 +886,7 @@ template struct is_locale> : std::true_type {}; } // namespace detail -FMT_MODULE_EXPORT_BEGIN +FMT_BEGIN_EXPORT // The number of characters to store in the basic_memory_buffer object itself // to avoid dynamic memory allocation. @@ -820,8 +919,8 @@ class basic_memory_buffer final : public detail::buffer { private: T store_[SIZE]; - // Don't inherit from Allocator avoid generating type_info for it. - Allocator alloc_; + // Don't inherit from Allocator to avoid generating type_info for it. + FMT_NO_UNIQUE_ADDRESS Allocator alloc_; // Deallocate memory allocated by the buffer. FMT_CONSTEXPR20 void deallocate() { @@ -830,7 +929,28 @@ class basic_memory_buffer final : public detail::buffer { } protected: - FMT_CONSTEXPR20 void grow(size_t size) override; + FMT_CONSTEXPR20 void grow(size_t size) override { + detail::abort_fuzzing_if(size > 5000); + const size_t max_size = std::allocator_traits::max_size(alloc_); + size_t old_capacity = this->capacity(); + size_t new_capacity = old_capacity + old_capacity / 2; + if (size > new_capacity) + new_capacity = size; + else if (new_capacity > max_size) + new_capacity = size > max_size ? size : max_size; + T* old_data = this->data(); + T* new_data = + std::allocator_traits::allocate(alloc_, new_capacity); + // Suppress a bogus -Wstringop-overflow in gcc 13.1 (#3481). + detail::assume(this->size() <= new_capacity); + // The following code doesn't throw, so the raw pointer above doesn't leak. + std::uninitialized_copy_n(old_data, this->size(), new_data); + this->set(new_data, new_capacity); + // deallocate must not throw according to the standard, but even if it does, + // the buffer already uses the new storage and will deallocate it in + // destructor. + if (old_data != store_) alloc_.deallocate(old_data, old_capacity); + } public: using value_type = T; @@ -852,8 +972,7 @@ class basic_memory_buffer final : public detail::buffer { size_t size = other.size(), capacity = other.capacity(); if (data == other.store_) { this->set(store_, capacity); - detail::copy_str(other.store_, other.store_ + size, - detail::make_checked(store_, capacity)); + detail::copy_str(other.store_, other.store_ + size, store_); } else { this->set(data, capacity); // Set pointer to the inline array so that delete is not called @@ -907,55 +1026,29 @@ class basic_memory_buffer final : public detail::buffer { } }; -template -FMT_CONSTEXPR20 void basic_memory_buffer::grow( - size_t size) { - detail::abort_fuzzing_if(size > 5000); - const size_t max_size = std::allocator_traits::max_size(alloc_); - size_t old_capacity = this->capacity(); - size_t new_capacity = old_capacity + old_capacity / 2; - if (size > new_capacity) - new_capacity = size; - else if (new_capacity > max_size) - new_capacity = size > max_size ? size : max_size; - T* old_data = this->data(); - T* new_data = - std::allocator_traits::allocate(alloc_, new_capacity); - // The following code doesn't throw, so the raw pointer above doesn't leak. - std::uninitialized_copy(old_data, old_data + this->size(), - detail::make_checked(new_data, new_capacity)); - this->set(new_data, new_capacity); - // deallocate must not throw according to the standard, but even if it does, - // the buffer already uses the new storage and will deallocate it in - // destructor. - if (old_data != store_) alloc_.deallocate(old_data, old_capacity); -} - using memory_buffer = basic_memory_buffer; template struct is_contiguous> : std::true_type { }; +FMT_END_EXPORT namespace detail { -#ifdef _WIN32 FMT_API bool write_console(std::FILE* f, string_view text); -#endif FMT_API void print(std::FILE*, string_view); } // namespace detail -/** A formatting error such as invalid format string. */ -FMT_CLASS_API -class FMT_API format_error : public std::runtime_error { +FMT_BEGIN_EXPORT + +// Suppress a misleading warning in older versions of clang. +#if FMT_CLANG_VERSION +# pragma clang diagnostic ignored "-Wweak-vtables" +#endif + +/** An error reported from a formatting function. */ +class FMT_VISIBILITY("default") format_error : public std::runtime_error { public: - explicit format_error(const char* message) : std::runtime_error(message) {} - explicit format_error(const std::string& message) - : std::runtime_error(message) {} - format_error(const format_error&) = default; - format_error& operator=(const format_error&) = default; - format_error(format_error&&) = default; - format_error& operator=(format_error&&) = default; - ~format_error() noexcept override FMT_MSC_DEFAULT; + using std::runtime_error::runtime_error; }; namespace detail_exported { @@ -984,16 +1077,52 @@ constexpr auto compile_string_to_view(detail::std_string_view s) } } // namespace detail_exported -FMT_BEGIN_DETAIL_NAMESPACE +class loc_value { + private: + basic_format_arg value_; -template struct is_integral : std::is_integral {}; -template <> struct is_integral : std::true_type {}; -template <> struct is_integral : std::true_type {}; + public: + template ::value)> + loc_value(T value) : value_(detail::make_arg(value)) {} -template -using is_signed = - std::integral_constant::is_signed || - std::is_same::value>; + template ::value)> + loc_value(T) {} + + template auto visit(Visitor&& vis) -> decltype(vis(0)) { + return visit_format_arg(vis, value_); + } +}; + +// A locale facet that formats values in UTF-8. +// It is parameterized on the locale to avoid the heavy include. +template class format_facet : public Locale::facet { + private: + std::string separator_; + std::string grouping_; + std::string decimal_point_; + + protected: + virtual auto do_put(appender out, loc_value val, + const format_specs<>& specs) const -> bool; + + public: + static FMT_API typename Locale::id id; + + explicit format_facet(Locale& loc); + explicit format_facet(string_view sep = "", + std::initializer_list g = {3}, + std::string decimal_point = ".") + : separator_(sep.data(), sep.size()), + grouping_(g.begin(), g.end()), + decimal_point_(decimal_point) {} + + auto put(appender out, loc_value val, const format_specs<>& specs) const + -> bool { + return do_put(out, val, specs); + } +}; + +namespace detail { // Returns true if value is negative, false otherwise. // Same as `value < 0` but doesn't produce warnings if T is an unsigned type. @@ -1122,7 +1251,7 @@ FMT_CONSTEXPR auto count_digits(UInt n) -> int { FMT_INLINE auto do_count_digits(uint32_t n) -> int { // An optimization by Kendall Willets from https://bit.ly/3uOIQrB. // This increments the upper 32 bits (log10(T) - 1) when >= T is added. -# define FMT_INC(T) (((sizeof(# T) - 1ull) << 32) - T) +# define FMT_INC(T) (((sizeof(#T) - 1ull) << 32) - T) static constexpr uint64_t table[] = { FMT_INC(0), FMT_INC(0), FMT_INC(0), // 8 FMT_INC(10), FMT_INC(10), FMT_INC(10), // 64 @@ -1238,7 +1367,7 @@ template format_decimal_result { // Buffer is large enough to hold all digits (digits10 + 1). - Char buffer[digits10() + 1]; + Char buffer[digits10() + 1] = {}; auto end = format_decimal(buffer, value, size).end; return {out, detail::copy_str_noinline(buffer, end, out)}; } @@ -1258,8 +1387,8 @@ FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits, } template -inline auto format_uint(It out, UInt value, int num_digits, bool upper = false) - -> It { +FMT_CONSTEXPR inline auto format_uint(It out, UInt value, int num_digits, + bool upper = false) -> It { if (auto ptr = to_pointer(out, to_unsigned(num_digits))) { format_uint(ptr, value, num_digits, upper); return out; @@ -1283,7 +1412,139 @@ class utf8_to_utf16 { auto str() const -> std::wstring { return {&buffer_[0], size()}; } }; +enum class to_utf8_error_policy { abort, replace }; + +// A converter from UTF-16/UTF-32 (host endian) to UTF-8. +template class to_utf8 { + private: + Buffer buffer_; + + public: + to_utf8() {} + explicit to_utf8(basic_string_view s, + to_utf8_error_policy policy = to_utf8_error_policy::abort) { + static_assert(sizeof(WChar) == 2 || sizeof(WChar) == 4, + "Expect utf16 or utf32"); + if (!convert(s, policy)) + FMT_THROW(std::runtime_error(sizeof(WChar) == 2 ? "invalid utf16" + : "invalid utf32")); + } + operator string_view() const { return string_view(&buffer_[0], size()); } + size_t size() const { return buffer_.size() - 1; } + const char* c_str() const { return &buffer_[0]; } + std::string str() const { return std::string(&buffer_[0], size()); } + + // Performs conversion returning a bool instead of throwing exception on + // conversion error. This method may still throw in case of memory allocation + // error. + bool convert(basic_string_view s, + to_utf8_error_policy policy = to_utf8_error_policy::abort) { + if (!convert(buffer_, s, policy)) return false; + buffer_.push_back(0); + return true; + } + static bool convert( + Buffer& buf, basic_string_view s, + to_utf8_error_policy policy = to_utf8_error_policy::abort) { + for (auto p = s.begin(); p != s.end(); ++p) { + uint32_t c = static_cast(*p); + if (sizeof(WChar) == 2 && c >= 0xd800 && c <= 0xdfff) { + // Handle a surrogate pair. + ++p; + if (p == s.end() || (c & 0xfc00) != 0xd800 || (*p & 0xfc00) != 0xdc00) { + if (policy == to_utf8_error_policy::abort) return false; + buf.append(string_view("\xEF\xBF\xBD")); + --p; + } else { + c = (c << 10) + static_cast(*p) - 0x35fdc00; + } + } else if (c < 0x80) { + buf.push_back(static_cast(c)); + } else if (c < 0x800) { + buf.push_back(static_cast(0xc0 | (c >> 6))); + buf.push_back(static_cast(0x80 | (c & 0x3f))); + } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) { + buf.push_back(static_cast(0xe0 | (c >> 12))); + buf.push_back(static_cast(0x80 | ((c & 0xfff) >> 6))); + buf.push_back(static_cast(0x80 | (c & 0x3f))); + } else if (c >= 0x10000 && c <= 0x10ffff) { + buf.push_back(static_cast(0xf0 | (c >> 18))); + buf.push_back(static_cast(0x80 | ((c & 0x3ffff) >> 12))); + buf.push_back(static_cast(0x80 | ((c & 0xfff) >> 6))); + buf.push_back(static_cast(0x80 | (c & 0x3f))); + } else { + return false; + } + } + return true; + } +}; + +// Computes 128-bit result of multiplication of two 64-bit unsigned integers. +inline uint128_fallback umul128(uint64_t x, uint64_t y) noexcept { +#if FMT_USE_INT128 + auto p = static_cast(x) * static_cast(y); + return {static_cast(p >> 64), static_cast(p)}; +#elif defined(_MSC_VER) && defined(_M_X64) + auto hi = uint64_t(); + auto lo = _umul128(x, y, &hi); + return {hi, lo}; +#else + const uint64_t mask = static_cast(max_value()); + + uint64_t a = x >> 32; + uint64_t b = x & mask; + uint64_t c = y >> 32; + uint64_t d = y & mask; + + uint64_t ac = a * c; + uint64_t bc = b * c; + uint64_t ad = a * d; + uint64_t bd = b * d; + + uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask); + + return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32), + (intermediate << 32) + (bd & mask)}; +#endif +} + namespace dragonbox { +// Computes floor(log10(pow(2, e))) for e in [-2620, 2620] using the method from +// https://fmt.dev/papers/Dragonbox.pdf#page=28, section 6.1. +inline int floor_log10_pow2(int e) noexcept { + FMT_ASSERT(e <= 2620 && e >= -2620, "too large exponent"); + static_assert((-1 >> 1) == -1, "right shift is not arithmetic"); + return (e * 315653) >> 20; +} + +inline int floor_log2_pow10(int e) noexcept { + FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent"); + return (e * 1741647) >> 19; +} + +// Computes upper 64 bits of multiplication of two 64-bit unsigned integers. +inline uint64_t umul128_upper64(uint64_t x, uint64_t y) noexcept { +#if FMT_USE_INT128 + auto p = static_cast(x) * static_cast(y); + return static_cast(p >> 64); +#elif defined(_MSC_VER) && defined(_M_X64) + return __umulh(x, y); +#else + return umul128(x, y).high(); +#endif +} + +// Computes upper 128 bits of multiplication of a 64-bit unsigned integer and a +// 128-bit unsigned integer. +inline uint128_fallback umul192_upper128(uint64_t x, + uint128_fallback y) noexcept { + uint128_fallback r = umul128(x, y.high()); + r += umul128_upper64(x, y.low()); + return r; +} + +FMT_API uint128_fallback get_cached_power(int k) noexcept; // Type-specific information that Dragonbox uses. template struct float_info; @@ -1307,7 +1568,7 @@ template <> struct float_info { static const int big_divisor = 1000; static const int small_divisor = 100; static const int min_k = -292; - static const int max_k = 326; + static const int max_k = 341; static const int shorter_interval_tie_lower_threshold = -77; static const int shorter_interval_tie_upper_threshold = -77; }; @@ -1354,8 +1615,8 @@ template constexpr int num_significand_bits() { template constexpr auto exponent_mask() -> typename dragonbox::float_info::carrier_uint { - using uint = typename dragonbox::float_info::carrier_uint; - return ((uint(1) << dragonbox::float_info::exponent_bits) - 1) + using float_uint = typename dragonbox::float_info::carrier_uint; + return ((float_uint(1) << dragonbox::float_info::exponent_bits) - 1) << num_significand_bits(); } template constexpr auto exponent_bias() -> int { @@ -1476,157 +1737,31 @@ FMT_CONSTEXPR inline fp operator*(fp x, fp y) { } template struct basic_data { - // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340. - // These are generated by support/compute-powers.py. - static constexpr uint64_t pow10_significands[87] = { - 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76, - 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df, - 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c, - 0x8dd01fad907ffc3c, 0xd3515c2831559a83, 0x9d71ac8fada6c9b5, - 0xea9c227723ee8bcb, 0xaecc49914078536d, 0x823c12795db6ce57, - 0xc21094364dfb5637, 0x9096ea6f3848984f, 0xd77485cb25823ac7, - 0xa086cfcd97bf97f4, 0xef340a98172aace5, 0xb23867fb2a35b28e, - 0x84c8d4dfd2c63f3b, 0xc5dd44271ad3cdba, 0x936b9fcebb25c996, - 0xdbac6c247d62a584, 0xa3ab66580d5fdaf6, 0xf3e2f893dec3f126, - 0xb5b5ada8aaff80b8, 0x87625f056c7c4a8b, 0xc9bcff6034c13053, - 0x964e858c91ba2655, 0xdff9772470297ebd, 0xa6dfbd9fb8e5b88f, - 0xf8a95fcf88747d94, 0xb94470938fa89bcf, 0x8a08f0f8bf0f156b, - 0xcdb02555653131b6, 0x993fe2c6d07b7fac, 0xe45c10c42a2b3b06, - 0xaa242499697392d3, 0xfd87b5f28300ca0e, 0xbce5086492111aeb, - 0x8cbccc096f5088cc, 0xd1b71758e219652c, 0x9c40000000000000, - 0xe8d4a51000000000, 0xad78ebc5ac620000, 0x813f3978f8940984, - 0xc097ce7bc90715b3, 0x8f7e32ce7bea5c70, 0xd5d238a4abe98068, - 0x9f4f2726179a2245, 0xed63a231d4c4fb27, 0xb0de65388cc8ada8, - 0x83c7088e1aab65db, 0xc45d1df942711d9a, 0x924d692ca61be758, - 0xda01ee641a708dea, 0xa26da3999aef774a, 0xf209787bb47d6b85, - 0xb454e4a179dd1877, 0x865b86925b9bc5c2, 0xc83553c5c8965d3d, - 0x952ab45cfa97a0b3, 0xde469fbd99a05fe3, 0xa59bc234db398c25, - 0xf6c69a72a3989f5c, 0xb7dcbf5354e9bece, 0x88fcf317f22241e2, - 0xcc20ce9bd35c78a5, 0x98165af37b2153df, 0xe2a0b5dc971f303a, - 0xa8d9d1535ce3b396, 0xfb9b7cd9a4a7443c, 0xbb764c4ca7a44410, - 0x8bab8eefb6409c1a, 0xd01fef10a657842c, 0x9b10a4e5e9913129, - 0xe7109bfba19c0c9d, 0xac2820d9623bf429, 0x80444b5e7aa7cf85, - 0xbf21e44003acdd2d, 0x8e679c2f5e44ff8f, 0xd433179d9c8cb841, - 0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b, + // For checking rounding thresholds. + // The kth entry is chosen to be the smallest integer such that the + // upper 32-bits of 10^(k+1) times it is strictly bigger than 5 * 10^k. + static constexpr uint32_t fractional_part_rounding_thresholds[8] = { + 2576980378U, // ceil(2^31 + 2^32/10^1) + 2190433321U, // ceil(2^31 + 2^32/10^2) + 2151778616U, // ceil(2^31 + 2^32/10^3) + 2147913145U, // ceil(2^31 + 2^32/10^4) + 2147526598U, // ceil(2^31 + 2^32/10^5) + 2147487943U, // ceil(2^31 + 2^32/10^6) + 2147484078U, // ceil(2^31 + 2^32/10^7) + 2147483691U // ceil(2^31 + 2^32/10^8) }; - -#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnarrowing" -#endif - // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding - // to significands above. - static constexpr int16_t pow10_exponents[87] = { - -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, - -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661, - -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369, - -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77, - -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216, - 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508, - 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, - 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066}; -#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 -# pragma GCC diagnostic pop -#endif - - static constexpr uint64_t power_of_10_64[20] = { - 1, FMT_POWERS_OF_10(1ULL), FMT_POWERS_OF_10(1000000000ULL), - 10000000000000000000ULL}; }; - -#if FMT_CPLUSPLUS < 201703L -template constexpr uint64_t basic_data::pow10_significands[]; -template constexpr int16_t basic_data::pow10_exponents[]; -template constexpr uint64_t basic_data::power_of_10_64[]; -#endif - // This is a struct rather than an alias to avoid shadowing warnings in gcc. struct data : basic_data<> {}; -// Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its -// (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`. -FMT_CONSTEXPR inline fp get_cached_power(int min_exponent, - int& pow10_exponent) { - const int shift = 32; - // log10(2) = 0x0.4d104d427de7fbcc... - const int64_t significand = 0x4d104d427de7fbcc; - int index = static_cast( - ((min_exponent + fp::num_significand_bits - 1) * (significand >> shift) + - ((int64_t(1) << shift) - 1)) // ceil - >> 32 // arithmetic shift - ); - // Decimal exponent of the first (smallest) cached power of 10. - const int first_dec_exp = -348; - // Difference between 2 consecutive decimal exponents in cached powers of 10. - const int dec_exp_step = 8; - index = (index - first_dec_exp - 1) / dec_exp_step + 1; - pow10_exponent = first_dec_exp + index * dec_exp_step; - // Using *(x + index) instead of x[index] avoids an issue with some compilers - // using the EDG frontend (e.g. nvhpc/22.3 in C++17 mode). - return {*(data::pow10_significands + index), - *(data::pow10_exponents + index)}; -} - -#ifndef _MSC_VER -# define FMT_SNPRINTF snprintf -#else -FMT_API auto fmt_snprintf(char* buf, size_t size, const char* fmt, ...) -> int; -# define FMT_SNPRINTF fmt_snprintf -#endif // _MSC_VER - -// Formats a floating-point number with snprintf using the hexfloat format. +#if FMT_CPLUSPLUS < 201703L template -auto snprintf_float(T value, int precision, float_specs specs, - buffer& buf) -> int { - // Buffer capacity must be non-zero, otherwise MSVC's vsnprintf_s will fail. - FMT_ASSERT(buf.capacity() > buf.size(), "empty buffer"); - FMT_ASSERT(specs.format == float_format::hex, ""); - static_assert(!std::is_same::value, ""); - - // Build the format string. - char format[7]; // The longest format is "%#.*Le". - char* format_ptr = format; - *format_ptr++ = '%'; - if (specs.showpoint) *format_ptr++ = '#'; - if (precision >= 0) { - *format_ptr++ = '.'; - *format_ptr++ = '*'; - } - if (std::is_same()) *format_ptr++ = 'L'; - *format_ptr++ = specs.upper ? 'A' : 'a'; - *format_ptr = '\0'; - - // Format using snprintf. - auto offset = buf.size(); - for (;;) { - auto begin = buf.data() + offset; - auto capacity = buf.capacity() - offset; - abort_fuzzing_if(precision > 100000); - // Suppress the warning about a nonliteral format string. - // Cannot use auto because of a bug in MinGW (#1532). - int (*snprintf_ptr)(char*, size_t, const char*, ...) = FMT_SNPRINTF; - int result = precision >= 0 - ? snprintf_ptr(begin, capacity, format, precision, value) - : snprintf_ptr(begin, capacity, format, value); - if (result < 0) { - // The buffer will grow exponentially. - buf.try_reserve(buf.capacity() + 1); - continue; - } - auto size = to_unsigned(result); - // Size equal to capacity means that the last character was truncated. - if (size < capacity) { - buf.try_resize(size + offset); - return 0; - } - buf.try_reserve(size + offset + 1); // Add 1 for the terminating '\0'. - } -} +constexpr uint32_t basic_data::fractional_part_rounding_thresholds[]; +#endif -template +template () == num_bits()> using convert_float_result = - conditional_t::value || sizeof(T) == sizeof(double), - double, T>; + conditional_t::value || doublish, double, T>; template constexpr auto convert_float(T value) -> convert_float_result { @@ -1649,8 +1784,7 @@ FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n, // width: output display width in (terminal) column positions. template -FMT_CONSTEXPR auto write_padded(OutputIt out, - const basic_format_specs& specs, +FMT_CONSTEXPR auto write_padded(OutputIt out, const format_specs& specs, size_t size, size_t width, F&& f) -> OutputIt { static_assert(align == align::left || align == align::right, ""); unsigned spec_width = to_unsigned(specs.width); @@ -1669,15 +1803,14 @@ FMT_CONSTEXPR auto write_padded(OutputIt out, template -constexpr auto write_padded(OutputIt out, const basic_format_specs& specs, +constexpr auto write_padded(OutputIt out, const format_specs& specs, size_t size, F&& f) -> OutputIt { return write_padded(out, specs, size, size, f); } template FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes, - const basic_format_specs& specs) - -> OutputIt { + const format_specs& specs) -> OutputIt { return write_padded( out, specs, bytes.size(), [bytes](reserve_iterator it) { const char* data = bytes.data(); @@ -1686,8 +1819,8 @@ FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes, } template -auto write_ptr(OutputIt out, UIntPtr value, - const basic_format_specs* specs) -> OutputIt { +auto write_ptr(OutputIt out, UIntPtr value, const format_specs* specs) + -> OutputIt { int num_digits = count_digits<4>(value); auto size = to_unsigned(num_digits) + size_t(2); auto write = [=](reserve_iterator it) { @@ -1749,7 +1882,7 @@ inline auto find_escape(const char* begin, const char* end) [] { \ /* Use the hidden visibility as a workaround for a GCC bug (#1973). */ \ /* Use a macro-like name to avoid shadowing warnings. */ \ - struct FMT_GCC_VISIBILITY_HIDDEN FMT_COMPILE_STRING : base { \ + struct FMT_VISIBILITY("hidden") FMT_COMPILE_STRING : base { \ using char_type FMT_MAYBE_UNUSED = fmt::remove_cvref_t; \ FMT_MAYBE_UNUSED FMT_CONSTEXPR explicit \ operator fmt::basic_string_view() const { \ @@ -1806,16 +1939,14 @@ auto write_escaped_cp(OutputIt out, const find_escape_result& escape) *out++ = static_cast('\\'); break; default: - if (is_utf8()) { - if (escape.cp < 0x100) { - return write_codepoint<2, Char>(out, 'x', escape.cp); - } - if (escape.cp < 0x10000) { - return write_codepoint<4, Char>(out, 'u', escape.cp); - } - if (escape.cp < 0x110000) { - return write_codepoint<8, Char>(out, 'U', escape.cp); - } + if (escape.cp < 0x100) { + return write_codepoint<2, Char>(out, 'x', escape.cp); + } + if (escape.cp < 0x10000) { + return write_codepoint<4, Char>(out, 'u', escape.cp); + } + if (escape.cp < 0x110000) { + return write_codepoint<8, Char>(out, 'U', escape.cp); } for (Char escape_char : basic_string_view( escape.begin, to_unsigned(escape.end - escape.begin))) { @@ -1860,8 +1991,7 @@ auto write_escaped_char(OutputIt out, Char v) -> OutputIt { template FMT_CONSTEXPR auto write_char(OutputIt out, Char value, - const basic_format_specs& specs) - -> OutputIt { + const format_specs& specs) -> OutputIt { bool is_debug = specs.type == presentation_type::debug; return write_padded(out, specs, 1, [=](reserve_iterator it) { if (is_debug) return write_escaped_char(it, value); @@ -1871,11 +2001,14 @@ FMT_CONSTEXPR auto write_char(OutputIt out, Char value, } template FMT_CONSTEXPR auto write(OutputIt out, Char value, - const basic_format_specs& specs, - locale_ref loc = {}) -> OutputIt { + const format_specs& specs, locale_ref loc = {}) + -> OutputIt { + // char is formatted as unsigned char for consistency across platforms. + using unsigned_type = + conditional_t::value, unsigned char, unsigned>; return check_char_specs(specs) ? write_char(out, value, specs) - : write(out, static_cast(value), specs, loc); + : write(out, static_cast(value), specs, loc); } // Data for write_int that doesn't depend on output iterator type. It is used to @@ -1885,7 +2018,7 @@ template struct write_int_data { size_t padding; FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix, - const basic_format_specs& specs) + const format_specs& specs) : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) { if (specs.align == align::numeric) { auto width = to_unsigned(specs.width); @@ -1907,7 +2040,7 @@ template struct write_int_data { template FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits, unsigned prefix, - const basic_format_specs& specs, + const format_specs& specs, W write_digits) -> OutputIt { // Slightly faster check for specs.width == 0 && specs.precision == -1. if ((specs.width | (specs.precision + 1)) == 0) { @@ -1930,19 +2063,19 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits, template class digit_grouping { private: - thousands_sep_result sep_; + std::string grouping_; + std::basic_string thousands_sep_; struct next_state { std::string::const_iterator group; int pos; }; - next_state initial_state() const { return {sep_.grouping.begin(), 0}; } + next_state initial_state() const { return {grouping_.begin(), 0}; } // Returns the next digit group separator position. int next(next_state& state) const { - if (!sep_.thousands_sep) return max_value(); - if (state.group == sep_.grouping.end()) - return state.pos += sep_.grouping.back(); + if (thousands_sep_.empty()) return max_value(); + if (state.group == grouping_.end()) return state.pos += grouping_.back(); if (*state.group <= 0 || *state.group == max_value()) return max_value(); state.pos += *state.group++; @@ -1951,14 +2084,15 @@ template class digit_grouping { public: explicit digit_grouping(locale_ref loc, bool localized = true) { - if (localized) - sep_ = thousands_sep(loc); - else - sep_.thousands_sep = Char(); + if (!localized) return; + auto sep = thousands_sep(loc); + grouping_ = sep.grouping; + if (sep.thousands_sep) thousands_sep_.assign(1, sep.thousands_sep); } - explicit digit_grouping(thousands_sep_result sep) : sep_(sep) {} + digit_grouping(std::string grouping, std::basic_string sep) + : grouping_(std::move(grouping)), thousands_sep_(std::move(sep)) {} - Char separator() const { return sep_.thousands_sep; } + bool has_separator() const { return !thousands_sep_.empty(); } int count_separators(int num_digits) const { int count = 0; @@ -1981,7 +2115,9 @@ template class digit_grouping { for (int i = 0, sep_index = static_cast(separators.size() - 1); i < num_digits; ++i) { if (num_digits - i == separators[sep_index]) { - *out++ = separator(); + out = + copy_str(thousands_sep_.data(), + thousands_sep_.data() + thousands_sep_.size(), out); --sep_index; } *out++ = static_cast(digits[to_unsigned(i)]); @@ -1990,10 +2126,11 @@ template class digit_grouping { } }; +// Writes a decimal integer with digit grouping. template -auto write_int_localized(OutputIt out, UInt value, unsigned prefix, - const basic_format_specs& specs, - const digit_grouping& grouping) -> OutputIt { +auto write_int(OutputIt out, UInt value, unsigned prefix, + const format_specs& specs, + const digit_grouping& grouping) -> OutputIt { static_assert(std::is_same, UInt>::value, ""); int num_digits = count_digits(value); char digits[40]; @@ -2010,13 +2147,13 @@ auto write_int_localized(OutputIt out, UInt value, unsigned prefix, }); } -template -auto write_int_localized(OutputIt& out, UInt value, unsigned prefix, - const basic_format_specs& specs, locale_ref loc) - -> bool { - auto grouping = digit_grouping(loc); - out = write_int_localized(out, value, prefix, specs, grouping); - return true; +// Writes a localized value. +FMT_API auto write_loc(appender out, loc_value value, + const format_specs<>& specs, locale_ref loc) -> bool; +template +inline auto write_loc(OutputIt, loc_value, const format_specs&, + locale_ref) -> bool { + return false; } FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) { @@ -2045,21 +2182,37 @@ FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign) return {abs_value, prefix}; } +template struct loc_writer { + buffer_appender out; + const format_specs& specs; + std::basic_string sep; + std::string grouping; + std::basic_string decimal_point; + + template ::value)> + auto operator()(T value) -> bool { + auto arg = make_write_int_arg(value, specs.sign); + write_int(out, static_cast>(arg.abs_value), arg.prefix, + specs, digit_grouping(grouping, sep)); + return true; + } + + template ::value)> + auto operator()(T) -> bool { + return false; + } +}; + template FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg arg, - const basic_format_specs& specs, - locale_ref loc) -> OutputIt { + const format_specs& specs, + locale_ref) -> OutputIt { static_assert(std::is_same>::value, ""); auto abs_value = arg.abs_value; auto prefix = arg.prefix; switch (specs.type) { case presentation_type::none: case presentation_type::dec: { - if (specs.localized && - write_int_localized(out, static_cast>(abs_value), - prefix, specs, loc)) { - return out; - } auto num_digits = count_digits(abs_value); return write_int( out, num_digits, prefix, specs, [=](reserve_iterator it) { @@ -2102,13 +2255,13 @@ FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg arg, case presentation_type::chr: return write_char(out, static_cast(abs_value), specs); default: - throw_format_error("invalid type specifier"); + throw_format_error("invalid format specifier"); } return out; } template FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline( - OutputIt out, write_int_arg arg, const basic_format_specs& specs, + OutputIt out, write_int_arg arg, const format_specs& specs, locale_ref loc) -> OutputIt { return write_int(out, arg, specs, loc); } @@ -2117,8 +2270,9 @@ template ::value && std::is_same>::value)> FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value, - const basic_format_specs& specs, + const format_specs& specs, locale_ref loc) -> OutputIt { + if (specs.localized && write_loc(out, value, specs, loc)) return out; return write_int_noinline(out, make_write_int_arg(value, specs.sign), specs, loc); } @@ -2128,8 +2282,9 @@ template ::value && !std::is_same>::value)> FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value, - const basic_format_specs& specs, + const format_specs& specs, locale_ref loc) -> OutputIt { + if (specs.localized && write_loc(out, value, specs, loc)) return out; return write_int(out, make_write_int_arg(value, specs.sign), specs, loc); } @@ -2175,7 +2330,7 @@ class counting_iterator { template FMT_CONSTEXPR auto write(OutputIt out, basic_string_view s, - const basic_format_specs& specs) -> OutputIt { + const format_specs& specs) -> OutputIt { auto data = s.data(); auto size = s.size(); if (specs.precision >= 0 && to_unsigned(specs.precision) < size) @@ -2197,16 +2352,15 @@ FMT_CONSTEXPR auto write(OutputIt out, basic_string_view s, template FMT_CONSTEXPR auto write(OutputIt out, basic_string_view> s, - const basic_format_specs& specs, locale_ref) + const format_specs& specs, locale_ref) -> OutputIt { - check_string_type_spec(specs.type); return write(out, s, specs); } template FMT_CONSTEXPR auto write(OutputIt out, const Char* s, - const basic_format_specs& specs, locale_ref) + const format_specs& specs, locale_ref) -> OutputIt { - return check_cstring_type_spec(specs.type) + return specs.type != presentation_type::pointer ? write(out, basic_string_view(s), specs, {}) : write_ptr(out, bit_cast(s), &specs); } @@ -2233,9 +2387,114 @@ FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { return base_iterator(out, it); } +// DEPRECATED! +template +FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end, + format_specs& specs) -> const Char* { + FMT_ASSERT(begin != end, ""); + auto align = align::none; + auto p = begin + code_point_length(begin); + if (end - p <= 0) p = begin; + for (;;) { + switch (to_ascii(*p)) { + case '<': + align = align::left; + break; + case '>': + align = align::right; + break; + case '^': + align = align::center; + break; + } + if (align != align::none) { + if (p != begin) { + auto c = *begin; + if (c == '}') return begin; + if (c == '{') { + throw_format_error("invalid fill character '{'"); + return begin; + } + specs.fill = {begin, to_unsigned(p - begin)}; + begin = p + 1; + } else { + ++begin; + } + break; + } else if (p == begin) { + break; + } + p = begin; + } + specs.align = align; + return begin; +} + +// A floating-point presentation format. +enum class float_format : unsigned char { + general, // General: exponent notation or fixed point based on magnitude. + exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. + fixed, // Fixed point with the default precision of 6, e.g. 0.0012. + hex +}; + +struct float_specs { + int precision; + float_format format : 8; + sign_t sign : 8; + bool upper : 1; + bool locale : 1; + bool binary32 : 1; + bool showpoint : 1; +}; + +template +FMT_CONSTEXPR auto parse_float_type_spec(const format_specs& specs, + ErrorHandler&& eh = {}) + -> float_specs { + auto result = float_specs(); + result.showpoint = specs.alt; + result.locale = specs.localized; + switch (specs.type) { + case presentation_type::none: + result.format = float_format::general; + break; + case presentation_type::general_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::general_lower: + result.format = float_format::general; + break; + case presentation_type::exp_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::exp_lower: + result.format = float_format::exp; + result.showpoint |= specs.precision != 0; + break; + case presentation_type::fixed_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::fixed_lower: + result.format = float_format::fixed; + result.showpoint |= specs.precision != 0; + break; + case presentation_type::hexfloat_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::hexfloat_lower: + result.format = float_format::hex; + break; + default: + eh.on_error("invalid format specifier"); + break; + } + return result; +} + template FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isnan, - basic_format_specs specs, + format_specs specs, const float_specs& fspecs) -> OutputIt { auto str = isnan ? (fspecs.upper ? "NAN" : "nan") : (fspecs.upper ? "INF" : "inf"); @@ -2281,7 +2540,7 @@ template FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand, int significand_size, int exponent, const Grouping& grouping) -> OutputIt { - if (!grouping.separator()) { + if (!grouping.has_separator()) { out = write_significand(out, significand, significand_size); return detail::fill_n(out, exponent, static_cast('0')); } @@ -2343,7 +2602,7 @@ FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand, int significand_size, int integral_size, Char decimal_point, const Grouping& grouping) -> OutputIt { - if (!grouping.separator()) { + if (!grouping.has_separator()) { return write_significand(out, significand, significand_size, integral_size, decimal_point); } @@ -2359,7 +2618,7 @@ FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand, template > FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f, - const basic_format_specs& specs, + const format_specs& specs, float_specs fspecs, locale_ref loc) -> OutputIt { auto significand = f.significand; @@ -2418,7 +2677,7 @@ FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f, abort_fuzzing_if(num_zeros > 5000); if (fspecs.showpoint) { ++size; - if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 1; + if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 0; if (num_zeros > 0) size += to_unsigned(num_zeros); } auto grouping = Grouping(loc, fspecs.locale); @@ -2436,7 +2695,7 @@ FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f, int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0; size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0); auto grouping = Grouping(loc, fspecs.locale); - size += to_unsigned(grouping.count_separators(significand_size)); + size += to_unsigned(grouping.count_separators(exp)); return write_padded(out, specs, size, [&](iterator it) { if (sign) *it++ = detail::sign(sign); it = write_significand(it, significand, significand_size, exp, @@ -2466,7 +2725,7 @@ template class fallback_digit_grouping { public: constexpr fallback_digit_grouping(locale_ref, bool) {} - constexpr Char separator() const { return Char(); } + constexpr bool has_separator() const { return false; } constexpr int count_separators(int) const { return 0; } @@ -2478,7 +2737,7 @@ template class fallback_digit_grouping { template FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& f, - const basic_format_specs& specs, + const format_specs& specs, float_specs fspecs, locale_ref loc) -> OutputIt { if (is_constant_evaluated()) { @@ -2506,14 +2765,14 @@ template ::value&& FMT_CONSTEXPR20 bool isfinite(T value) { constexpr T inf = T(std::numeric_limits::infinity()); if (is_constant_evaluated()) - return !detail::isnan(value) && value != inf && value != -inf; + return !detail::isnan(value) && value < inf && value > -inf; return std::isfinite(value); } template ::value)> FMT_CONSTEXPR bool isfinite(T value) { T inf = T(std::numeric_limits::infinity()); // std::isfinite doesn't support __float128. - return !detail::isnan(value) && value != inf && value != -inf; + return !detail::isnan(value) && value < inf && value > -inf; } template ::value)> @@ -2529,78 +2788,6 @@ FMT_INLINE FMT_CONSTEXPR bool signbit(T value) { return std::signbit(static_cast(value)); } -enum class round_direction { unknown, up, down }; - -// Given the divisor (normally a power of 10), the remainder = v % divisor for -// some number v and the error, returns whether v should be rounded up, down, or -// whether the rounding direction can't be determined due to error. -// error should be less than divisor / 2. -FMT_CONSTEXPR inline round_direction get_round_direction(uint64_t divisor, - uint64_t remainder, - uint64_t error) { - FMT_ASSERT(remainder < divisor, ""); // divisor - remainder won't overflow. - FMT_ASSERT(error < divisor, ""); // divisor - error won't overflow. - FMT_ASSERT(error < divisor - error, ""); // error * 2 won't overflow. - // Round down if (remainder + error) * 2 <= divisor. - if (remainder <= divisor - remainder && error * 2 <= divisor - remainder * 2) - return round_direction::down; - // Round up if (remainder - error) * 2 >= divisor. - if (remainder >= error && - remainder - error >= divisor - (remainder - error)) { - return round_direction::up; - } - return round_direction::unknown; -} - -namespace digits { -enum result { - more, // Generate more digits. - done, // Done generating digits. - error // Digit generation cancelled due to an error. -}; -} - -struct gen_digits_handler { - char* buf; - int size; - int precision; - int exp10; - bool fixed; - - FMT_CONSTEXPR digits::result on_digit(char digit, uint64_t divisor, - uint64_t remainder, uint64_t error, - bool integral) { - FMT_ASSERT(remainder < divisor, ""); - buf[size++] = digit; - if (!integral && error >= remainder) return digits::error; - if (size < precision) return digits::more; - if (!integral) { - // Check if error * 2 < divisor with overflow prevention. - // The check is not needed for the integral part because error = 1 - // and divisor > (1 << 32) there. - if (error >= divisor || error >= divisor - error) return digits::error; - } else { - FMT_ASSERT(error == 1 && divisor > 2, ""); - } - auto dir = get_round_direction(divisor, remainder, error); - if (dir != round_direction::up) - return dir == round_direction::down ? digits::done : digits::error; - ++buf[size - 1]; - for (int i = size - 1; i > 0 && buf[i] > '9'; --i) { - buf[i] = '0'; - ++buf[i - 1]; - } - if (buf[0] > '9') { - buf[0] = '1'; - if (fixed) - buf[size++] = '0'; - else - ++exp10; - } - return digits::done; - } -}; - inline FMT_CONSTEXPR20 void adjust_precision(int& precision, int exp10) { // Adjust fixed precision by exponent because it is relative to decimal // point. @@ -2609,101 +2796,6 @@ inline FMT_CONSTEXPR20 void adjust_precision(int& precision, int exp10) { precision += exp10; } -// Generates output using the Grisu digit-gen algorithm. -// error: the size of the region (lower, upper) outside of which numbers -// definitely do not round to value (Delta in Grisu3). -FMT_INLINE FMT_CONSTEXPR20 auto grisu_gen_digits(fp value, uint64_t error, - int& exp, - gen_digits_handler& handler) - -> digits::result { - const fp one(1ULL << -value.e, value.e); - // The integral part of scaled value (p1 in Grisu) = value / one. It cannot be - // zero because it contains a product of two 64-bit numbers with MSB set (due - // to normalization) - 1, shifted right by at most 60 bits. - auto integral = static_cast(value.f >> -one.e); - FMT_ASSERT(integral != 0, ""); - FMT_ASSERT(integral == value.f >> -one.e, ""); - // The fractional part of scaled value (p2 in Grisu) c = value % one. - uint64_t fractional = value.f & (one.f - 1); - exp = count_digits(integral); // kappa in Grisu. - // Non-fixed formats require at least one digit and no precision adjustment. - if (handler.fixed) { - adjust_precision(handler.precision, exp + handler.exp10); - // Check if precision is satisfied just by leading zeros, e.g. - // format("{:.2f}", 0.001) gives "0.00" without generating any digits. - if (handler.precision <= 0) { - if (handler.precision < 0) return digits::done; - // Divide by 10 to prevent overflow. - uint64_t divisor = data::power_of_10_64[exp - 1] << -one.e; - auto dir = get_round_direction(divisor, value.f / 10, error * 10); - if (dir == round_direction::unknown) return digits::error; - handler.buf[handler.size++] = dir == round_direction::up ? '1' : '0'; - return digits::done; - } - } - // Generate digits for the integral part. This can produce up to 10 digits. - do { - uint32_t digit = 0; - auto divmod_integral = [&](uint32_t divisor) { - digit = integral / divisor; - integral %= divisor; - }; - // This optimization by Milo Yip reduces the number of integer divisions by - // one per iteration. - switch (exp) { - case 10: - divmod_integral(1000000000); - break; - case 9: - divmod_integral(100000000); - break; - case 8: - divmod_integral(10000000); - break; - case 7: - divmod_integral(1000000); - break; - case 6: - divmod_integral(100000); - break; - case 5: - divmod_integral(10000); - break; - case 4: - divmod_integral(1000); - break; - case 3: - divmod_integral(100); - break; - case 2: - divmod_integral(10); - break; - case 1: - digit = integral; - integral = 0; - break; - default: - FMT_ASSERT(false, "invalid number of digits"); - } - --exp; - auto remainder = (static_cast(integral) << -one.e) + fractional; - auto result = handler.on_digit(static_cast('0' + digit), - data::power_of_10_64[exp] << -one.e, - remainder, error, true); - if (result != digits::more) return result; - } while (exp > 0); - // Generate digits for the fractional part. - for (;;) { - fractional *= 10; - error *= 10; - char digit = static_cast('0' + (fractional >> -one.e)); - fractional &= one.f - 1; - --exp; - auto result = handler.on_digit(digit, one.f, fractional, error, false); - if (result != digits::more) return result; - } -} - class bigint { private: // A bigint is stored as an array of bigits (big digits), with bigit at index @@ -2804,7 +2896,7 @@ class bigint { auto size = other.bigits_.size(); bigits_.resize(size); auto data = other.bigits_.data(); - std::copy(data, data + size, make_checked(bigits_.data(), size)); + copy_str(data, data + size, bigits_.data()); exp_ = other.exp_; } @@ -3018,6 +3110,7 @@ FMT_CONSTEXPR20 inline void format_dragon(basic_fp value, } int even = static_cast((value.f & 1) == 0); if (!upper) upper = &lower; + bool shortest = num_digits < 0; if ((flags & dragon::fixup) != 0) { if (add_compare(numerator, *upper, denominator) + even <= 0) { --exp10; @@ -3030,7 +3123,7 @@ FMT_CONSTEXPR20 inline void format_dragon(basic_fp value, if ((flags & dragon::fixed) != 0) adjust_precision(num_digits, exp10 + 1); } // Invariant: value == (numerator / denominator) * pow(10, exp10). - if (num_digits < 0) { + if (shortest) { // Generate the shortest representation. num_digits = 0; char* data = buf.data(); @@ -3060,7 +3153,7 @@ FMT_CONSTEXPR20 inline void format_dragon(basic_fp value, } // Generate the given number of digits. exp10 -= num_digits - 1; - if (num_digits == 0) { + if (num_digits <= 0) { denominator *= 10; auto digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0'; buf.push_back(digit); @@ -3085,7 +3178,8 @@ FMT_CONSTEXPR20 inline void format_dragon(basic_fp value, } if (buf[0] == overflow) { buf[0] = '1'; - ++exp10; + if ((flags & dragon::fixed) != 0) buf.push_back('0'); + else ++exp10; } return; } @@ -3094,6 +3188,94 @@ FMT_CONSTEXPR20 inline void format_dragon(basic_fp value, buf[num_digits - 1] = static_cast('0' + digit); } +// Formats a floating-point number using the hexfloat format. +template ::value)> +FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision, + float_specs specs, buffer& buf) { + // float is passed as double to reduce the number of instantiations and to + // simplify implementation. + static_assert(!std::is_same::value, ""); + + using info = dragonbox::float_info; + + // Assume Float is in the format [sign][exponent][significand]. + using carrier_uint = typename info::carrier_uint; + + constexpr auto num_float_significand_bits = + detail::num_significand_bits(); + + basic_fp f(value); + f.e += num_float_significand_bits; + if (!has_implicit_bit()) --f.e; + + constexpr auto num_fraction_bits = + num_float_significand_bits + (has_implicit_bit() ? 1 : 0); + constexpr auto num_xdigits = (num_fraction_bits + 3) / 4; + + constexpr auto leading_shift = ((num_xdigits - 1) * 4); + const auto leading_mask = carrier_uint(0xF) << leading_shift; + const auto leading_xdigit = + static_cast((f.f & leading_mask) >> leading_shift); + if (leading_xdigit > 1) f.e -= (32 - countl_zero(leading_xdigit) - 1); + + int print_xdigits = num_xdigits - 1; + if (precision >= 0 && print_xdigits > precision) { + const int shift = ((print_xdigits - precision - 1) * 4); + const auto mask = carrier_uint(0xF) << shift; + const auto v = static_cast((f.f & mask) >> shift); + + if (v >= 8) { + const auto inc = carrier_uint(1) << (shift + 4); + f.f += inc; + f.f &= ~(inc - 1); + } + + // Check long double overflow + if (!has_implicit_bit()) { + const auto implicit_bit = carrier_uint(1) << num_float_significand_bits; + if ((f.f & implicit_bit) == implicit_bit) { + f.f >>= 4; + f.e += 4; + } + } + + print_xdigits = precision; + } + + char xdigits[num_bits() / 4]; + detail::fill_n(xdigits, sizeof(xdigits), '0'); + format_uint<4>(xdigits, f.f, num_xdigits, specs.upper); + + // Remove zero tail + while (print_xdigits > 0 && xdigits[print_xdigits] == '0') --print_xdigits; + + buf.push_back('0'); + buf.push_back(specs.upper ? 'X' : 'x'); + buf.push_back(xdigits[0]); + if (specs.showpoint || print_xdigits > 0 || print_xdigits < precision) + buf.push_back('.'); + buf.append(xdigits + 1, xdigits + 1 + print_xdigits); + for (; print_xdigits < precision; ++print_xdigits) buf.push_back('0'); + + buf.push_back(specs.upper ? 'P' : 'p'); + + uint32_t abs_e; + if (f.e < 0) { + buf.push_back('-'); + abs_e = static_cast(-f.e); + } else { + buf.push_back('+'); + abs_e = static_cast(f.e); + } + format_decimal(appender(buf), abs_e, detail::count_digits(abs_e)); +} + +template ::value)> +FMT_CONSTEXPR20 void format_hexfloat(Float value, int precision, + float_specs specs, buffer& buf) { + format_hexfloat(static_cast(value), precision, specs, buf); +} + template FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs, buffer& buf) -> int { @@ -3116,7 +3298,7 @@ FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs, int exp = 0; bool use_dragon = true; unsigned dragon_flags = 0; - if (!is_fast_float()) { + if (!is_fast_float() || is_constant_evaluated()) { const auto inv_log2_10 = 0.3010299956639812; // 1 / log2(10) using info = dragonbox::float_info; const auto f = basic_fp(converted_value); @@ -3124,10 +3306,11 @@ FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs, // 10^(exp - 1) <= value < 10^exp or 10^exp <= value < 10^(exp + 1). // This is based on log10(value) == log2(value) / log2(10) and approximation // of log2(value) by e + num_fraction_bits idea from double-conversion. - exp = static_cast( - std::ceil((f.e + count_digits<1>(f.f) - 1) * inv_log2_10 - 1e-10)); + auto e = (f.e + count_digits<1>(f.f) - 1) * inv_log2_10 - 1e-10; + exp = static_cast(e); + if (e > exp) ++exp; // Compute ceil. dragon_flags = dragon::fixup; - } else if (!is_constant_evaluated() && precision < 0) { + } else if (precision < 0) { // Use Dragonbox for the shortest format. if (specs.binary32) { auto dec = dragonbox::to_decimal(static_cast(value)); @@ -3138,23 +3321,244 @@ FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs, write(buffer_appender(buf), dec.significand); return dec.exponent; } else { - // Use Grisu + Dragon4 for the given precision: - // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf. - const int min_exp = -60; // alpha in Grisu. - int cached_exp10 = 0; // K in Grisu. - fp normalized = normalize(fp(converted_value)); - const auto cached_pow = get_cached_power( - min_exp - (normalized.e + fp::num_significand_bits), cached_exp10); - normalized = normalized * cached_pow; - gen_digits_handler handler{buf.data(), 0, precision, -cached_exp10, fixed}; - if (grisu_gen_digits(normalized, 1, exp, handler) != digits::error && - !is_constant_evaluated()) { - exp += handler.exp10; - buf.try_resize(to_unsigned(handler.size)); - use_dragon = false; + // Extract significand bits and exponent bits. + using info = dragonbox::float_info; + auto br = bit_cast(static_cast(value)); + + const uint64_t significand_mask = + (static_cast(1) << num_significand_bits()) - 1; + uint64_t significand = (br & significand_mask); + int exponent = static_cast((br & exponent_mask()) >> + num_significand_bits()); + + if (exponent != 0) { // Check if normal. + exponent -= exponent_bias() + num_significand_bits(); + significand |= + (static_cast(1) << num_significand_bits()); + significand <<= 1; } else { - exp += handler.size - cached_exp10 - 1; - precision = handler.precision; + // Normalize subnormal inputs. + FMT_ASSERT(significand != 0, "zeros should not appear here"); + int shift = countl_zero(significand); + FMT_ASSERT(shift >= num_bits() - num_significand_bits(), + ""); + shift -= (num_bits() - num_significand_bits() - 2); + exponent = (std::numeric_limits::min_exponent - + num_significand_bits()) - + shift; + significand <<= shift; + } + + // Compute the first several nonzero decimal significand digits. + // We call the number we get the first segment. + const int k = info::kappa - dragonbox::floor_log10_pow2(exponent); + exp = -k; + const int beta = exponent + dragonbox::floor_log2_pow10(k); + uint64_t first_segment; + bool has_more_segments; + int digits_in_the_first_segment; + { + const auto r = dragonbox::umul192_upper128( + significand << beta, dragonbox::get_cached_power(k)); + first_segment = r.high(); + has_more_segments = r.low() != 0; + + // The first segment can have 18 ~ 19 digits. + if (first_segment >= 1000000000000000000ULL) { + digits_in_the_first_segment = 19; + } else { + // When it is of 18-digits, we align it to 19-digits by adding a bogus + // zero at the end. + digits_in_the_first_segment = 18; + first_segment *= 10; + } + } + + // Compute the actual number of decimal digits to print. + if (fixed) adjust_precision(precision, exp + digits_in_the_first_segment); + + // Use Dragon4 only when there might be not enough digits in the first + // segment. + if (digits_in_the_first_segment > precision) { + use_dragon = false; + + if (precision <= 0) { + exp += digits_in_the_first_segment; + + if (precision < 0) { + // Nothing to do, since all we have are just leading zeros. + buf.try_resize(0); + } else { + // We may need to round-up. + buf.try_resize(1); + if ((first_segment | static_cast(has_more_segments)) > + 5000000000000000000ULL) { + buf[0] = '1'; + } else { + buf[0] = '0'; + } + } + } // precision <= 0 + else { + exp += digits_in_the_first_segment - precision; + + // When precision > 0, we divide the first segment into three + // subsegments, each with 9, 9, and 0 ~ 1 digits so that each fits + // in 32-bits which usually allows faster calculation than in + // 64-bits. Since some compiler (e.g. MSVC) doesn't know how to optimize + // division-by-constant for large 64-bit divisors, we do it here + // manually. The magic number 7922816251426433760 below is equal to + // ceil(2^(64+32) / 10^10). + const uint32_t first_subsegment = static_cast( + dragonbox::umul128_upper64(first_segment, 7922816251426433760ULL) >> + 32); + const uint64_t second_third_subsegments = + first_segment - first_subsegment * 10000000000ULL; + + uint64_t prod; + uint32_t digits; + bool should_round_up; + int number_of_digits_to_print = precision > 9 ? 9 : precision; + + // Print a 9-digits subsegment, either the first or the second. + auto print_subsegment = [&](uint32_t subsegment, char* buffer) { + int number_of_digits_printed = 0; + + // If we want to print an odd number of digits from the subsegment, + if ((number_of_digits_to_print & 1) != 0) { + // Convert to 64-bit fixed-point fractional form with 1-digit + // integer part. The magic number 720575941 is a good enough + // approximation of 2^(32 + 24) / 10^8; see + // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case + // for details. + prod = ((subsegment * static_cast(720575941)) >> 24) + 1; + digits = static_cast(prod >> 32); + *buffer = static_cast('0' + digits); + number_of_digits_printed++; + } + // If we want to print an even number of digits from the + // first_subsegment, + else { + // Convert to 64-bit fixed-point fractional form with 2-digits + // integer part. The magic number 450359963 is a good enough + // approximation of 2^(32 + 20) / 10^7; see + // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case + // for details. + prod = ((subsegment * static_cast(450359963)) >> 20) + 1; + digits = static_cast(prod >> 32); + copy2(buffer, digits2(digits)); + number_of_digits_printed += 2; + } + + // Print all digit pairs. + while (number_of_digits_printed < number_of_digits_to_print) { + prod = static_cast(prod) * static_cast(100); + digits = static_cast(prod >> 32); + copy2(buffer + number_of_digits_printed, digits2(digits)); + number_of_digits_printed += 2; + } + }; + + // Print first subsegment. + print_subsegment(first_subsegment, buf.data()); + + // Perform rounding if the first subsegment is the last subsegment to + // print. + if (precision <= 9) { + // Rounding inside the subsegment. + // We round-up if: + // - either the fractional part is strictly larger than 1/2, or + // - the fractional part is exactly 1/2 and the last digit is odd. + // We rely on the following observations: + // - If fractional_part >= threshold, then the fractional part is + // strictly larger than 1/2. + // - If the MSB of fractional_part is set, then the fractional part + // must be at least 1/2. + // - When the MSB of fractional_part is set, either + // second_third_subsegments being nonzero or has_more_segments + // being true means there are further digits not printed, so the + // fractional part is strictly larger than 1/2. + if (precision < 9) { + uint32_t fractional_part = static_cast(prod); + should_round_up = fractional_part >= + data::fractional_part_rounding_thresholds + [8 - number_of_digits_to_print] || + ((fractional_part >> 31) & + ((digits & 1) | (second_third_subsegments != 0) | + has_more_segments)) != 0; + } + // Rounding at the subsegment boundary. + // In this case, the fractional part is at least 1/2 if and only if + // second_third_subsegments >= 5000000000ULL, and is strictly larger + // than 1/2 if we further have either second_third_subsegments > + // 5000000000ULL or has_more_segments == true. + else { + should_round_up = second_third_subsegments > 5000000000ULL || + (second_third_subsegments == 5000000000ULL && + ((digits & 1) != 0 || has_more_segments)); + } + } + // Otherwise, print the second subsegment. + else { + // Compilers are not aware of how to leverage the maximum value of + // second_third_subsegments to find out a better magic number which + // allows us to eliminate an additional shift. 1844674407370955162 = + // ceil(2^64/10) < ceil(2^64*(10^9/(10^10 - 1))). + const uint32_t second_subsegment = + static_cast(dragonbox::umul128_upper64( + second_third_subsegments, 1844674407370955162ULL)); + const uint32_t third_subsegment = + static_cast(second_third_subsegments) - + second_subsegment * 10; + + number_of_digits_to_print = precision - 9; + print_subsegment(second_subsegment, buf.data() + 9); + + // Rounding inside the subsegment. + if (precision < 18) { + // The condition third_subsegment != 0 implies that the segment was + // of 19 digits, so in this case the third segment should be + // consisting of a genuine digit from the input. + uint32_t fractional_part = static_cast(prod); + should_round_up = fractional_part >= + data::fractional_part_rounding_thresholds + [8 - number_of_digits_to_print] || + ((fractional_part >> 31) & + ((digits & 1) | (third_subsegment != 0) | + has_more_segments)) != 0; + } + // Rounding at the subsegment boundary. + else { + // In this case, the segment must be of 19 digits, thus + // the third subsegment should be consisting of a genuine digit from + // the input. + should_round_up = third_subsegment > 5 || + (third_subsegment == 5 && + ((digits & 1) != 0 || has_more_segments)); + } + } + + // Round-up if necessary. + if (should_round_up) { + ++buf[precision - 1]; + for (int i = precision - 1; i > 0 && buf[i] > '9'; --i) { + buf[i] = '0'; + ++buf[i - 1]; + } + if (buf[0] > '9') { + buf[0] = '1'; + if (fixed) + buf[precision++] = '0'; + else + ++exp; + } + } + buf.try_resize(to_unsigned(precision)); + } + } // if (digits_in_the_first_segment > precision) + else { + // Adjust the exponent for its use in Dragon4. + exp += digits_in_the_first_segment - 1; } } if (use_dragon) { @@ -3181,13 +3585,10 @@ FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs, } return exp; } - -template ::value)> -FMT_CONSTEXPR20 auto write(OutputIt out, T value, - basic_format_specs specs, locale_ref loc = {}) +template +FMT_CONSTEXPR20 auto write_float(OutputIt out, T value, + format_specs specs, locale_ref loc) -> OutputIt { - if (const_check(!is_supported_floating_point(value))) return out; float_specs fspecs = parse_float_type_spec(specs); fspecs.sign = specs.sign; if (detail::signbit(value)) { // value < 0 is false for NaN so use signbit. @@ -3211,7 +3612,7 @@ FMT_CONSTEXPR20 auto write(OutputIt out, T value, memory_buffer buffer; if (fspecs.format == float_format::hex) { if (fspecs.sign) buffer.push_back(detail::sign(fspecs.sign)); - snprintf_float(convert_float(value), specs.precision, fspecs, buffer); + format_hexfloat(convert_float(value), specs.precision, fspecs, buffer); return write_bytes(out, {buffer.data(), buffer.size()}, specs); } @@ -3233,11 +3634,20 @@ FMT_CONSTEXPR20 auto write(OutputIt out, T value, return write_float(out, f, specs, fspecs, loc); } +template ::value)> +FMT_CONSTEXPR20 auto write(OutputIt out, T value, format_specs specs, + locale_ref loc = {}) -> OutputIt { + if (const_check(!is_supported_floating_point(value))) return out; + return specs.localized && write_loc(out, value, specs, loc) + ? out + : write_float(out, value, specs, loc); +} + template ::value)> FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt { - if (is_constant_evaluated()) - return write(out, value, basic_format_specs()); + if (is_constant_evaluated()) return write(out, value, format_specs()); if (const_check(!is_supported_floating_point(value))) return out; auto fspecs = float_specs(); @@ -3246,11 +3656,11 @@ FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt { value = -value; } - constexpr auto specs = basic_format_specs(); + constexpr auto specs = format_specs(); using floaty = conditional_t::value, double, T>; - using uint = typename dragonbox::float_info::carrier_uint; - uint mask = exponent_mask(); - if ((bit_cast(value) & mask) == mask) + using floaty_uint = typename dragonbox::float_info::carrier_uint; + floaty_uint mask = exponent_mask(); + if ((bit_cast(value) & mask) == mask) return write_nonfinite(out, std::isnan(value), specs, fspecs); auto dec = dragonbox::to_decimal(static_cast(value)); @@ -3261,12 +3671,12 @@ template ::value && !is_fast_float::value)> inline auto write(OutputIt out, T value) -> OutputIt { - return write(out, value, basic_format_specs()); + return write(out, value, format_specs()); } template -auto write(OutputIt out, monostate, basic_format_specs = {}, - locale_ref = {}) -> OutputIt { +auto write(OutputIt out, monostate, format_specs = {}, locale_ref = {}) + -> OutputIt { FMT_ASSERT(false, ""); return out; } @@ -3300,8 +3710,8 @@ FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { template ::value)> FMT_CONSTEXPR auto write(OutputIt out, T value, - const basic_format_specs& specs = {}, - locale_ref = {}) -> OutputIt { + const format_specs& specs = {}, locale_ref = {}) + -> OutputIt { return specs.type != presentation_type::none && specs.type != presentation_type::string ? write(out, value ? 1 : 0, specs, {}) @@ -3318,20 +3728,15 @@ FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt { template FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value) -> OutputIt { - if (!value) { - throw_format_error("string pointer is null"); - } else { - out = write(out, basic_string_view(value)); - } + if (value) return write(out, basic_string_view(value)); + throw_format_error("string pointer is null"); return out; } template ::value)> -auto write(OutputIt out, const T* value, - const basic_format_specs& specs = {}, locale_ref = {}) - -> OutputIt { - check_pointer_type_spec(specs.type, error_handler()); +auto write(OutputIt out, const T* value, const format_specs& specs = {}, + locale_ref = {}) -> OutputIt { return write_ptr(out, bit_cast(value), &specs); } @@ -3341,8 +3746,8 @@ template enable_if_t< std::is_class::value && !is_string::value && !is_floating_point::value && !std::is_same::value && - !std::is_same().map(value))>::value, + !std::is_same().map( + value))>>::value, OutputIt> { return write(out, arg_mapper().map(value)); } @@ -3352,12 +3757,8 @@ template enable_if_t::value == type::custom_type, OutputIt> { - using formatter_type = - conditional_t::value, - typename Context::template formatter_type, - fallback_formatter>; auto ctx = Context(out, {}, {}); - return formatter_type().format(value, ctx); + return typename Context::template formatter_type().format(value, ctx); } // An argument visitor that formats the argument and writes it via the output @@ -3386,7 +3787,7 @@ template struct arg_formatter { using context = buffer_context; iterator out; - const basic_format_specs& specs; + const format_specs& specs; locale_ref locale; template @@ -3411,12 +3812,6 @@ template struct custom_formatter { template void operator()(T) const {} }; -template -using is_integer = - bool_constant::value && !std::is_same::value && - !std::is_same::value && - !std::is_same::value>; - template class width_checker { public: explicit FMT_CONSTEXPR width_checker(ErrorHandler& eh) : handler_(eh) {} @@ -3466,55 +3861,12 @@ FMT_CONSTEXPR auto get_dynamic_spec(FormatArg arg, ErrorHandler eh) -> int { } template -FMT_CONSTEXPR auto get_arg(Context& ctx, ID id) -> - typename Context::format_arg { +FMT_CONSTEXPR auto get_arg(Context& ctx, ID id) -> decltype(ctx.arg(id)) { auto arg = ctx.arg(id); if (!arg) ctx.on_error("argument not found"); return arg; } -// The standard format specifier handler with checking. -template class specs_handler : public specs_setter { - private: - basic_format_parse_context& parse_context_; - buffer_context& context_; - - // This is only needed for compatibility with gcc 4.4. - using format_arg = basic_format_arg>; - - FMT_CONSTEXPR auto get_arg(auto_id) -> format_arg { - return detail::get_arg(context_, parse_context_.next_arg_id()); - } - - FMT_CONSTEXPR auto get_arg(int arg_id) -> format_arg { - parse_context_.check_arg_id(arg_id); - return detail::get_arg(context_, arg_id); - } - - FMT_CONSTEXPR auto get_arg(basic_string_view arg_id) -> format_arg { - parse_context_.check_arg_id(arg_id); - return detail::get_arg(context_, arg_id); - } - - public: - FMT_CONSTEXPR specs_handler(basic_format_specs& specs, - basic_format_parse_context& parse_ctx, - buffer_context& ctx) - : specs_setter(specs), parse_context_(parse_ctx), context_(ctx) {} - - template FMT_CONSTEXPR void on_dynamic_width(Id arg_id) { - this->specs_.width = get_dynamic_spec( - get_arg(arg_id), context_.error_handler()); - } - - template FMT_CONSTEXPR void on_dynamic_precision(Id arg_id) { - this->specs_.precision = get_dynamic_spec( - get_arg(arg_id), context_.error_handler()); - } - - void on_error(const char* message) { context_.on_error(message); } -}; - template