diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a11325fdd14..6f2490327880 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `::consensus` is now `ccf::consensus` - `::tls` is now `ccf::tls` - The `programmability` sample app now demonstrates how applications can define their own extensions, creating bindings between C++ and JS state, and allowing JS endpoints to call functions implemented in C++. +- Introduce `DynamicJSEndpointRegistry::record_action_for_audit_v1` and `DynamicJSEndpointRegistry::check_action_not_replayed_v1` to allow an application making use of the programmability feature to easily implement auditability, and protect users allowed to update the application against replay attacks (#6285). - Endpoints now support a `ToBackup` redirection strategy, for requests which should never be executed on a primary. These must also be read-only. These are configured similar to `ToPrimary` endpoints, with a `to_backup` object (specifying by-role or statically-addressed targets) in each node's configuration. ### Removed diff --git a/doc/build_apps/api.rst b/doc/build_apps/api.rst index f89b5ef8068e..3383e92df673 100644 --- a/doc/build_apps/api.rst +++ b/doc/build_apps/api.rst @@ -72,6 +72,9 @@ Policies .. doxygenvariable:: ccf::jwt_auth_policy :project: CCF +.. doxygenvariable:: ccf::TypedUserCOSESign1AuthnPolicy + :project: CCF + Identities ~~~~~~~~~~ diff --git a/include/ccf/base_endpoint_registry.h b/include/ccf/base_endpoint_registry.h index 17487562b145..ae1e76f7a500 100644 --- a/include/ccf/base_endpoint_registry.h +++ b/include/ccf/base_endpoint_registry.h @@ -63,6 +63,47 @@ namespace ccf } } + /** Lists possible reasons for an ApiResult::InvalidArgs being return in @c + * ccf::BaseEndpointRegistry + */ + enum class InvalidArgsReason + { + NoReason = 0, + /** Views start at 1 (one) in CCF */ + ViewSmallerThanOne, + /** Action has already been applied on this instance */ + ActionAlreadyApplied, + /** Action created_at is older than the median of recent action */ + StaleActionCreatedTimestamp, + }; + + constexpr char const* invalid_args_reason_to_str(InvalidArgsReason reason) + { + switch (reason) + { + case InvalidArgsReason::NoReason: + { + return "NoReason"; + } + case InvalidArgsReason::ViewSmallerThanOne: + { + return "ViewSmallerThanOne"; + } + case InvalidArgsReason::ActionAlreadyApplied: + { + return "ActionAlreadyApplied"; + } + case InvalidArgsReason::StaleActionCreatedTimestamp: + { + return "StaleActionCreatedTimestamp"; + } + default: + { + return "Unhandled InvalidArgsReason"; + } + } + } + /** Extends the basic @ref ccf::endpoints::EndpointRegistry with helper API * methods for retrieving core CCF properties. * @@ -96,6 +137,19 @@ namespace ccf ApiResult get_view_history_v1( std::vector& history, ccf::View since = 1); + /** Get the history of the consensus view changes. + * + * Returns the history of view changes since the given view, which defaults + * to the start of time. + * + * A view change is characterised by the first sequence number in the new + * view. + */ + ApiResult get_view_history_v2( + std::vector& history, + ccf::View since, + ccf::InvalidArgsReason& reason); + /** Get the status of a transaction by ID, provided as a view+seqno pair. * * Note that this value is the node's local understanding of the status diff --git a/include/ccf/endpoints/authentication/cose_auth.h b/include/ccf/endpoints/authentication/cose_auth.h index b714154c205e..36b0aad5e262 100644 --- a/include/ccf/endpoints/authentication/cose_auth.h +++ b/include/ccf/endpoints/authentication/cose_auth.h @@ -22,6 +22,12 @@ namespace ccf uint64_t gov_msg_created_at; }; + struct TimestampedProtectedHeader : ProtectedHeader + { + std::optional msg_type; + std::optional msg_created_at; + }; + struct COSESign1AuthnIdentity : public AuthnIdentity { /** COSE Content */ @@ -83,7 +89,7 @@ namespace ccf crypto::Pem user_cert; /** COSE Protected Header */ - ProtectedHeader protected_header; + TimestampedProtectedHeader protected_header; UserCOSESign1AuthnIdentity( const std::span& content_, @@ -91,7 +97,7 @@ namespace ccf const std::span& signature_, const UserId& user_id_, const crypto::Pem& user_cert_, - const ProtectedHeader& protected_header_) : + const TimestampedProtectedHeader& protected_header_) : COSESign1AuthnIdentity(content_, envelope_, signature_), user_id(user_id_), user_cert(user_cert_), @@ -163,16 +169,32 @@ namespace ccf }; /** User COSE Sign1 Authentication Policy + * + * Allows parametrising two optional protected header entries + * which are exposed to the endpoint if present. */ class UserCOSESign1AuthnPolicy : public AuthnPolicy { + std::string msg_type_name; + std::string msg_created_at_name; + protected: static const OpenAPISecuritySchema security_schema; + virtual std::unique_ptr _authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason); + public: static constexpr auto SECURITY_SCHEME_NAME = "user_cose_sign1"; - UserCOSESign1AuthnPolicy(); + UserCOSESign1AuthnPolicy( + const std::string& msg_type_name_ = "ccf.msg.type", + const std::string& msg_created_at_name_ = "ccf.msg.created_at") : + msg_type_name(msg_type_name_), + msg_created_at_name(msg_created_at_name_) + {} ~UserCOSESign1AuthnPolicy(); std::unique_ptr authenticate( @@ -195,4 +217,36 @@ namespace ccf return SECURITY_SCHEME_NAME; } }; -} + + /** Typed User COSE Sign1 Authentication Policy + * + * Extends UserCOSESign1AuthPolicy, to require that a specific message + * type is present in the corresponding protected header. + */ + class TypedUserCOSESign1AuthnPolicy : public UserCOSESign1AuthnPolicy + { + private: + std::string expected_msg_type; + + public: + static constexpr auto SECURITY_SCHEME_NAME = "typed_user_cose_sign1"; + + TypedUserCOSESign1AuthnPolicy( + const std::string& expected_msg_type_, + const std::string& msg_type_name_ = "ccf.msg.type", + const std::string& msg_created_at_name_ = "ccf.msg.created_at") : + UserCOSESign1AuthnPolicy(msg_type_name_, msg_created_at_name_), + expected_msg_type(expected_msg_type_) + {} + + std::unique_ptr authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) override; + + std::string get_security_scheme_name() override + { + return SECURITY_SCHEME_NAME; + } + }; +} \ No newline at end of file diff --git a/include/ccf/js/audit_format.h b/include/ccf/js/audit_format.h new file mode 100644 index 000000000000..96a36932990f --- /dev/null +++ b/include/ccf/js/audit_format.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#pragma once +#include "ccf/ds/json.h" + +#include + +namespace ccf +{ + enum class ActionFormat + { + COSE = 0, + JSON = 1 + }; + DECLARE_JSON_ENUM( + ActionFormat, {{ActionFormat::COSE, "COSE"}, {ActionFormat::JSON, "JSON"}}); + + struct AuditInfo + { + ActionFormat format; + // Deliberately a string and not a ccf::UserId to allow extended usage, for + // example with OpenID + std::string user_id; + // Format left to the application, Verb + URL with some of kind of + // versioning is recommended + std::string action_name; + }; + + DECLARE_JSON_TYPE(AuditInfo) + DECLARE_JSON_REQUIRED_FIELDS(AuditInfo, format, user_id, action_name) +} \ No newline at end of file diff --git a/include/ccf/js/registry.h b/include/ccf/js/registry.h index 5ca1c1872827..8c7c0ea7d4ca 100644 --- a/include/ccf/js/registry.h +++ b/include/ccf/js/registry.h @@ -4,6 +4,7 @@ // CCF #include "ccf/app_interface.h" #include "ccf/endpoint.h" +#include "ccf/js/audit_format.h" #include "ccf/js/bundle.h" #include "ccf/js/core/context.h" #include "ccf/js/interpreter_cache_interface.h" @@ -51,6 +52,9 @@ namespace ccf::js std::string modules_quickjs_version_map; std::string modules_quickjs_bytecode_map; std::string runtime_options_map; + std::string recent_actions_map; + std::string audit_input_map; + std::string audit_info_map; ccf::js::NamespaceRestriction namespace_restriction; @@ -127,6 +131,28 @@ namespace ccf::js ccf::ApiResult get_js_runtime_options_v1( ccf::JSRuntimeOptions& options, kv::ReadOnlyTx& tx); + /** + * Record action details by storing them in KV maps using a common format, + * for the purposes of offline audit using the ledger. + */ + ccf::ApiResult record_action_for_audit_v1( + kv::Tx& tx, + ccf::ActionFormat format, + const std::string& user_id, + const std::string& action_name, + const std::vector& action_body); + + /** + * Check an action is not being replayed, by looking it up + * in the history of recent actions. To place an upper bound on the history + * size, an authenticated timestamp (@p created_at) is required. + */ + ccf::ApiResult check_action_not_replayed_v1( + kv::Tx& tx, + uint64_t created_at, + const std::span action, + ccf::InvalidArgsReason& reason); + /// \defgroup Overrides for base EndpointRegistry functions, looking up JS /// endpoints before delegating to base implementation. ///@{ diff --git a/samples/apps/programmability/audit_info.h b/samples/apps/programmability/audit_info.h deleted file mode 100644 index e485a39c2224..000000000000 --- a/samples/apps/programmability/audit_info.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. - -#pragma once -#include "ccf/ds/json.h" -#include "ccf/entity_id.h" - -#include - -namespace programmabilityapp -{ - enum class AuditInputFormat - { - COSE = 0, - JSON = 1 - }; - DECLARE_JSON_ENUM( - AuditInputFormat, - {{AuditInputFormat::COSE, "COSE"}, {AuditInputFormat::JSON, "JSON"}}); - - enum class AuditInputContent - { - BUNDLE = 0, - OPTIONS = 1 - }; - DECLARE_JSON_ENUM( - AuditInputContent, - {{AuditInputContent::BUNDLE, "BUNDLE"}, - {AuditInputContent::OPTIONS, "OPTIONS"}}); - - struct AuditInfo - { - AuditInputFormat format; - AuditInputContent content; - ccf::UserId user_id; - }; - - DECLARE_JSON_TYPE(AuditInfo) - DECLARE_JSON_REQUIRED_FIELDS(AuditInfo, format, content, user_id) -} \ No newline at end of file diff --git a/samples/apps/programmability/programmability.cpp b/samples/apps/programmability/programmability.cpp index 514e216861d1..d05c25031a7c 100644 --- a/samples/apps/programmability/programmability.cpp +++ b/samples/apps/programmability/programmability.cpp @@ -2,7 +2,6 @@ // Licensed under the Apache 2.0 License. // CCF -#include "audit_info.h" #include "ccf/app_interface.h" #include "ccf/common_auth_policies.h" #include "ccf/ds/hash.h" @@ -21,11 +20,26 @@ using namespace nlohmann; namespace programmabilityapp { using RecordsMap = kv::Map>; - using AuditInputValue = kv::Value>; - using AuditInfoValue = kv::Value; static constexpr auto PRIVATE_RECORDS = "programmability.records"; static constexpr auto CUSTOM_ENDPOINTS_NAMESPACE = "public:custom_endpoints"; + // The programmability sample demonstrates how signed payloads can be used to + // provide offline auditability without requiring trusting the hardware or the + // service owners/consortium. + // COSE Sign1 payloads must set these protected headers in order to guarantee + // the specificity of the payload for the endpoint, and avoid possible replay + // of payloads signed in the past. + static constexpr auto MSG_TYPE_NAME = "app.msg.type"; + static constexpr auto CREATED_AT_NAME = "app.msg.created_at"; + // Instances of ccf::TypedUserCOSESign1AuthnPolicy for the endpoints that + // support COSE Sign1 authentication. + static auto endpoints_user_cose_sign1_auth_policy = + std::make_shared( + "custom_endpoints", MSG_TYPE_NAME, CREATED_AT_NAME); + static auto options_user_cose_sign1_auth_policy = + std::make_shared( + "runtime_options", MSG_TYPE_NAME, CREATED_AT_NAME); + // This is a pure helper function which can be called from either C++ or JS, // to implement common functionality in a single place static inline bool has_role_permitting_action( @@ -185,18 +199,64 @@ namespace programmabilityapp return std::nullopt; } - std::pair> get_body( - ccf::endpoints::EndpointContext& ctx) + std::tuple< + ccf::ActionFormat, // JSON or COSE + std::span, // Content + std::optional // Created at timestamp, if passed + > + get_action_content(ccf::endpoints::EndpointContext& ctx) { if ( const auto* cose_ident = ctx.try_get_caller()) { - return {AuditInputFormat::COSE, cose_ident->content}; + return { + ccf::ActionFormat::COSE, + cose_ident->content, + cose_ident->protected_header.msg_created_at}; } else { - return {AuditInputFormat::JSON, ctx.rpc_ctx->get_request_body()}; + return { + ccf::ActionFormat::JSON, + ctx.rpc_ctx->get_request_body(), + std::nullopt}; + } + } + + bool set_error_details( + ccf::endpoints::EndpointContext& ctx, + ccf::ApiResult result, + ccf::InvalidArgsReason reason) + { + switch (result) + { + case ccf::ApiResult::OK: + { + return false; + } + case ccf::ApiResult::InvalidArgs: + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InvalidInput, + reason == ccf::InvalidArgsReason::ActionAlreadyApplied ? + "Action was already applied" : + "Action created_at timestamp is too old"); + return true; + } + case ccf::ApiResult::InternalError: + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Failed to check if action is original"); + return true; + } + default: + { + return true; + } } } @@ -360,18 +420,45 @@ namespace programmabilityapp } // End of Authorization Check - const auto [format, bundle] = get_body(ctx); - const auto j = nlohmann::json::parse(bundle.begin(), bundle.end()); - const auto parsed_bundle = j.get(); - - // Make operation auditable by writing user-supplied - // document to the ledger - auto audit_input = ctx.tx.template rw( - fmt::format("{}.audit.input", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_input->put(ctx.rpc_ctx->get_request_body()); - auto audit_info = ctx.tx.template rw( - fmt::format("{}.audit.info", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_info->put({format, AuditInputContent::BUNDLE, user_id.value()}); + const auto [format, content, created_at] = get_action_content(ctx); + const auto parsed_content = + nlohmann::json::parse(content.begin(), content.end()); + const auto parsed_bundle = parsed_content.get(); + + // Make operation auditable + record_action_for_audit_v1( + ctx.tx, + format, + user_id.value(), + fmt::format( + "{} {}", + ctx.rpc_ctx->get_method(), + ctx.rpc_ctx->get_request_path()), + ctx.rpc_ctx->get_request_body()); + + // Ensure signed actions are not replayed + if (format == ccf::ActionFormat::COSE) + { + if (!created_at.has_value()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::MissingRequiredHeader, + fmt::format("Missing {} protected header", CREATED_AT_NAME)); + return; + } + ccf::InvalidArgsReason reason; + result = check_action_not_replayed_v1( + ctx.tx, + created_at.value(), + ctx.rpc_ctx->get_request_body(), + reason); + + if (set_error_details(ctx, result, reason)) + { + return; + } + } result = install_custom_endpoints_v1(ctx.tx, parsed_bundle); if (result != ccf::ApiResult::OK) @@ -392,7 +479,7 @@ namespace programmabilityapp "/custom_endpoints", HTTP_PUT, put_custom_endpoints, - {ccf::user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) + {endpoints_user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) .set_auto_schema() .install(); @@ -523,26 +610,53 @@ namespace programmabilityapp // - Convert current options to JSON auto j_options = nlohmann::json(options); - const auto [format, body] = get_body(ctx); - // - Parse argument as JSON body - const auto arg_body = nlohmann::json::parse(body.begin(), body.end()); + const auto [format, content, created_at] = get_action_content(ctx); + // - Parse content as JSON options + const auto arg_content = + nlohmann::json::parse(content.begin(), content.end()); // - Merge, to overwrite current options with anything from body. Note // that nulls mean deletions, which results in resetting to a default // value - j_options.merge_patch(arg_body); + j_options.merge_patch(arg_content); // - Parse patched options from JSON options = j_options.get(); - // Make operation auditable by writing user-supplied - // document to the ledger - auto audit = ctx.tx.template rw( - fmt::format("{}.audit.input", CUSTOM_ENDPOINTS_NAMESPACE)); - audit->put(ctx.rpc_ctx->get_request_body()); - auto audit_info = ctx.tx.template rw( - fmt::format("{}.audit.info", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_info->put({format, AuditInputContent::BUNDLE, user_id.value()}); + // Make operation auditable + record_action_for_audit_v1( + ctx.tx, + format, + user_id.value(), + fmt::format( + "{} {}", + ctx.rpc_ctx->get_method(), + ctx.rpc_ctx->get_request_path()), + ctx.rpc_ctx->get_request_body()); + + // Ensure signed actions are not replayed + if (format == ccf::ActionFormat::COSE) + { + if (!created_at.has_value()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::MissingRequiredHeader, + fmt::format("Missing {} protected header", CREATED_AT_NAME)); + return; + } + ccf::InvalidArgsReason reason; + result = check_action_not_replayed_v1( + ctx.tx, + created_at.value(), + ctx.rpc_ctx->get_request_body(), + reason); + + if (set_error_details(ctx, result, reason)) + { + return; + } + } result = set_js_runtime_options_v1(ctx.tx, options); if (result != ccf::ApiResult::OK) @@ -564,7 +678,7 @@ namespace programmabilityapp "/custom_endpoints/runtime_options", HTTP_PATCH, patch_runtime_options, - {ccf::user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) + {options_user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) .install(); auto get_runtime_options = [this](ccf::endpoints::EndpointContext& ctx) { diff --git a/src/endpoints/authentication/cose_auth.cpp b/src/endpoints/authentication/cose_auth.cpp index 1574982a3ae4..8a7ea9c4743a 100644 --- a/src/endpoints/authentication/cose_auth.cpp +++ b/src/endpoints/authentication/cose_auth.cpp @@ -64,7 +64,7 @@ namespace ccf KID_INDEX, GOV_MSG_TYPE, GOV_MSG_PROPOSAL_ID, - GOV_MSG_CREATED_AT, + GOV_MSG_MSG_CREATED_AT, END_INDEX, }; QCBORItem header_items[END_INDEX + 1]; @@ -90,13 +90,13 @@ namespace ccf header_items[GOV_MSG_PROPOSAL_ID].uDataType = QCBOR_TYPE_TEXT_STRING; auto gov_msg_proposal_created_at = HEADER_PARAM_MSG_CREATED_AT; - header_items[GOV_MSG_CREATED_AT].label.string = + header_items[GOV_MSG_MSG_CREATED_AT].label.string = UsefulBuf_FromSZ(gov_msg_proposal_created_at); - header_items[GOV_MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; + header_items[GOV_MSG_MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; // Although this is really uint, specify QCBOR_TYPE_INT64 // QCBOR_TYPE_UINT64 only matches uint values that are greater than // INT64_MAX - header_items[GOV_MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; + header_items[GOV_MSG_MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE; @@ -121,7 +121,7 @@ namespace ccf } parsed.kid = qcbor_buf_to_string(header_items[KID_INDEX].val.string); - if (header_items[GOV_MSG_CREATED_AT].uDataType == QCBOR_TYPE_NONE) + if (header_items[GOV_MSG_MSG_CREATED_AT].uDataType == QCBOR_TYPE_NONE) { throw COSEDecodeError("Missing created_at in protected header"); } @@ -137,11 +137,12 @@ namespace ccf qcbor_buf_to_string(header_items[GOV_MSG_PROPOSAL_ID].val.string); } // Really uint, but the parser doesn't enforce that, so we must check - if (header_items[GOV_MSG_CREATED_AT].val.int64 < 0) + if (header_items[GOV_MSG_MSG_CREATED_AT].val.int64 < 0) { throw COSEDecodeError("Header parameter created_at must be positive"); } - parsed.gov_msg_created_at = header_items[GOV_MSG_CREATED_AT].val.int64; + parsed.gov_msg_created_at = + header_items[GOV_MSG_MSG_CREATED_AT].val.int64; QCBORDecode_ExitMap(&ctx); QCBORDecode_ExitBstrWrapped(&ctx); @@ -166,11 +167,13 @@ namespace ccf return {parsed, sig}; } - std::pair + std::pair extract_protected_header_and_signature( - const std::vector& cose_sign1) + const std::vector& cose_sign1, + const std::string& msg_type_name, + const std::string& created_at_name) { - ccf::ProtectedHeader parsed; + ccf::TimestampedProtectedHeader parsed; // Adapted from parse_cose_header_parameters in t_cose_parameters.c. // t_cose doesn't support custom header parameters yet. @@ -203,6 +206,8 @@ namespace ccf { ALG_INDEX, KID_INDEX, + MSG_TYPE, + MSG_CREATED_AT, END_INDEX, }; QCBORItem header_items[END_INDEX + 1]; @@ -215,6 +220,20 @@ namespace ccf header_items[KID_INDEX].uLabelType = QCBOR_TYPE_INT64; header_items[KID_INDEX].uDataType = QCBOR_TYPE_BYTE_STRING; + header_items[MSG_TYPE].label.string = + UsefulBuf_FromSZ(msg_type_name.c_str()); + header_items[MSG_TYPE].uLabelType = QCBOR_TYPE_TEXT_STRING; + header_items[MSG_TYPE].uDataType = QCBOR_TYPE_TEXT_STRING; + + auto gov_msg_proposal_created_at = HEADER_PARAM_MSG_CREATED_AT; + header_items[MSG_CREATED_AT].label.string = + UsefulBuf_FromSZ(created_at_name.c_str()); + header_items[MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; + // Although this is really uint, specify QCBOR_TYPE_INT64 + // QCBOR_TYPE_UINT64 only matches uint values that are greater than + // INT64_MAX + header_items[MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; + header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE; QCBORDecode_GetItemsInMap(&ctx, header_items); @@ -238,6 +257,19 @@ namespace ccf } parsed.kid = qcbor_buf_to_string(header_items[KID_INDEX].val.string); + if (header_items[MSG_TYPE].uDataType != QCBOR_TYPE_NONE) + { + parsed.msg_type = + qcbor_buf_to_string(header_items[MSG_TYPE].val.string); + } + if ( + header_items[MSG_CREATED_AT].uDataType != QCBOR_TYPE_NONE && + // Really uint, but the parser doesn't enforce that, so we must check + header_items[MSG_CREATED_AT].val.int64 > 0) + { + parsed.msg_created_at = header_items[MSG_CREATED_AT].val.int64; + } + QCBORDecode_ExitMap(&ctx); QCBORDecode_ExitBstrWrapped(&ctx); @@ -404,13 +436,13 @@ namespace ccf return ident; } - UserCOSESign1AuthnPolicy::UserCOSESign1AuthnPolicy() = default; UserCOSESign1AuthnPolicy::~UserCOSESign1AuthnPolicy() = default; - std::unique_ptr UserCOSESign1AuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, - const std::shared_ptr& ctx, - std::string& error_reason) + std::unique_ptr UserCOSESign1AuthnPolicy:: + _authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) { const auto& headers = ctx->get_request_headers(); const auto content_type_it = headers.find(http::headers::CONTENT_TYPE); @@ -427,8 +459,8 @@ namespace ccf return nullptr; } - auto [phdr, cose_signature] = - cose::extract_protected_header_and_signature(ctx->get_request_body()); + auto [phdr, cose_signature] = cose::extract_protected_header_and_signature( + ctx->get_request_body(), msg_type_name, msg_created_at_name); if (!cose::is_ecdsa_alg(phdr.alg)) { @@ -467,6 +499,14 @@ namespace ccf } } + std::unique_ptr UserCOSESign1AuthnPolicy::authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) + { + return _authenticate(tx, ctx, error_reason); + } + void UserCOSESign1AuthnPolicy::set_unauthenticated_error( std::shared_ptr ctx, std::string&& error_reason) { @@ -489,4 +529,25 @@ namespace ccf "Request payload must be a COSE Sign1 document, with expected " "protected headers. " "Signer must be a user identity registered with this service."}}); + + std::unique_ptr TypedUserCOSESign1AuthnPolicy::authenticate( + kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) + { + auto identity = _authenticate(tx, ctx, error_reason); + + if ( + identity != nullptr && + identity->protected_header.msg_type != expected_msg_type) + { + error_reason = fmt::format( + "Unexpected message type: {}, expected: {}", + identity->protected_header.msg_type, + expected_msg_type); + return nullptr; + } + + return identity; + } } diff --git a/src/endpoints/base_endpoint_registry.cpp b/src/endpoints/base_endpoint_registry.cpp index 4d96f5b62a99..951a501b7e7b 100644 --- a/src/endpoints/base_endpoint_registry.cpp +++ b/src/endpoints/base_endpoint_registry.cpp @@ -18,8 +18,10 @@ namespace ccf context(context_) {} - ApiResult BaseEndpointRegistry::get_view_history_v1( - std::vector& history, ccf::View since) + ApiResult BaseEndpointRegistry::get_view_history_v2( + std::vector& history, + ccf::View since, + ccf::InvalidArgsReason& reason) { try { @@ -27,7 +29,7 @@ namespace ccf { if (since < 1) { - // views start at 1 + reason = ccf::InvalidArgsReason::ViewSmallerThanOne; return ApiResult::InvalidArgs; } auto latest_view = consensus->get_view(); @@ -54,6 +56,13 @@ namespace ccf } } + ApiResult BaseEndpointRegistry::get_view_history_v1( + std::vector& history, ccf::View since) + { + ccf::InvalidArgsReason ignored; + return get_view_history_v2(history, since, ignored); + } + ApiResult BaseEndpointRegistry::get_status_for_txid_v1( ccf::View view, ccf::SeqNo seqno, ccf::TxStatus& tx_status) { diff --git a/src/js/registry.cpp b/src/js/registry.cpp index 2ecebbc69241..fd5f6aec5f51 100644 --- a/src/js/registry.cpp +++ b/src/js/registry.cpp @@ -17,6 +17,8 @@ #include // Custom Endpoints +#include "ccf/crypto/sha256.h" +#include "ccf/ds/hex.h" #include "ccf/endpoint.h" #include "ccf/endpoints/authentication/js.h" #include "ccf/historical_queries_adapter.h" @@ -459,7 +461,10 @@ namespace ccf::js fmt::format("{}.modules_quickjs_version", kv_prefix)), modules_quickjs_bytecode_map( fmt::format("{}.modules_quickjs_bytecode", kv_prefix)), - runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)) + runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)), + recent_actions_map(fmt::format("{}.recent_actions", kv_prefix)), + audit_input_map(fmt::format("{}.audit.input", kv_prefix)), + audit_info_map(fmt::format("{}.audit.info", kv_prefix)) { interpreter_cache = context.get_subsystem(); @@ -876,4 +881,100 @@ namespace ccf::js return true; }); } + + ccf::ApiResult DynamicJSEndpointRegistry::check_action_not_replayed_v1( + kv::Tx& tx, + uint64_t created_at, + const std::span action, + ccf::InvalidArgsReason& reason) + { + try + { + const auto created_at_str = fmt::format("{:0>10}", created_at); + const auto action_digest = crypto::sha256(action.data(), action.size()); + + using RecentActions = kv::Set; + + auto recent_actions = tx.rw(recent_actions_map); + auto key = + fmt::format("{}:{}", created_at_str, ds::to_hex(action_digest)); + + if (recent_actions->contains(key)) + { + reason = ccf::InvalidArgsReason::ActionAlreadyApplied; + return ApiResult::InvalidArgs; + } + + // In the absence of in-KV support for sorted sets, we need + // to extract them and sort them here. + std::vector replay_keys; + recent_actions->foreach([&replay_keys](const std::string& replay_key) { + replay_keys.push_back(replay_key); + return true; + }); + std::sort(replay_keys.begin(), replay_keys.end()); + + // Actions must be more recent than the median of recent actions + if (!replay_keys.empty()) + { + const auto [min_created_at, _] = + nonstd::split_1(replay_keys[replay_keys.size() / 2], ":"); + auto [key_ts, __] = nonstd::split_1(key, ":"); + if (key_ts < min_created_at) + { + reason = ccf::InvalidArgsReason::StaleActionCreatedTimestamp; + return ApiResult::InvalidArgs; + } + } + + // The action is neither stale, nor a replay + recent_actions->insert(key); + + // Only keep the most recent window_size proposals, do not + // allow the set to grow indefinitely. + // Should this be configurable through runtime options? + size_t window_size = 100; + if (replay_keys.size() >= (window_size - 1) /* We just added one */) + { + for (size_t i = 0; i < (replay_keys.size() - (window_size - 1)); i++) + { + recent_actions->remove(replay_keys[i]); + } + } + + return ApiResult::OK; + } + catch (const std::exception& e) + { + LOG_FAIL_FMT("{}", e.what()); + return ApiResult::InternalError; + } + } + + ccf::ApiResult DynamicJSEndpointRegistry::record_action_for_audit_v1( + kv::Tx& tx, + ccf::ActionFormat format, + const std::string& user_id, + const std::string& action_name, + const std::vector& action_body) + { + try + { + using AuditInputValue = kv::Value>; + using AuditInfoValue = kv::Value; + + auto audit_input = tx.template rw(audit_input_map); + audit_input->put(action_body); + + auto audit_info = tx.template rw(audit_info_map); + audit_info->put({format, user_id, action_name}); + + return ApiResult::OK; + } + catch (const std::exception& e) + { + LOG_FAIL_FMT("{}", e.what()); + return ApiResult::InternalError; + } + } } diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 64b1acc1da9a..7c9b567e9215 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -35,6 +35,7 @@ from types import MappingProxyType import threading import copy +import programmability import e2e_common_endpoints from loguru import logger as LOG @@ -1962,7 +1963,7 @@ def run_app_space_js(args): primary, user.service_id, user_data={"isAdmin": True} ) - with primary.client(None, None, user.local_id) as c: + with primary.client() as c: parent_dir = os.path.normpath( os.path.join(os.path.dirname(__file__), os.path.pardir) ) @@ -1974,15 +1975,26 @@ def run_app_space_js(args): "js", ) bundle = network.consortium.read_bundle_from_dir(logging_js_dir) - r = c.put("/app/custom_endpoints", body=bundle) + signed_bundle = programmability.sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle + ) + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Also modify the runtime options to log and return errors, to aid debugging - r = c.call( + options = {"log_exception_details": True, "return_exception_details": True} + signed_options = programmability.sign_payload( + network.identity(user.local_id), "runtime_options", options + ) + r = c.patch( "/app/custom_endpoints/runtime_options", - {"log_exception_details": True, "return_exception_details": True}, - http_verb="PATCH", + signed_options, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.OK.value, r.status_code diff --git a/tests/programmability.py b/tests/programmability.py index fb6df45fcc79..b912d8e6d86c 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -10,6 +10,8 @@ import json from infra.runner import ConcurrentRunner from governance_js import action, proposal, ballot_yes +import ccf.cose +import infra.clients import npm_tests @@ -76,6 +78,17 @@ def endpoint_properties( } +def sign_payload(identity, msg_type, json_payload): + serialised_payload = json.dumps(json_payload).encode() + key = open(identity.key, "r").read() + cert = open(identity.cert, "r").read() + phdr = { + "app.msg.type": msg_type, + "app.msg.created_at": int(infra.clients.get_clock().moment().timestamp()), + } + return ccf.cose.create_cose_sign1(serialised_payload, key, cert, phdr) + + def test_custom_endpoints(network, args): primary, _ = network.find_primary() user = network.users[0] @@ -133,10 +146,16 @@ def test_getters(c, expected_body): r.body.text() == module_def["module"] ), f"Expected:\n{module_def['module']}\n\n\nActual:\n{r.body.text()}" - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code - test_getters(c, bundle_with_content) # Install also works with cert authentication, at the expense of potential offline @@ -153,8 +172,15 @@ def test_getters(c, expected_body): assert r.status_code == http.HTTPStatus.OK.value, r.status_code assert r.body.json()["payload"] == "Test content", r.body.json() - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_other_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_other_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code test_getters(c, bundle_with_other_content) @@ -208,8 +234,15 @@ def test_custom_endpoints_circular_includes(network, args): "modules": modules, } - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=recursive_bundle) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", recursive_bundle + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code with primary.client() as c: @@ -257,8 +290,15 @@ def test_custom_endpoints_kv_restrictions(network, args): "modules": modules, } - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code with primary.client() as c: @@ -325,8 +365,14 @@ def test_custom_endpoints_js_options(network, args): ) def test_options_patch(c, **kwargs): - r = c.call( - "/app/custom_endpoints/runtime_options", {**kwargs}, http_verb="PATCH" + signed_bundle = sign_payload( + network.identity(user.local_id), "runtime_options", {**kwargs} + ) + + r = c.patch( + "/app/custom_endpoints/runtime_options", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.OK.value, r.status_code new_options = r.body.json() @@ -339,7 +385,7 @@ def test_options_patch(c, **kwargs): assert new_options == get_options, f"{new_options} != {get_options}" return new_options - with primary.client(None, None, user.local_id) as c: + with primary.client(None, None) as c: r = c.get("/app/custom_endpoints/runtime_options") assert r.status_code == http.HTTPStatus.OK.value, r.status_code @@ -411,9 +457,16 @@ def test_custom_role_definitions(network, args): "modules": [{"name": "test.js", "module": TESTJS_ROLE}], } + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_auth + ) # Install app with auth/role support - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_auth) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Add role definition @@ -481,9 +534,16 @@ def test_custom_role_definitions(network, args): "modules": [{"name": "test.js", "module": TESTJS_ROLE}], } + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_auth_both + ) # Install two endpoints with role auth - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_auth_both) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Assign the new role to user0 @@ -517,10 +577,16 @@ def deploy_npm_app_custom(network, args): app_dir, "dist", "bundle.json" ) # Produced by build_npm_app - with primary.client(None, None, user.local_id) as c: + signed_bundle = sign_payload( + network.identity(user.local_id), + "custom_endpoints", + json.load(open(bundle_path)), + ) + with primary.client() as c: r = c.put( "/app/custom_endpoints", - body=json.load(open(bundle_path)), + body=signed_bundle, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code @@ -548,6 +614,7 @@ def run(args): network = test_custom_endpoints_circular_includes(network, args) network = test_custom_endpoints_kv_restrictions(network, args) network = test_custom_role_definitions(network, args) + network = test_custom_endpoints_js_options(network, args) network = npm_tests.build_npm_app(network, args) network = deploy_npm_app_custom(network, args)