From 8306717559d5dba4e427c26b17a31ad8b67ee273 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Thu, 19 Dec 2024 15:28:22 +0000 Subject: [PATCH] Refactor `js_generic` to use common `DynamicJSEndpointRegistry` hierarchy, and expose as an extensible public header (#6710) --- CHANGELOG.md | 8 + CMakeLists.txt | 38 +- include/ccf/js/registry.h | 108 ++- .../js/samples/governance_driven_registry.h | 36 + src/apps/js_generic/js_generic.cpp | 6 +- src/apps/js_generic/js_generic_base.cpp | 802 ------------------ src/apps/js_generic/js_generic_base.h | 13 - src/js/registry.cpp | 111 ++- 8 files changed, 197 insertions(+), 925 deletions(-) create mode 100644 include/ccf/js/samples/governance_driven_registry.h delete mode 100644 src/apps/js_generic/js_generic_base.cpp delete mode 100644 src/apps/js_generic/js_generic_base.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7facdf8835..48ef3a375ec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ and this project adheres Fto [Semantic Versioning](http://semver.org/spec/v2.0.0 - The function `ccf::get_js_plugins()` and associated FFI plugin system for JS is deprecated. Similar functionality should now be implemented through a `js::Extension` returned from `DynamicJSEndpointRegistry::get_extensions()`. +## [6.0.0-dev11] + +[6.0.0-dev11]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev11 + +### Added + +- Applications can now extend `js_generic` (ie - a JS app where JS endpoints are edited by governance transactions), from the public header `ccf/js/samples/governance_driven_registry.h`. The API for existing JS-programmability apps using `DynamicJSEndpointRegistry` should be unaffected. + ## [6.0.0-dev10] [6.0.0-dev10]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev10 diff --git a/CMakeLists.txt b/CMakeLists.txt index 7af34eb84ab3..3f81b61ea28c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -416,45 +416,9 @@ set(CCF_NETWORK_TEST_ARGS ${TEST_LOGGING_LEVEL} --worker-threads ${WORKER_THREADS} ) -set(JS_GENERIC_SOURCES ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp) -if(COMPILE_TARGET STREQUAL "snp") - add_library(js_generic_base.snp STATIC ${JS_GENERIC_SOURCES}) - add_san(js_generic_base.snp) - add_warning_checks(js_generic_base.snp) - target_link_libraries(js_generic_base.snp PUBLIC ccf.snp) - target_compile_options(js_generic_base.snp PRIVATE ${COMPILE_LIBCXX}) - target_compile_definitions( - js_generic_base.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE - _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP - ) - set_property(TARGET js_generic_base.snp PROPERTY POSITION_INDEPENDENT_CODE ON) - install( - TARGETS js_generic_base.snp - EXPORT ccf - DESTINATION lib - ) -elseif(COMPILE_TARGET STREQUAL "virtual") - add_library(js_generic_base.virtual STATIC ${JS_GENERIC_SOURCES}) - add_san(js_generic_base.virtual) - add_warning_checks(js_generic_base.virtual) - target_link_libraries(js_generic_base.virtual PUBLIC ccf.virtual) - target_compile_options(js_generic_base.virtual PRIVATE ${COMPILE_LIBCXX}) - set_property( - TARGET js_generic_base.virtual PROPERTY POSITION_INDEPENDENT_CODE ON - ) - install( - TARGETS js_generic_base.virtual - EXPORT ccf - DESTINATION lib - ) -endif() # SNIPPET_START: JS generic application add_ccf_app( - js_generic - SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp - LINK_LIBS_ENCLAVE js_generic_base.enclave - LINK_LIBS_VIRTUAL js_generic_base.virtual - LINK_LIBS_SNP js_generic_base.snp INSTALL_LIBS ON + js_generic SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp INSTALL_LIBS ON ) # SNIPPET_END: JS generic application diff --git a/include/ccf/js/registry.h b/include/ccf/js/registry.h index fab02e4b173f..9cdca205dbbd 100644 --- a/include/ccf/js/registry.h +++ b/include/ccf/js/registry.h @@ -19,10 +19,13 @@ namespace ccf::js { + static constexpr auto default_js_registry_kv_prefix = + "public:custom_endpoints"; + struct CustomJSEndpoint : public ccf::endpoints::Endpoint {}; - // By subclassing DynamicJSEndpointRegistry, an application gains the + // By subclassing BaseDynamicJSEndpointRegistry, an application gains the // ability to execute custom JavaScript endpoints, and exposes the ability to // install them via install_custom_endpoints(). The JavaScript code for these // endpoints is stored in the internal KV store under a namespace configured @@ -31,31 +34,14 @@ namespace ccf::js // proposal in governance, and the payload format is currently identical, // except the controlling logic resides in the application space. // - // Known limitations: - // - // No auditability yet, COSE Sign1 auth is recommended, but the signature is - // not stored. - // No support for historical endpoints yet. - // No support for import from external modules. - // // Additional functionality compared to set_js_app: - // - // The KV namespace can be private, to keep the application confidential if + // - The KV namespace can be private, to keep the application confidential if // desired. - class DynamicJSEndpointRegistry : public ccf::UserEndpointRegistry + class BaseDynamicJSEndpointRegistry : public ccf::UserEndpointRegistry { private: std::shared_ptr interpreter_cache = nullptr; - std::string modules_map; - std::string metadata_map; - std::string interpreter_flush_map; - std::string modules_quickjs_version_map; - std::string modules_quickjs_bytecode_map; - std::string runtime_options_map; - std::string recent_actions_map; - std::string audit_input_map; - std::string audit_info_map; ccf::js::NamespaceRestriction namespace_restriction; @@ -75,10 +61,18 @@ namespace ccf::js ccf::endpoints::CommandEndpointContext& endpoint_ctx, const ccf::TxID& tx_id); + protected: + std::string modules_map; + std::string metadata_map; + std::string interpreter_flush_map; + std::string modules_quickjs_version_map; + std::string modules_quickjs_bytecode_map; + std::string runtime_options_map; + public: - DynamicJSEndpointRegistry( + BaseDynamicJSEndpointRegistry( ccf::AbstractNodeContext& context, - const std::string& kv_prefix = "public:custom_endpoints"); + const std::string& kv_prefix = default_js_registry_kv_prefix); /** * Call this to populate the KV with JS endpoint definitions, so they can @@ -133,29 +127,6 @@ namespace ccf::js */ ccf::ApiResult get_js_runtime_options_v1( ccf::JSRuntimeOptions& options, ccf::kv::ReadOnlyTx& tx); - - /** - * Record action details by storing them in KV maps using a common format, - * for the purposes of offline audit using the ledger. - */ - ccf::ApiResult record_action_for_audit_v1( - ccf::kv::Tx& tx, - ccf::ActionFormat format, - const std::string& user_id, - const std::string& action_name, - const std::vector& action_body); - - /** - * Check an action is not being replayed, by looking it up - * in the history of recent actions. To place an upper bound on the history - * size, an authenticated timestamp (@p created_at) is required. - */ - ccf::ApiResult check_action_not_replayed_v1( - ccf::kv::Tx& tx, - uint64_t created_at, - const std::span action, - ccf::InvalidArgsReason& reason); - /// \defgroup Overrides for base EndpointRegistry functions, looking up JS /// endpoints before delegating to base implementation. ///@{ @@ -172,6 +143,9 @@ namespace ccf::js const ccf::TxID& tx_id) override; void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) override; + + std::set get_allowed_verbs( + ccf::kv::Tx&, const ccf::RpcContext& rpc_ctx) override; ///@} virtual ccf::js::extensions::Extensions get_extensions( @@ -180,4 +154,48 @@ namespace ccf::js return {}; }; }; + + // Extends BaseDynamicJSEndpointRegistry with methods for making actions + // auditable and preventing replay. These should be used if apps are not + // deployed through governance, to ensure that app-modification is safely and + // clearly tracked in the ledger history + class DynamicJSEndpointRegistry : public BaseDynamicJSEndpointRegistry + { + protected: + std::string recent_actions_map; + std::string audit_input_map; + std::string audit_info_map; + + public: + DynamicJSEndpointRegistry( + ccf::AbstractNodeContext& context, + const std::string& kv_prefix = default_js_registry_kv_prefix) : + BaseDynamicJSEndpointRegistry(context, kv_prefix), + recent_actions_map(fmt::format("{}.recent_actions", kv_prefix)), + audit_input_map(fmt::format("{}.audit.input", kv_prefix)), + audit_info_map(fmt::format("{}.audit.info", kv_prefix)) + {} + + /** + * Record action details by storing them in KV maps using a common format, + * for the purposes of offline audit using the ledger. + */ + ccf::ApiResult record_action_for_audit_v1( + ccf::kv::Tx& tx, + ccf::ActionFormat format, + const std::string& user_id, + const std::string& action_name, + const std::vector& action_body); + + /** + * Check an action is not being replayed, by looking it up + * in the history of recent actions. To place an upper bound on the history + * size, an authenticated timestamp (@p created_at) is required. + */ + ccf::ApiResult check_action_not_replayed_v1( + ccf::kv::Tx& tx, + uint64_t created_at, + const std::span action, + ccf::InvalidArgsReason& reason); + }; } diff --git a/include/ccf/js/samples/governance_driven_registry.h b/include/ccf/js/samples/governance_driven_registry.h new file mode 100644 index 000000000000..ccd88d62e68f --- /dev/null +++ b/include/ccf/js/samples/governance_driven_registry.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "ccf/js/registry.h" +#include "ccf/service/tables/jsengine.h" +#include "ccf/service/tables/modules.h" + +namespace ccf::js +{ + // This sample extends the generic BaseDynamicJSEndpointRegistry to read JS + // endpoints (code, metadata, options) from governance tables. Specifically, + // tables populated by actions in the default sample CCF constitution + // (set_js_app). This can be sub-classed to modify the dispatch or execution + // behaviour, or to provide further JS extension APIs via get_extensions(). + // + // An application running this registry with no further extensions is shipped + // with the CCF releases as `js_generic`. + class GovernanceDrivenJSRegistry + : public ccf::js::BaseDynamicJSEndpointRegistry + { + public: + GovernanceDrivenJSRegistry(AbstractNodeContext& context) : + // Note: We do not pass a kv_prefix here, instead we explicitly, manually + // construct each map name to match previously used values + ccf::js::BaseDynamicJSEndpointRegistry(context) + { + modules_map = ccf::Tables::MODULES; + metadata_map = ccf::endpoints::Tables::ENDPOINTS; + interpreter_flush_map = ccf::Tables::INTERPRETER_FLUSH; + modules_quickjs_version_map = ccf::Tables::MODULES_QUICKJS_VERSION; + modules_quickjs_bytecode_map = ccf::Tables::MODULES_QUICKJS_BYTECODE; + runtime_options_map = ccf::Tables::JSENGINE; + } + }; +} // namespace ccf::js diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 836a99352c82..dc175b44b13e 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. + #include "ccf/app_interface.h" -#include "js_generic_base.h" +#include "ccf/js/samples/governance_driven_registry.h" namespace ccf { std::unique_ptr make_user_endpoints( ccf::AbstractNodeContext& context) { - return make_user_endpoints_impl(context); + return std::make_unique(context); } + } // namespace ccf diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp deleted file mode 100644 index 32a5c2a5fee9..000000000000 --- a/src/apps/js_generic/js_generic_base.cpp +++ /dev/null @@ -1,802 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. -#include "ccf/app_interface.h" -#include "ccf/crypto/key_wrap.h" -#include "ccf/crypto/rsa_key_pair.h" -#include "ccf/endpoints/authentication/all_of_auth.h" -#include "ccf/endpoints/authentication/js.h" -#include "ccf/historical_queries_adapter.h" -#include "ccf/js/common_context.h" -#include "ccf/js/core/context.h" -#include "ccf/js/core/wrapped_property_enum.h" -#include "ccf/js/extensions/ccf/consensus.h" -#include "ccf/js/extensions/ccf/historical.h" -#include "ccf/js/extensions/ccf/host.h" -#include "ccf/js/extensions/ccf/kv.h" -#include "ccf/js/extensions/ccf/request.h" -#include "ccf/js/extensions/ccf/rpc.h" -#include "ccf/js/interpreter_cache_interface.h" -#include "ccf/node/host_processes_interface.h" -#include "ccf/service/tables/jsengine.h" -#include "ccf/version.h" -#include "ds/actors.h" -#include "enclave/enclave_time.h" -#include "js/global_class_ids.h" -#include "js/modules/chained_module_loader.h" -#include "js/modules/kv_bytecode_module_loader.h" -#include "js/modules/kv_module_loader.h" -#include "node/rpc_context_impl.h" -#include "service/tables/endpoints.h" - -#include -#include -#include -#include -#include - -namespace ccf -{ - using namespace std; - using namespace ccf::kv; - using namespace ccf; - - class JSHandlers : public UserEndpointRegistry - { - private: - ccf::AbstractNodeContext& context; - std::shared_ptr interpreter_cache = - nullptr; - - void execute_request( - const ccf::js::JSDynamicEndpoint* endpoint, - ccf::endpoints::EndpointContext& endpoint_ctx) - { - if (endpoint->properties.mode == ccf::endpoints::Mode::Historical) - { - auto is_tx_committed = - [this](ccf::View view, ccf::SeqNo seqno, std::string& error_reason) { - return ccf::historical::is_tx_committed_v2( - consensus, view, seqno, error_reason); - }; - - ccf::historical::read_write_adapter_v4( - [this, endpoint]( - ccf::endpoints::EndpointContext& endpoint_ctx, - ccf::historical::StatePtr state) { - auto add_historical_globals = [&](js::core::Context& ctx) { - auto ccf = - ctx.get_or_create_global_property("ccf", ctx.new_obj()); - auto extension = - ctx.get_extension(); - if (extension != nullptr) - { - auto val = - extension->create_historical_state_object(ctx, state); - ccf.set("historicalState", std::move(val)); - } - else - { - LOG_FAIL_FMT( - "Error while inserting historicalState into JS interpreter - " - "no extension found"); - } - }; - do_execute_request(endpoint, endpoint_ctx, add_historical_globals); - }, - context, - is_tx_committed)(endpoint_ctx); - } - else - { - do_execute_request(endpoint, endpoint_ctx); - } - } - - using PreExecutionHook = std::function; - - void do_execute_request( - const ccf::js::JSDynamicEndpoint* endpoint, - ccf::endpoints::EndpointContext& endpoint_ctx, - const std::optional& pre_exec_hook = std::nullopt) - { - // This KV Value should be updated by any governance actions which modify - // the JS app (including _any_ of its contained modules). We then use the - // version where it was last modified as a safe approximation of when an - // interpreter is unsafe to use. If this value is written to, the - // version_of_previous_write will advance, and all cached interpreters - // will be flushed. - const auto interpreter_flush = endpoint_ctx.tx.ro( - ccf::Tables::INTERPRETER_FLUSH); - const auto flush_marker = - interpreter_flush->get_version_of_previous_write().value_or(0); - - const std::optional js_runtime_options = - endpoint_ctx.tx.ro(ccf::Tables::JSENGINE)->get(); - if (js_runtime_options.has_value()) - { - interpreter_cache->set_max_cached_interpreters( - js_runtime_options->max_cached_interpreters); - } - - const auto rw_access = - endpoint->properties.mode == ccf::endpoints::Mode::ReadWrite ? - js::TxAccess::APP_RW : - js::TxAccess::APP_RO; - std::shared_ptr interpreter = - interpreter_cache->get_interpreter( - rw_access, endpoint->properties.interpreter_reuse, flush_marker); - if (interpreter == nullptr) - { - throw std::logic_error("Cache failed to produce interpreter"); - } - js::core::Context& ctx = *interpreter; - - // Prevent any other thread modifying this interpreter, until this - // function completes. We could create interpreters per-thread, but then - // we would get no cross-thread caching benefit (and would need to either - // enforce, or share, caps across per-thread caches). We choose - // instead to allow interpreters to be maximally reused, even across - // threads, at the cost of locking (and potentially stalling another - // thread's request execution) here. - std::lock_guard guard(ctx.lock); - // Update the top of the stack for the current thread, used by the stack - // guard Note this is only active outside SGX - JS_UpdateStackTop(ctx.runtime()); - // Make the heap and stack limits safe while we init the runtime - ctx.runtime().reset_runtime_options(); - - ccf::js::modules::ModuleLoaders sub_loaders = { - std::make_shared( - endpoint_ctx.tx.ro( - ccf::Tables::MODULES_QUICKJS_BYTECODE), - endpoint_ctx.tx.ro( - ccf::Tables::MODULES_QUICKJS_VERSION)), - std::make_shared( - endpoint_ctx.tx.ro(ccf::Tables::MODULES))}; - auto module_loader = std::make_shared( - std::move(sub_loaders)); - ctx.set_module_loader(std::move(module_loader)); - - // Extensions with a dependency on this endpoint context (invocation), - // which must be removed after execution. - js::extensions::Extensions local_extensions; - - // ccf.kv.* - local_extensions.emplace_back( - std::make_shared(&endpoint_ctx.tx)); - - // ccf.rpc.* - local_extensions.emplace_back( - std::make_shared( - endpoint_ctx.rpc_ctx.get())); - - auto request_extension = - std::make_shared( - endpoint_ctx.rpc_ctx.get()); - local_extensions.push_back(request_extension); - - for (auto extension : local_extensions) - { - ctx.add_extension(extension); - } - - if (pre_exec_hook.has_value()) - { - pre_exec_hook.value()(ctx); - } - - js::core::JSWrappedValue export_func; - try - { - const auto& props = endpoint->properties; - auto module_val = ctx.get_module(props.js_module); - if (!module_val.has_value()) - { - throw std::runtime_error( - fmt::format("Unable to load module: {}", props.js_module)); - } - export_func = ctx.get_exported_function( - *module_val, props.js_function, props.js_module); - } - catch (const std::exception& exc) - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - exc.what()); - return; - } - - // Call exported function; - auto request = request_extension->create_request_obj( - ctx, endpoint->full_uri_path, endpoint_ctx, this); - - auto options = endpoint_ctx.tx.ro(ccf::Tables::JSENGINE) - ->get() - .value_or(ccf::JSRuntimeOptions()); - - auto val = ctx.call_with_rt_options( - export_func, - {request}, - options, - ccf::js::core::RuntimeLimitsPolicy::NONE); - - for (auto extension : local_extensions) - { - ctx.remove_extension(extension); - } - - ctx.set_module_loader(nullptr); - - const auto& rt = ctx.runtime(); - - if (val.is_exception()) - { - bool time_out = ctx.interrupt_data.request_timed_out; - std::string error_msg = "Exception thrown while executing."; - if (time_out) - { - error_msg = "Operation took too long to complete."; - } - - auto [reason, trace] = ctx.error_message(); - - if (options.log_exception_details) - { - CCF_APP_FAIL("{}: {}", reason, trace.value_or("")); - } - - if (options.return_exception_details) - { - std::vector details = { - ODataJSExceptionDetails{ccf::errors::JSException, reason, trace}}; - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - std::move(error_msg), - std::move(details)); - } - else - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - std::move(error_msg)); - } - - return; - } - - // Handle return value: {body, headers, statusCode} - if (!val.is_obj()) - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (not an object)."); - return; - } - - // Response body (also sets a default response content-type header) - { - auto response_body_js = val["body"]; - if (!response_body_js.is_undefined()) - { - std::vector response_body; - size_t buf_size; - size_t buf_offset; - auto typed_array_buffer = ctx.get_typed_array_buffer( - response_body_js, &buf_offset, &buf_size, nullptr); - uint8_t* array_buffer; - if (!typed_array_buffer.is_exception()) - { - size_t buf_size_total; - array_buffer = - JS_GetArrayBuffer(ctx, &buf_size_total, typed_array_buffer.val); - array_buffer += buf_offset; - } - else - { - array_buffer = - JS_GetArrayBuffer(ctx, &buf_size, response_body_js.val); - } - if (array_buffer) - { - endpoint_ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - http::headervalues::contenttype::OCTET_STREAM); - response_body = - std::vector(array_buffer, array_buffer + buf_size); - } - else - { - std::optional str; - if (response_body_js.is_str()) - { - endpoint_ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - http::headervalues::contenttype::TEXT); - str = ctx.to_str(response_body_js); - } - else - { - endpoint_ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - http::headervalues::contenttype::JSON); - auto rval = ctx.json_stringify(response_body_js); - if (rval.is_exception()) - { - auto [reason, trace] = ctx.error_message(); - - if (options.log_exception_details) - { - CCF_APP_FAIL( - "Failed to convert return value to JSON:{} {}", - reason, - trace.value_or("")); - } - - if (options.return_exception_details) - { - std::vector details = { - ODataJSExceptionDetails{ - ccf::errors::JSException, reason, trace}}; - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (error during JSON " - "conversion of body)", - std::move(details)); - } - else - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (error during JSON " - "conversion of body)."); - } - return; - } - str = ctx.to_str(rval); - } - - if (!str) - { - auto [reason, trace] = ctx.error_message(); - - if (options.log_exception_details) - { - CCF_APP_FAIL( - "Failed to convert return value to JSON:{} {}", - reason, - trace.value_or("")); - } - - if (options.return_exception_details) - { - std::vector details = {ODataJSExceptionDetails{ - ccf::errors::JSException, reason, trace}}; - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (error during string " - "conversion of body).", - std::move(details)); - } - else - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (error during string " - "conversion of body)."); - } - return; - } - - response_body = std::vector(str->begin(), str->end()); - } - endpoint_ctx.rpc_ctx->set_response_body(std::move(response_body)); - } - } - - // Response headers - { - auto response_headers_js = val["headers"]; - if (response_headers_js.is_obj()) - { - js::core::JSWrappedPropertyEnum prop_enum(ctx, response_headers_js); - for (size_t i = 0; i < prop_enum.size(); i++) - { - auto prop_name = ctx.to_str(prop_enum[i]); - if (!prop_name) - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (header type)."); - return; - } - auto prop_val = response_headers_js[*prop_name]; - auto prop_val_str = ctx.to_str(prop_val); - if (!prop_val_str) - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (header value type)."); - return; - } - endpoint_ctx.rpc_ctx->set_response_header( - *prop_name, *prop_val_str); - } - } - } - - // Response status code - int response_status_code = HTTP_STATUS_OK; - { - auto status_code_js = val["statusCode"]; - if (!status_code_js.is_undefined() && !JS_IsNull(status_code_js.val)) - { - if (JS_VALUE_GET_TAG(status_code_js.val) != JS_TAG_INT) - { - endpoint_ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - "Invalid endpoint function return value (status code type)."); - return; - } - response_status_code = JS_VALUE_GET_INT(status_code_js.val); - } - endpoint_ctx.rpc_ctx->set_response_status(response_status_code); - } - - // Log execution metrics - if (ctx.log_execution_metrics) - { - const auto time_now = ccf::get_enclave_time(); - // Although enclave time returns a microsecond value, the actual - // precision/granularity depends on the host's TimeUpdater. By default - // this only advances each millisecond. Avoid implying more precision - // than that, by rounding to milliseconds - const auto exec_time = - std::chrono::duration_cast( - time_now - ctx.interrupt_data.start_time); - CCF_LOG_FMT(INFO, "js") - ("JS execution complete: Method={}, Path={}, Status={}, " - "ExecMilliseconds={}", - endpoint->dispatch.verb.c_str(), - endpoint->full_uri_path, - response_status_code, - exec_time.count()); - } - - return; - } - - void execute_request_locally_committed( - const ccf::js::JSDynamicEndpoint* endpoint, - ccf::endpoints::CommandEndpointContext& endpoint_ctx, - const ccf::TxID& tx_id) - { - ccf::endpoints::default_locally_committed_func(endpoint_ctx, tx_id); - } - - public: - JSHandlers(AbstractNodeContext& context) : - UserEndpointRegistry(context), - context(context) - { - interpreter_cache = - context.get_subsystem(); - if (interpreter_cache == nullptr) - { - throw std::logic_error( - "Unexpected: Could not access AbstractInterpreterCache subsytem"); - } - - // Install dependency-less (ie reusable) extensions on interpreters _at - // creation_, rather than on every run - js::extensions::Extensions extensions; - - // add ccf.consensus.* - extensions.emplace_back( - std::make_shared(this)); - // add ccf.host.* - extensions.emplace_back( - std::make_shared( - context.get_subsystem().get())); - // add ccf.historical.* - extensions.emplace_back( - std::make_shared( - &context.get_historical_state())); - - interpreter_cache->set_interpreter_factory( - [extensions](ccf::js::TxAccess access) { - // CommonContext also adds many extensions - auto interpreter = std::make_shared(access); - - for (auto extension : extensions) - { - interpreter->add_extension(extension); - } - - return interpreter; - }); - } - - void instantiate_authn_policies(ccf::js::JSDynamicEndpoint& endpoint) - { - for (const auto& policy_desc : endpoint.properties.authn_policies) - { - if (policy_desc.is_string()) - { - const auto policy_name = policy_desc.get(); - auto policy = get_policy_by_name(policy_name); - if (policy == nullptr) - { - throw std::logic_error( - fmt::format("Unknown auth policy: {}", policy_name)); - } - endpoint.authn_policies.push_back(std::move(policy)); - } - else - { - if (policy_desc.is_object()) - { - const auto it = policy_desc.find("all_of"); - if (it != policy_desc.end()) - { - if (it.value().is_array()) - { - std::vector> - constituent_policies; - for (const auto& val : it.value()) - { - if (!val.is_string()) - { - constituent_policies.clear(); - break; - } - - const auto policy_name = val.get(); - auto policy = get_policy_by_name(policy_name); - if (policy == nullptr) - { - throw std::logic_error( - fmt::format("Unknown auth policy: {}", policy_name)); - } - constituent_policies.push_back(std::move(policy)); - } - - if (!constituent_policies.empty()) - { - endpoint.authn_policies.push_back( - std::make_shared( - constituent_policies)); - continue; - } - } - } - } - - // Any failure in above checks falls through to this detailed error. - throw std::logic_error(fmt::format( - "Unsupported auth policy. Policies must be either a string, or an " - "object containing an \"all_of\" key with list-of-strings value. " - "Unsupported value: {}", - policy_desc.dump())); - } - } - } - - ccf::endpoints::EndpointDefinitionPtr find_endpoint( - ccf::kv::Tx& tx, ccf::RpcContext& rpc_ctx) override - { - const auto method = rpc_ctx.get_method(); - const auto verb = rpc_ctx.get_request_verb(); - - auto endpoints = - tx.ro(ccf::endpoints::Tables::ENDPOINTS); - - const auto key = ccf::endpoints::EndpointKey{method, verb}; - - // Look for a direct match of the given path - const auto it = endpoints->get(key); - if (it.has_value()) - { - auto endpoint_def = std::make_shared(); - endpoint_def->dispatch = key; - endpoint_def->properties = it.value(); - endpoint_def->full_uri_path = - fmt::format("/{}{}", method_prefix, endpoint_def->dispatch.uri_path); - instantiate_authn_policies(*endpoint_def); - return endpoint_def; - } - - // If that doesn't exist, look through _all_ the endpoints to find - // templated matches. If there is one, that's a match. More is an error, - // none means delegate to the base class. - { - std::vector matches; - - endpoints->foreach_key([this, &endpoints, &matches, &key, &rpc_ctx]( - const auto& other_key) { - if (key.verb == other_key.verb) - { - const auto opt_spec = - ccf::endpoints::PathTemplateSpec::parse(other_key.uri_path); - if (opt_spec.has_value()) - { - const auto& template_spec = opt_spec.value(); - // This endpoint has templates in its path, and the correct verb - // - now check if template matches the current request's path - std::smatch match; - if (std::regex_match( - key.uri_path, match, template_spec.template_regex)) - { - if (matches.empty()) - { - auto ctx_impl = static_cast(&rpc_ctx); - if (ctx_impl == nullptr) - { - throw std::logic_error("Unexpected type of RpcContext"); - } - // Populate the request_path_params while we have the match, - // though this will be discarded on error if we later find - // multiple matches - auto& path_params = ctx_impl->path_params; - for (size_t i = 0; - i < template_spec.template_component_names.size(); - ++i) - { - const auto& template_name = - template_spec.template_component_names[i]; - const auto& template_value = match[i + 1].str(); - path_params[template_name] = template_value; - } - } - - auto endpoint = std::make_shared(); - endpoint->dispatch = other_key; - endpoint->full_uri_path = fmt::format( - "/{}{}", method_prefix, endpoint->dispatch.uri_path); - endpoint->properties = endpoints->get(other_key).value(); - instantiate_authn_policies(*endpoint); - matches.push_back(endpoint); - } - } - } - return true; - }); - - if (matches.size() > 1) - { - report_ambiguous_templated_path(key.uri_path, matches); - } - else if (matches.size() == 1) - { - return matches[0]; - } - } - - return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); - } - - std::set get_allowed_verbs( - ccf::kv::Tx& tx, const ccf::RpcContext& rpc_ctx) override - { - const auto method = rpc_ctx.get_method(); - - std::set verbs = - ccf::endpoints::EndpointRegistry::get_allowed_verbs(tx, rpc_ctx); - - auto endpoints = - tx.ro(ccf::endpoints::Tables::ENDPOINTS); - - endpoints->foreach_key([this, &verbs, &method](const auto& key) { - const auto opt_spec = - ccf::endpoints::PathTemplateSpec::parse(key.uri_path); - if (opt_spec.has_value()) - { - const auto& template_spec = opt_spec.value(); - // This endpoint has templates in its path - now check if template - // matches the current request's path - std::smatch match; - if (std::regex_match(method, match, template_spec.template_regex)) - { - verbs.insert(key.verb); - } - } - else if (key.uri_path == method) - { - verbs.insert(key.verb); - } - return true; - }); - - return verbs; - } - - void execute_endpoint( - ccf::endpoints::EndpointDefinitionPtr e, - ccf::endpoints::EndpointContext& endpoint_ctx) override - { - auto endpoint = dynamic_cast(e.get()); - if (endpoint != nullptr) - { - execute_request(endpoint, endpoint_ctx); - return; - } - - ccf::endpoints::EndpointRegistry::execute_endpoint(e, endpoint_ctx); - } - - void execute_endpoint_locally_committed( - ccf::endpoints::EndpointDefinitionPtr e, - ccf::endpoints::CommandEndpointContext& endpoint_ctx, - const ccf::TxID& tx_id) override - { - auto endpoint = dynamic_cast(e.get()); - if (endpoint != nullptr) - { - execute_request_locally_committed(endpoint, endpoint_ctx, tx_id); - return; - } - - ccf::endpoints::EndpointRegistry::execute_endpoint_locally_committed( - e, endpoint_ctx, tx_id); - } - - // Since we do our own dispatch within the default handler, report the - // supported methods here - void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) override - { - UserEndpointRegistry::build_api(document, tx); - - auto endpoints = - tx.ro(ccf::endpoints::Tables::ENDPOINTS); - - endpoints->foreach([&document](const auto& key, const auto& properties) { - const auto http_verb = key.verb.get_http_method(); - if (!http_verb.has_value()) - { - return true; - } - - if (!properties.openapi_hidden) - { - auto& path_op = ds::openapi::path_operation( - ds::openapi::path( - document, - fmt::format( - "/{}{}", - ccf::get_actor_prefix(ccf::ActorsType::users), - key.uri_path)), - http_verb.value(), - false); - if (!properties.openapi.empty()) - { - for (const auto& [k, v] : properties.openapi.items()) - { - LOG_INFO_FMT("Inserting field {}", k); - } - path_op.insert( - properties.openapi.cbegin(), properties.openapi.cend()); - } - } - - return true; - }); - } - }; - - std::unique_ptr make_user_endpoints_impl( - ccf::AbstractNodeContext& context) - { - return std::make_unique(context); - } - -} // namespace ccf diff --git a/src/apps/js_generic/js_generic_base.h b/src/apps/js_generic/js_generic_base.h deleted file mode 100644 index d992358fab2e..000000000000 --- a/src/apps/js_generic/js_generic_base.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. -#pragma once - -#include "ccf/app_interface.h" - -#include - -namespace ccf -{ - std::unique_ptr make_user_endpoints_impl( - ccf::AbstractNodeContext& context); -} \ No newline at end of file diff --git a/src/js/registry.cpp b/src/js/registry.cpp index 2078b8cd8f77..d446b4ac596b 100644 --- a/src/js/registry.cpp +++ b/src/js/registry.cpp @@ -34,6 +34,7 @@ #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/interpreter_cache_interface.h" #include "ds/actors.h" +#include "enclave/enclave_time.h" #include "js/modules/chained_module_loader.h" #include "js/modules/kv_bytecode_module_loader.h" #include "js/modules/kv_module_loader.h" @@ -51,7 +52,7 @@ namespace ccf::js return std::string(sv); } - void DynamicJSEndpointRegistry::do_execute_request( + void BaseDynamicJSEndpointRegistry::do_execute_request( const CustomJSEndpoint* endpoint, ccf::endpoints::EndpointContext& endpoint_ctx, const std::optional& pre_exec_hook) @@ -67,6 +68,14 @@ namespace ccf::js const auto flush_marker = interpreter_flush->get_version_of_previous_write().value_or(0); + auto options_opt = + endpoint_ctx.tx.ro(runtime_options_map)->get(); + if (options_opt.has_value()) + { + interpreter_cache->set_max_cached_interpreters( + options_opt->max_cached_interpreters); + } + const auto rw_access = endpoint->properties.mode == ccf::endpoints::Mode::ReadWrite ? ccf::js::TxAccess::APP_RW : @@ -159,9 +168,7 @@ namespace ccf::js auto request = request_extension->create_request_obj( ctx, endpoint->full_uri_path, endpoint_ctx, this); - auto options = endpoint_ctx.tx.ro(runtime_options_map) - ->get() - .value_or(ccf::JSRuntimeOptions()); + const auto options = options_opt.value_or(ccf::JSRuntimeOptions()); auto val = ctx.call_with_rt_options( export_func, @@ -399,9 +406,29 @@ namespace ccf::js } endpoint_ctx.rpc_ctx->set_response_status(response_status_code); } + + // Log execution metrics + if (ctx.log_execution_metrics) + { + const auto time_now = ccf::get_enclave_time(); + // Although enclave time returns a microsecond value, the actual + // precision/granularity depends on the host's TimeUpdater. By default + // this only advances each millisecond. Avoid implying more precision + // than that, by rounding to milliseconds + const auto exec_time = + std::chrono::duration_cast( + time_now - ctx.interrupt_data.start_time); + CCF_LOG_FMT(INFO, "js") + ("JS execution complete: Method={}, Path={}, Status={}, " + "ExecMilliseconds={}", + endpoint->dispatch.verb.c_str(), + endpoint->full_uri_path, + response_status_code, + exec_time.count()); + } } - void DynamicJSEndpointRegistry::execute_request( + void BaseDynamicJSEndpointRegistry::execute_request( const CustomJSEndpoint* endpoint, ccf::endpoints::EndpointContext& endpoint_ctx) { @@ -444,7 +471,7 @@ namespace ccf::js } } - void DynamicJSEndpointRegistry::execute_request_locally_committed( + void BaseDynamicJSEndpointRegistry::execute_request_locally_committed( const CustomJSEndpoint* endpoint, ccf::endpoints::CommandEndpointContext& endpoint_ctx, const ccf::TxID& tx_id) @@ -452,7 +479,7 @@ namespace ccf::js ccf::endpoints::default_locally_committed_func(endpoint_ctx, tx_id); } - DynamicJSEndpointRegistry::DynamicJSEndpointRegistry( + BaseDynamicJSEndpointRegistry::BaseDynamicJSEndpointRegistry( ccf::AbstractNodeContext& context, const std::string& kv_prefix) : ccf::UserEndpointRegistry(context), modules_map(fmt::format("{}.modules", kv_prefix)), @@ -462,10 +489,7 @@ namespace ccf::js fmt::format("{}.modules_quickjs_version", kv_prefix)), modules_quickjs_bytecode_map( fmt::format("{}.modules_quickjs_bytecode", kv_prefix)), - runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)), - recent_actions_map(fmt::format("{}.recent_actions", kv_prefix)), - audit_input_map(fmt::format("{}.audit.input", kv_prefix)), - audit_info_map(fmt::format("{}.audit.info", kv_prefix)) + runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)) { interpreter_cache = context.get_subsystem(); @@ -505,7 +529,7 @@ namespace ccf::js }); } - ccf::ApiResult DynamicJSEndpointRegistry::install_custom_endpoints_v1( + ccf::ApiResult BaseDynamicJSEndpointRegistry::install_custom_endpoints_v1( ccf::kv::Tx& tx, const ccf::js::Bundle& bundle) { try @@ -585,7 +609,7 @@ namespace ccf::js } } - ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoints_v1( + ccf::ApiResult BaseDynamicJSEndpointRegistry::get_custom_endpoints_v1( ccf::js::Bundle& bundle, ccf::kv::ReadOnlyTx& tx) { try @@ -630,11 +654,12 @@ namespace ccf::js } } - ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoint_properties_v1( - ccf::endpoints::EndpointProperties& properties, - ccf::kv::ReadOnlyTx& tx, - const ccf::RESTVerb& verb, - const ccf::endpoints::URI& uri) + ccf::ApiResult BaseDynamicJSEndpointRegistry:: + get_custom_endpoint_properties_v1( + ccf::endpoints::EndpointProperties& properties, + ccf::kv::ReadOnlyTx& tx, + const ccf::RESTVerb& verb, + const ccf::endpoints::URI& uri) { try { @@ -659,7 +684,7 @@ namespace ccf::js } } - ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoint_module_v1( + ccf::ApiResult BaseDynamicJSEndpointRegistry::get_custom_endpoint_module_v1( std::string& code, ccf::kv::ReadOnlyTx& tx, const std::string& module_name) { try @@ -684,13 +709,13 @@ namespace ccf::js } } - void DynamicJSEndpointRegistry::set_js_kv_namespace_restriction( + void BaseDynamicJSEndpointRegistry::set_js_kv_namespace_restriction( const ccf::js::NamespaceRestriction& nr) { namespace_restriction = nr; } - ccf::ApiResult DynamicJSEndpointRegistry::set_js_runtime_options_v1( + ccf::ApiResult BaseDynamicJSEndpointRegistry::set_js_runtime_options_v1( ccf::kv::Tx& tx, const ccf::JSRuntimeOptions& options) { try @@ -704,7 +729,7 @@ namespace ccf::js } } - ccf::ApiResult DynamicJSEndpointRegistry::get_js_runtime_options_v1( + ccf::ApiResult BaseDynamicJSEndpointRegistry::get_js_runtime_options_v1( ccf::JSRuntimeOptions& options, ccf::kv::ReadOnlyTx& tx) { try @@ -721,7 +746,7 @@ namespace ccf::js } } - ccf::endpoints::EndpointDefinitionPtr DynamicJSEndpointRegistry:: + ccf::endpoints::EndpointDefinitionPtr BaseDynamicJSEndpointRegistry:: find_endpoint(ccf::kv::Tx& tx, ccf::RpcContext& rpc_ctx) { // Look up the endpoint definition @@ -814,7 +839,7 @@ namespace ccf::js return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); } - void DynamicJSEndpointRegistry::execute_endpoint( + void BaseDynamicJSEndpointRegistry::execute_endpoint( ccf::endpoints::EndpointDefinitionPtr e, ccf::endpoints::EndpointContext& endpoint_ctx) { @@ -829,7 +854,7 @@ namespace ccf::js ccf::endpoints::EndpointRegistry::execute_endpoint(e, endpoint_ctx); } - void DynamicJSEndpointRegistry::execute_endpoint_locally_committed( + void BaseDynamicJSEndpointRegistry::execute_endpoint_locally_committed( ccf::endpoints::EndpointDefinitionPtr e, ccf::endpoints::CommandEndpointContext& endpoint_ctx, const ccf::TxID& tx_id) @@ -847,7 +872,7 @@ namespace ccf::js // Since we do our own dispatch (overriding find_endpoint), make sure we // describe those operations in the auto-generated OpenAPI - void DynamicJSEndpointRegistry::build_api( + void BaseDynamicJSEndpointRegistry::build_api( nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) { ccf::UserEndpointRegistry::build_api(document, tx); @@ -883,6 +908,40 @@ namespace ccf::js }); } + std::set BaseDynamicJSEndpointRegistry::get_allowed_verbs( + ccf::kv::Tx& tx, const ccf::RpcContext& rpc_ctx) + { + const auto method = rpc_ctx.get_method(); + + std::set verbs = + ccf::endpoints::EndpointRegistry::get_allowed_verbs(tx, rpc_ctx); + + auto endpoints = tx.template ro(metadata_map); + + endpoints->foreach_key([this, &verbs, &method](const auto& key) { + const auto opt_spec = + ccf::endpoints::PathTemplateSpec::parse(key.uri_path); + if (opt_spec.has_value()) + { + const auto& template_spec = opt_spec.value(); + // This endpoint has templates in its path - now check if template + // matches the current request's path + std::smatch match; + if (std::regex_match(method, match, template_spec.template_regex)) + { + verbs.insert(key.verb); + } + } + else if (key.uri_path == method) + { + verbs.insert(key.verb); + } + return true; + }); + + return verbs; + } + ccf::ApiResult DynamicJSEndpointRegistry::check_action_not_replayed_v1( ccf::kv::Tx& tx, uint64_t created_at,