From 69419d59f66318e9b82af256d8965cdacd0c6646 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 15 May 2024 16:29:27 +0000 Subject: [PATCH 01/25] Move dispatch and install skeleton from logging to basic --- include/ccf/bundle.h | 39 +++++ include/ccf/endpoint.h | 34 +++- include/ccf/endpoints/authentication/js.h | 150 ++++++++++++++++++ samples/apps/basic/basic.cpp | 75 ++++++++- .../apps/basic/custom_endpoints/endpoint.h | 10 ++ src/service/tables/endpoints.h | 29 ---- 6 files changed, 305 insertions(+), 32 deletions(-) create mode 100644 include/ccf/bundle.h create mode 100644 include/ccf/endpoints/authentication/js.h create mode 100644 samples/apps/basic/custom_endpoints/endpoint.h diff --git a/include/ccf/bundle.h b/include/ccf/bundle.h new file mode 100644 index 000000000000..7e773594735b --- /dev/null +++ b/include/ccf/bundle.h @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "ccf/ds/json.h" +#include "ccf/endpoint.h" + +#include +#include + +namespace ccf::js +{ + struct Metadata + { + std::map< + std::string, + std::map> + endpoints; + }; + DECLARE_JSON_TYPE(Metadata); + DECLARE_JSON_REQUIRED_FIELDS(Metadata, endpoints); + + struct Bundle + { + std::map modules; + Metadata metadata; + }; + + DECLARE_JSON_TYPE(Bundle); + DECLARE_JSON_REQUIRED_FIELDS(Bundle, modules, metadata); + + struct BundleWrapper + { + Bundle bundle; + }; + + DECLARE_JSON_TYPE(BundleWrapper); + DECLARE_JSON_REQUIRED_FIELDS(BundleWrapper, bundle); +} \ No newline at end of file diff --git a/include/ccf/endpoint.h b/include/ccf/endpoint.h index 24543b83cef2..3ad91549217c 100644 --- a/include/ccf/endpoint.h +++ b/include/ccf/endpoint.h @@ -28,7 +28,39 @@ namespace ccf::endpoints return fmt::format("{} {}", verb.c_str(), uri_path); } }; +} + +namespace kv::serialisers +{ + template <> + struct BlitSerialiser + { + static SerialisedEntry to_serialised( + const ccf::endpoints::EndpointKey& endpoint_key) + { + auto str = + fmt::format("{} {}", endpoint_key.verb.c_str(), endpoint_key.uri_path); + return SerialisedEntry(str.begin(), str.end()); + } + static ccf::endpoints::EndpointKey from_serialised( + const SerialisedEntry& data) + { + std::string str{data.begin(), data.end()}; + auto i = str.find(' '); + if (i == std::string::npos) + { + throw std::logic_error("invalid encoding of endpoint key"); + } + auto verb = str.substr(0, i); + auto uri_path = str.substr(i + 1); + return {uri_path, verb}; + } + }; +} + +namespace ccf::endpoints +{ DECLARE_JSON_TYPE(EndpointKey); DECLARE_JSON_REQUIRED_FIELDS(EndpointKey, uri_path, verb); @@ -467,4 +499,4 @@ struct formatter return format_to(ctx.out(), "{}", s); } }; -FMT_END_NAMESPACE \ No newline at end of file +FMT_END_NAMESPACE diff --git a/include/ccf/endpoints/authentication/js.h b/include/ccf/endpoints/authentication/js.h new file mode 100644 index 000000000000..eac748b3e784 --- /dev/null +++ b/include/ccf/endpoints/authentication/js.h @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ccf/endpoint.h" +#include "ccf/endpoints/authentication/all_of_auth.h" + +namespace ccf +{ + using NamedAuthPolicies = + std::unordered_map>; + + static inline NamedAuthPolicies& auth_policies_by_name() + { + static NamedAuthPolicies policies; + if (policies.empty()) + { + policies.emplace( + ccf::UserCertAuthnPolicy::SECURITY_SCHEME_NAME, + ccf::user_cert_auth_policy); + + policies.emplace( + ccf::MemberCertAuthnPolicy::SECURITY_SCHEME_NAME, + ccf::member_cert_auth_policy); + + policies.emplace( + ccf::JwtAuthnPolicy::SECURITY_SCHEME_NAME, ccf::jwt_auth_policy); + + policies.emplace( + ccf::UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME, + ccf::user_cose_sign1_auth_policy); + + policies.emplace( + ccf::EmptyAuthnPolicy::SECURITY_SCHEME_NAME, ccf::empty_auth_policy); + } + + return policies; + } + + static inline std::shared_ptr get_policy_by_name( + const std::string& name) + { + auto& policies = auth_policies_by_name(); + auto it = policies.find(name); + if (it == policies.end()) + { + return nullptr; + } + + return it->second; + } + + template + static inline constexpr char const* get_policy_name_from_ident(const T*) + { + if constexpr (std::is_same_v) + { + return ccf::UserCertAuthnPolicy::SECURITY_SCHEME_NAME; + } + else if constexpr (std::is_same_v) + { + return ccf::MemberCertAuthnPolicy::SECURITY_SCHEME_NAME; + } + else if constexpr (std::is_same_v) + { + return ccf::JwtAuthnPolicy::SECURITY_SCHEME_NAME; + } + else if constexpr (std::is_same_v) + { + return ccf::UserCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME; + } + else if constexpr (std::is_same_v) + { + return ccf::MemberCOSESign1AuthnPolicy::SECURITY_SCHEME_NAME; + } + else if constexpr (std::is_same_v) + { + return ccf::EmptyAuthnPolicy::SECURITY_SCHEME_NAME; + } + else + { + return nullptr; + } + } + + static inline void instantiate_authn_policies( + ccf::endpoints::EndpointDefinition& endpoint) + { + for (const auto& policy_desc : endpoint.properties.authn_policies) + { + if (policy_desc.is_string()) + { + const auto policy_name = policy_desc.get(); + auto policy = get_policy_by_name(policy_name); + if (policy == nullptr) + { + throw std::logic_error( + fmt::format("Unknown auth policy: {}", policy_name)); + } + endpoint.authn_policies.push_back(std::move(policy)); + } + else + { + if (policy_desc.is_object()) + { + const auto it = policy_desc.find("all_of"); + if (it != policy_desc.end()) + { + if (it.value().is_array()) + { + std::vector> + constituent_policies; + for (const auto& val : it.value()) + { + if (!val.is_string()) + { + constituent_policies.clear(); + break; + } + + const auto policy_name = val.get(); + auto policy = get_policy_by_name(policy_name); + if (policy == nullptr) + { + throw std::logic_error( + fmt::format("Unknown auth policy: {}", policy_name)); + } + constituent_policies.push_back(std::move(policy)); + } + + if (!constituent_policies.empty()) + { + endpoint.authn_policies.push_back( + std::make_shared( + constituent_policies)); + continue; + } + } + } + } + + // Any failure in above checks falls through to this detailed error. + throw std::logic_error(fmt::format( + "Unsupported auth policy. Policies must be either a string, or an " + "object containing an \"all_of\" key with list-of-strings value. " + "Unsupported value: {}", + policy_desc.dump())); + } + } + } +} \ No newline at end of file diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 5c5250aa4d22..59519ab80e0d 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -13,12 +13,16 @@ #define FMT_HEADER_ONLY #include -using namespace std; +// Custom Endpoints +#include "ccf/endpoints/authentication/js.h" +#include "custom_endpoints/endpoint.h" +#include "js/interpreter_cache_interface.h" + using namespace nlohmann; namespace basicapp { - using RecordsMap = kv::Map>; + using RecordsMap = kv::Map>; static constexpr auto PRIVATE_RECORDS = "records"; class BasicHandlers : public ccf::UserEndpointRegistry @@ -106,6 +110,73 @@ namespace basicapp make_endpoint("/records", HTTP_POST, post, {ccf::user_cert_auth_policy}) .install(); } + + // Custom Endpoints + + ccf::endpoints::EndpointDefinitionPtr find_endpoint( + kv::Tx& tx, ccf::RpcContext& rpc_ctx) override + { + // Look up the endpoint definition + // First in the user-defined endpoints, and then fall-back to built-ins + const auto method = rpc_ctx.get_method(); + const auto verb = rpc_ctx.get_request_verb(); + + auto endpoints = + tx.ro("custom_endpoints.metadata"); + const auto key = ccf::endpoints::EndpointKey{method, verb}; + + // Look for a direct match of the given path + const auto it = endpoints->get(key); + if (it.has_value()) + { + auto endpoint_def = std::make_shared(); + endpoint_def->dispatch = key; + endpoint_def->properties = it.value(); + endpoint_def->full_uri_path = + fmt::format("/{}{}", method_prefix, endpoint_def->dispatch.uri_path); + ccf::instantiate_authn_policies(*endpoint_def); + return endpoint_def; + } + + // TBD: templated endpoints + return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); + } + + using PreExecutionHook = std::function; + + void do_execute_request( + const CustomJSEndpoint* endpoint, + ccf::endpoints::EndpointContext& endpoint_ctx, + const std::optional& pre_exec_hook = std::nullopt) + { + // TBD: interpreter re-use logic + // TBD: runtime options + const auto interpreter_cache = + context.get_subsystem(); + } + + void execute_request( + const CustomJSEndpoint* endpoint, + ccf::endpoints::EndpointContext& endpoint_ctx) + { + // TBD: historical queries + do_execute_request(endpoint, endpoint_ctx); + } + + void execute_endpoint( + ccf::endpoints::EndpointDefinitionPtr e, + ccf::endpoints::EndpointContext& endpoint_ctx) override + { + // Handle endpoint execution + auto endpoint = dynamic_cast(e.get()); + if (endpoint != nullptr) + { + execute_request(endpoint, endpoint_ctx); + return; + } + + ccf::endpoints::EndpointRegistry::execute_endpoint(e, endpoint_ctx); + } }; } diff --git a/samples/apps/basic/custom_endpoints/endpoint.h b/samples/apps/basic/custom_endpoints/endpoint.h new file mode 100644 index 000000000000..180d53e00c52 --- /dev/null +++ b/samples/apps/basic/custom_endpoints/endpoint.h @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ccf/app_interface.h" + +namespace basicapp +{ + struct CustomJSEndpoint : public ccf::endpoints::Endpoint + {}; +} \ No newline at end of file diff --git a/src/service/tables/endpoints.h b/src/service/tables/endpoints.h index c7e71d2c57f2..1b58677e26ca 100644 --- a/src/service/tables/endpoints.h +++ b/src/service/tables/endpoints.h @@ -9,33 +9,4 @@ namespace ccf { using DynamicEndpoints = ccf::ServiceMap; -} - -namespace kv::serialisers -{ - template <> - struct BlitSerialiser - { - static SerialisedEntry to_serialised( - const ccf::endpoints::EndpointKey& endpoint_key) - { - auto str = - fmt::format("{} {}", endpoint_key.verb.c_str(), endpoint_key.uri_path); - return SerialisedEntry(str.begin(), str.end()); - } - - static ccf::endpoints::EndpointKey from_serialised( - const SerialisedEntry& data) - { - std::string str{data.begin(), data.end()}; - auto i = str.find(' '); - if (i == std::string::npos) - { - throw std::logic_error("invalid encoding of endpoint key"); - } - auto verb = str.substr(0, i); - auto uri_path = str.substr(i + 1); - return {uri_path, verb}; - } - }; } \ No newline at end of file From 776e4a33e7062290919e9298ebc38f270eac4bd2 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 16 May 2024 09:53:44 +0000 Subject: [PATCH 02/25] Add install endpoint --- CMakeLists.txt | 5 ++ samples/apps/basic/basic.cpp | 74 ++++++++++++++++++++++++++++++ tests/programmability.py | 89 ++++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/programmability.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 30ac3b7fea81..0590e51debfa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1418,6 +1418,11 @@ if(BUILD_TESTS) ${CMAKE_SOURCE_DIR}/samples/apps/logging/js ) + add_e2e_test( + NAME programmability + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py + ) + # This test uses large requests (so too slow for SAN) if(NOT SAN) add_e2e_test( diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 59519ab80e0d..0ffa45c44efe 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -14,7 +14,9 @@ #include // Custom Endpoints +#include "ccf/bundle.h" #include "ccf/endpoints/authentication/js.h" +#include "ccf/service/tables/modules.h" #include "custom_endpoints/endpoint.h" #include "js/interpreter_cache_interface.h" @@ -109,6 +111,78 @@ namespace basicapp }; make_endpoint("/records", HTTP_POST, post, {ccf::user_cert_auth_policy}) .install(); + + auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { + const auto& caller_identity = + ctx.template get_caller(); + + // Authorization Check + nlohmann::json user_data = nullptr; + auto result = + get_user_data_v1(ctx.tx, caller_identity.user_id, user_data); + if (result == ccf::ApiResult::InternalError) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + fmt::format( + "Failed to get user data for user {}: {}", + caller_identity.user_id, + ccf::api_result_to_str(result))); + return; + } + const auto is_admin_it = user_data.find("isAdmin"); + + // Not every user gets to define custom endpoints, only users with + // isAdmin + if ( + !user_data.is_object() || is_admin_it == user_data.end() || + !is_admin_it.value().get()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_FORBIDDEN, + ccf::errors::AuthorizationFailed, + "Only admins may access this endpoint."); + return; + } + // End of Authorization Check + + const auto j = nlohmann::json::parse( + caller_identity.content.begin(), caller_identity.content.end()); + const auto wrapper = j.get(); + + auto endpoints = ctx.tx.template rw( + "custom_endpoints.metadata"); + // Similar to set_js_app + for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) + { + for (const auto& [method, metadata] : methods) + { + std::string method_upper = method; + nonstd::to_upper(method_upper); + const auto key = ccf::endpoints::EndpointKey{url, method_upper}; + endpoints->put(key, metadata); + } + } + + auto modules = + ctx.tx.template rw("custom_endpoints.modules"); + for (const auto& [name, module] : wrapper.bundle.modules) + { + modules->put(name, module); + } + // TBD: Bytecode compilation support + + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); + }; + + make_endpoint( + "custom_endpoints", + HTTP_PUT, + put_custom_endpoints, + {ccf::user_cose_sign1_auth_policy}) + .set_auto_schema() + .install(); } // Custom Endpoints diff --git a/tests/programmability.py b/tests/programmability.py new file mode 100644 index 000000000000..f7688ee93497 --- /dev/null +++ b/tests/programmability.py @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. +import infra.network +import infra.e2e_args +import infra.checker +import infra.jwt_issuer +import infra.proc +import http +from infra.runner import ConcurrentRunner + + +TESTJS = """ +export function content(request) { + return { + statusCode: 200, + body: { + error: "Test content", + }, + }; +} +""" + + +def test_custom_endpoints(network, args): + primary, _ = network.find_primary() + + # Make user0 admin, so it can install custom endpoints + user = network.users[0] + network.consortium.set_user_data( + primary, user.service_id, user_data={"isAdmin": True} + ) + + bundle = { + "metadata": { + "endpoints": { + "/content": { + "get": { + "js_module": "test.js", + "js_function": "content", + "forwarding_required": "never", + "redirection_strategy": "none", + "authn_policies": ["no_auth"], + "mode": "readonly", + "openapi": {}, + } + }, + } + }, + "modules": {"test.js": TESTJS}, + } + + with primary.client(None, None, user.local_id) as c: + r = c.put("/app/custom_endpoints", body={"bundle": bundle}) + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code + + with primary.client() as c: + r = c.get("/app/content") + assert r.status_code == http.HTTPStatus.OK.value, r.status_code + + return network + + +def run(args): + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + args.perf_nodes, + pdb=args.pdb, + ) as network: + network.start_and_open(args) + + test_custom_endpoints(network, args) + + +if __name__ == "__main__": + cr = ConcurrentRunner() + + cr.add( + "basic", + run, + package="samples/apps/basic/libbasic", + js_app_bundle=None, + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + initial_user_count=2, + initial_member_count=1, + ) + + cr.run() From 610088101cf423829009f5166a4ea885ffdd6b1c Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 16 May 2024 10:27:04 +0000 Subject: [PATCH 03/25] Split out base class --- samples/apps/basic/basic.cpp | 149 +------------ .../apps/basic/custom_endpoints/registry.h | 197 ++++++++++++++++++ 2 files changed, 200 insertions(+), 146 deletions(-) create mode 100644 samples/apps/basic/custom_endpoints/registry.h diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 0ffa45c44efe..88d34ff0e391 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -14,11 +14,7 @@ #include // Custom Endpoints -#include "ccf/bundle.h" -#include "ccf/endpoints/authentication/js.h" -#include "ccf/service/tables/modules.h" -#include "custom_endpoints/endpoint.h" -#include "js/interpreter_cache_interface.h" +#include "custom_endpoints/registry.h" using namespace nlohmann; @@ -27,11 +23,11 @@ namespace basicapp using RecordsMap = kv::Map>; static constexpr auto PRIVATE_RECORDS = "records"; - class BasicHandlers : public ccf::UserEndpointRegistry + class BasicHandlers : public basicapp::CustomJSEndpointRegistry { public: BasicHandlers(ccfapp::AbstractNodeContext& context) : - ccf::UserEndpointRegistry(context) + basicapp::CustomJSEndpointRegistry(context) { openapi_info.title = "CCF Basic App"; openapi_info.description = @@ -111,145 +107,6 @@ namespace basicapp }; make_endpoint("/records", HTTP_POST, post, {ccf::user_cert_auth_policy}) .install(); - - auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { - const auto& caller_identity = - ctx.template get_caller(); - - // Authorization Check - nlohmann::json user_data = nullptr; - auto result = - get_user_data_v1(ctx.tx, caller_identity.user_id, user_data); - if (result == ccf::ApiResult::InternalError) - { - ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - fmt::format( - "Failed to get user data for user {}: {}", - caller_identity.user_id, - ccf::api_result_to_str(result))); - return; - } - const auto is_admin_it = user_data.find("isAdmin"); - - // Not every user gets to define custom endpoints, only users with - // isAdmin - if ( - !user_data.is_object() || is_admin_it == user_data.end() || - !is_admin_it.value().get()) - { - ctx.rpc_ctx->set_error( - HTTP_STATUS_FORBIDDEN, - ccf::errors::AuthorizationFailed, - "Only admins may access this endpoint."); - return; - } - // End of Authorization Check - - const auto j = nlohmann::json::parse( - caller_identity.content.begin(), caller_identity.content.end()); - const auto wrapper = j.get(); - - auto endpoints = ctx.tx.template rw( - "custom_endpoints.metadata"); - // Similar to set_js_app - for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) - { - for (const auto& [method, metadata] : methods) - { - std::string method_upper = method; - nonstd::to_upper(method_upper); - const auto key = ccf::endpoints::EndpointKey{url, method_upper}; - endpoints->put(key, metadata); - } - } - - auto modules = - ctx.tx.template rw("custom_endpoints.modules"); - for (const auto& [name, module] : wrapper.bundle.modules) - { - modules->put(name, module); - } - // TBD: Bytecode compilation support - - ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); - }; - - make_endpoint( - "custom_endpoints", - HTTP_PUT, - put_custom_endpoints, - {ccf::user_cose_sign1_auth_policy}) - .set_auto_schema() - .install(); - } - - // Custom Endpoints - - ccf::endpoints::EndpointDefinitionPtr find_endpoint( - kv::Tx& tx, ccf::RpcContext& rpc_ctx) override - { - // Look up the endpoint definition - // First in the user-defined endpoints, and then fall-back to built-ins - const auto method = rpc_ctx.get_method(); - const auto verb = rpc_ctx.get_request_verb(); - - auto endpoints = - tx.ro("custom_endpoints.metadata"); - const auto key = ccf::endpoints::EndpointKey{method, verb}; - - // Look for a direct match of the given path - const auto it = endpoints->get(key); - if (it.has_value()) - { - auto endpoint_def = std::make_shared(); - endpoint_def->dispatch = key; - endpoint_def->properties = it.value(); - endpoint_def->full_uri_path = - fmt::format("/{}{}", method_prefix, endpoint_def->dispatch.uri_path); - ccf::instantiate_authn_policies(*endpoint_def); - return endpoint_def; - } - - // TBD: templated endpoints - return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); - } - - using PreExecutionHook = std::function; - - void do_execute_request( - const CustomJSEndpoint* endpoint, - ccf::endpoints::EndpointContext& endpoint_ctx, - const std::optional& pre_exec_hook = std::nullopt) - { - // TBD: interpreter re-use logic - // TBD: runtime options - const auto interpreter_cache = - context.get_subsystem(); - } - - void execute_request( - const CustomJSEndpoint* endpoint, - ccf::endpoints::EndpointContext& endpoint_ctx) - { - // TBD: historical queries - do_execute_request(endpoint, endpoint_ctx); - } - - void execute_endpoint( - ccf::endpoints::EndpointDefinitionPtr e, - ccf::endpoints::EndpointContext& endpoint_ctx) override - { - // Handle endpoint execution - auto endpoint = dynamic_cast(e.get()); - if (endpoint != nullptr) - { - execute_request(endpoint, endpoint_ctx); - return; - } - - ccf::endpoints::EndpointRegistry::execute_endpoint(e, endpoint_ctx); } }; } diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h new file mode 100644 index 000000000000..00c337521ef7 --- /dev/null +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +// CCF +#include "ccf/app_interface.h" +#include "ccf/common_auth_policies.h" +#include "ccf/ds/hash.h" +#include "ccf/http_query.h" +#include "ccf/json_handler.h" +#include "ccf/version.h" + +#include +#define FMT_HEADER_ONLY +#include + +// Custom Endpoints +#include "ccf/bundle.h" +#include "ccf/endpoints/authentication/js.h" +#include "ccf/service/tables/modules.h" +#include "endpoint.h" +#include "js/interpreter_cache_interface.h" + +using namespace nlohmann; + +namespace basicapp +{ + class CustomJSEndpointRegistry : public ccf::UserEndpointRegistry + { + public: + CustomJSEndpointRegistry(ccfapp::AbstractNodeContext& context) : + ccf::UserEndpointRegistry(context) + { + auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { + const auto& caller_identity = + ctx.template get_caller(); + + // Authorization Check + nlohmann::json user_data = nullptr; + auto result = + get_user_data_v1(ctx.tx, caller_identity.user_id, user_data); + if (result == ccf::ApiResult::InternalError) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + fmt::format( + "Failed to get user data for user {}: {}", + caller_identity.user_id, + ccf::api_result_to_str(result))); + return; + } + const auto is_admin_it = user_data.find("isAdmin"); + + // Not every user gets to define custom endpoints, only users with + // isAdmin + if ( + !user_data.is_object() || is_admin_it == user_data.end() || + !is_admin_it.value().get()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_FORBIDDEN, + ccf::errors::AuthorizationFailed, + "Only admins may access this endpoint."); + return; + } + // End of Authorization Check + + const auto j = nlohmann::json::parse( + caller_identity.content.begin(), caller_identity.content.end()); + const auto wrapper = j.get(); + + auto endpoints = ctx.tx.template rw( + "custom_endpoints.metadata"); + // Similar to set_js_app + for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) + { + for (const auto& [method, metadata] : methods) + { + std::string method_upper = method; + nonstd::to_upper(method_upper); + const auto key = ccf::endpoints::EndpointKey{url, method_upper}; + endpoints->put(key, metadata); + } + } + + auto modules = + ctx.tx.template rw("custom_endpoints.modules"); + for (const auto& [name, module] : wrapper.bundle.modules) + { + modules->put(name, module); + } + // TBD: Bytecode compilation support + + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); + }; + + make_endpoint( + "custom_endpoints", + HTTP_PUT, + put_custom_endpoints, + {ccf::user_cose_sign1_auth_policy}) + .set_auto_schema() + .install(); + } + + // Custom Endpoints + + ccf::endpoints::EndpointDefinitionPtr find_endpoint( + kv::Tx& tx, ccf::RpcContext& rpc_ctx) override + { + // Look up the endpoint definition + // First in the user-defined endpoints, and then fall-back to built-ins + const auto method = rpc_ctx.get_method(); + const auto verb = rpc_ctx.get_request_verb(); + + auto endpoints = + tx.ro("custom_endpoints.metadata"); + const auto key = ccf::endpoints::EndpointKey{method, verb}; + + // Look for a direct match of the given path + const auto it = endpoints->get(key); + if (it.has_value()) + { + auto endpoint_def = std::make_shared(); + endpoint_def->dispatch = key; + endpoint_def->properties = it.value(); + endpoint_def->full_uri_path = + fmt::format("/{}{}", method_prefix, endpoint_def->dispatch.uri_path); + ccf::instantiate_authn_policies(*endpoint_def); + return endpoint_def; + } + + // TBD: templated endpoints + return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); + } + + using PreExecutionHook = std::function; + + void do_execute_request( + const CustomJSEndpoint* endpoint, + ccf::endpoints::EndpointContext& endpoint_ctx, + const std::optional& pre_exec_hook = std::nullopt) + { + // TBD: interpreter re-use logic + // TBD: runtime options + const auto interpreter_cache = + context.get_subsystem(); + } + + void execute_request( + const CustomJSEndpoint* endpoint, + ccf::endpoints::EndpointContext& endpoint_ctx) + { + // TBD: historical queries + do_execute_request(endpoint, endpoint_ctx); + } + + void execute_endpoint( + ccf::endpoints::EndpointDefinitionPtr e, + ccf::endpoints::EndpointContext& endpoint_ctx) override + { + // Handle endpoint execution + auto endpoint = dynamic_cast(e.get()); + if (endpoint != nullptr) + { + execute_request(endpoint, endpoint_ctx); + return; + } + + ccf::endpoints::EndpointRegistry::execute_endpoint(e, endpoint_ctx); + } + + void execute_request_locally_committed( + const CustomJSEndpoint* endpoint, + ccf::endpoints::CommandEndpointContext& endpoint_ctx, + const ccf::TxID& tx_id) + { + ccf::endpoints::default_locally_committed_func(endpoint_ctx, tx_id); + } + + void execute_endpoint_locally_committed( + ccf::endpoints::EndpointDefinitionPtr e, + ccf::endpoints::CommandEndpointContext& endpoint_ctx, + const ccf::TxID& tx_id) override + { + auto endpoint = dynamic_cast(e.get()); + if (endpoint != nullptr) + { + execute_request_locally_committed(endpoint, endpoint_ctx, tx_id); + return; + } + + ccf::endpoints::EndpointRegistry::execute_endpoint_locally_committed( + e, endpoint_ctx, tx_id); + } + }; +} From 9dbf253d15db30970617e0bbd74d1ff9c7aebc5a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 16 May 2024 13:53:53 +0000 Subject: [PATCH 04/25] wip --- samples/apps/basic/custom_endpoints/registry.h | 18 ++++++++++++++++++ tests/programmability.py | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 00c337521ef7..be6564779e19 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -145,6 +145,24 @@ namespace basicapp // TBD: runtime options const auto interpreter_cache = context.get_subsystem(); + + // TBD: private headers + // const auto rw_access = + // endpoint->properties.mode == ccf::endpoints::Mode::ReadWrite ? + // js::TxAccess::APP_RW : + // js::TxAccess::APP_RO; + + // std::shared_ptr interpreter = + // interpreter_cache->get_interpreter(rw_access, *endpoint, + // flush_marker); + // if (interpreter == nullptr) + // { + // throw std::logic_error("Cache failed to produce interpreter"); + // } + // js::core::Context& ctx = *interpreter; + + // TBD: Run fetched endpoint + CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); } void execute_request( diff --git a/tests/programmability.py b/tests/programmability.py index f7688ee93497..56d5c20ee822 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -53,6 +53,10 @@ def test_custom_endpoints(network, args): r = c.put("/app/custom_endpoints", body={"bundle": bundle}) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code + with primary.client() as c: + r = c.get("/app/not_content") + assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code + with primary.client() as c: r = c.get("/app/content") assert r.status_code == http.HTTPStatus.OK.value, r.status_code From 80b9fbd397f8d36129cec07366aa744e250c928a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 08:09:00 +0000 Subject: [PATCH 05/25] Move enough headers that we can instantiate an interpreter factory in an app --- {src => include/ccf}/js/core/context.h | 4 +- {src => include/ccf}/js/extensions/README.md | 0 .../ccf}/js/extensions/ccf/consensus.h | 2 +- .../ccf}/js/extensions/ccf/converters.h | 2 +- .../ccf}/js/extensions/ccf/crypto.h | 2 +- .../ccf}/js/extensions/ccf/gov_effects.h | 2 +- .../ccf}/js/extensions/ccf/historical.h | 2 +- {src => include/ccf}/js/extensions/ccf/host.h | 2 +- {src => include/ccf}/js/extensions/ccf/kv.h | 2 +- .../ccf}/js/extensions/ccf/kv_helpers.h | 0 .../ccf}/js/extensions/ccf/network.h | 2 +- {src => include/ccf}/js/extensions/ccf/node.h | 2 +- {src => include/ccf}/js/extensions/ccf/rpc.h | 2 +- {src => include/ccf}/js/extensions/console.h | 4 +- .../ccf}/js/extensions/extension_interface.h | 0 .../ccf}/js/extensions/math/random.h | 2 +- {src => include/ccf}/js/tx_access.h | 0 .../apps/basic/custom_endpoints/registry.h | 63 ++++++++++++++++++- src/apps/js_generic/js_generic_base.cpp | 20 +++--- src/apps/js_generic/request_extension.cpp | 2 +- src/apps/js_generic/request_extension.h | 2 +- src/enclave/enclave.h | 2 +- src/js/common_context.h | 12 ++-- src/js/core/context.cpp | 6 +- src/js/core/runtime.h | 2 +- src/js/core/wrapped_property_enum.h | 2 +- src/js/extensions/ccf/consensus.cpp | 4 +- src/js/extensions/ccf/converters.cpp | 4 +- src/js/extensions/ccf/crypto.cpp | 4 +- src/js/extensions/ccf/gov_effects.cpp | 4 +- src/js/extensions/ccf/historical.cpp | 6 +- src/js/extensions/ccf/host.cpp | 4 +- src/js/extensions/ccf/kv.cpp | 6 +- src/js/extensions/ccf/network.cpp | 4 +- src/js/extensions/ccf/node.cpp | 4 +- src/js/extensions/ccf/rpc.cpp | 4 +- src/js/extensions/console.cpp | 4 +- src/js/extensions/math/random.cpp | 4 +- src/js/global_class_ids.cpp | 2 +- src/js/interpreter_cache_interface.h | 2 +- src/js/map_access_permissions.h | 2 +- src/js/openenclave.cpp | 2 +- src/js/snp_attestation.cpp | 2 +- src/node/gov/handlers/proposals.h | 6 +- src/node/node_state.h | 2 +- src/node/rpc/member_frontend.h | 4 +- src/node/rpc/node_frontend.h | 2 +- 47 files changed, 138 insertions(+), 79 deletions(-) rename {src => include/ccf}/js/core/context.h (98%) rename {src => include/ccf}/js/extensions/README.md (100%) rename {src => include/ccf}/js/extensions/ccf/consensus.h (92%) rename {src => include/ccf}/js/extensions/ccf/converters.h (91%) rename {src => include/ccf}/js/extensions/ccf/crypto.h (95%) rename {src => include/ccf}/js/extensions/ccf/gov_effects.h (92%) rename {src => include/ccf}/js/extensions/ccf/historical.h (94%) rename {src => include/ccf}/js/extensions/ccf/host.h (91%) rename {src => include/ccf}/js/extensions/ccf/kv.h (90%) rename {src => include/ccf}/js/extensions/ccf/kv_helpers.h (100%) rename {src => include/ccf}/js/extensions/ccf/network.h (92%) rename {src => include/ccf}/js/extensions/ccf/node.h (93%) rename {src => include/ccf}/js/extensions/ccf/rpc.h (90%) rename {src => include/ccf}/js/extensions/console.h (89%) rename {src => include/ccf}/js/extensions/extension_interface.h (100%) rename {src => include/ccf}/js/extensions/math/random.h (87%) rename {src => include/ccf}/js/tx_access.h (100%) diff --git a/src/js/core/context.h b/include/ccf/js/core/context.h similarity index 98% rename from src/js/core/context.h rename to include/ccf/js/core/context.h index 72191374fb9e..026a3a39f738 100644 --- a/src/js/core/context.h +++ b/include/ccf/js/core/context.h @@ -5,8 +5,8 @@ #include "ccf/pal/locking.h" #include "js/core/runtime.h" #include "js/core/wrapped_value.h" -#include "js/extensions/extension_interface.h" -#include "js/tx_access.h" +#include "ccf/js/extensions/extension_interface.h" +#include "ccf/js/tx_access.h" #include #include diff --git a/src/js/extensions/README.md b/include/ccf/js/extensions/README.md similarity index 100% rename from src/js/extensions/README.md rename to include/ccf/js/extensions/README.md diff --git a/src/js/extensions/ccf/consensus.h b/include/ccf/js/extensions/ccf/consensus.h similarity index 92% rename from src/js/extensions/ccf/consensus.h rename to include/ccf/js/extensions/ccf/consensus.h index ba02ee30fe55..8749f79cdb8d 100644 --- a/src/js/extensions/ccf/consensus.h +++ b/include/ccf/js/extensions/ccf/consensus.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/base_endpoint_registry.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/ccf/converters.h b/include/ccf/js/extensions/ccf/converters.h similarity index 91% rename from src/js/extensions/ccf/converters.h rename to include/ccf/js/extensions/ccf/converters.h index 059710190363..8af18051fe52 100644 --- a/src/js/extensions/ccf/converters.h +++ b/include/ccf/js/extensions/ccf/converters.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/ccf/crypto.h b/include/ccf/js/extensions/ccf/crypto.h similarity index 95% rename from src/js/extensions/ccf/crypto.h rename to include/ccf/js/extensions/ccf/crypto.h index 4508f877d0fc..88d54c4c32b8 100644 --- a/src/js/extensions/ccf/crypto.h +++ b/include/ccf/js/extensions/ccf/crypto.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/ccf/gov_effects.h b/include/ccf/js/extensions/ccf/gov_effects.h similarity index 92% rename from src/js/extensions/ccf/gov_effects.h rename to include/ccf/js/extensions/ccf/gov_effects.h index 610d33f377a4..d22c13608a9b 100644 --- a/src/js/extensions/ccf/gov_effects.h +++ b/include/ccf/js/extensions/ccf/gov_effects.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/tx.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/ccf/historical.h b/include/ccf/js/extensions/ccf/historical.h similarity index 94% rename from src/js/extensions/ccf/historical.h rename to include/ccf/js/extensions/ccf/historical.h index ab1aabf6a9b0..12b716b0687a 100644 --- a/src/js/extensions/ccf/historical.h +++ b/include/ccf/js/extensions/ccf/historical.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/historical_queries_interface.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" #include diff --git a/src/js/extensions/ccf/host.h b/include/ccf/js/extensions/ccf/host.h similarity index 91% rename from src/js/extensions/ccf/host.h rename to include/ccf/js/extensions/ccf/host.h index 9358c03fcbb1..59a9e410c4e3 100644 --- a/src/js/extensions/ccf/host.h +++ b/include/ccf/js/extensions/ccf/host.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/node/host_processes_interface.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/ccf/kv.h b/include/ccf/js/extensions/ccf/kv.h similarity index 90% rename from src/js/extensions/ccf/kv.h rename to include/ccf/js/extensions/ccf/kv.h index 3e42a61c24f0..f6127b3a7cc8 100644 --- a/src/js/extensions/ccf/kv.h +++ b/include/ccf/js/extensions/ccf/kv.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/tx.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" #include diff --git a/src/js/extensions/ccf/kv_helpers.h b/include/ccf/js/extensions/ccf/kv_helpers.h similarity index 100% rename from src/js/extensions/ccf/kv_helpers.h rename to include/ccf/js/extensions/ccf/kv_helpers.h diff --git a/src/js/extensions/ccf/network.h b/include/ccf/js/extensions/ccf/network.h similarity index 92% rename from src/js/extensions/ccf/network.h rename to include/ccf/js/extensions/ccf/network.h index 19dd0e6142c0..e1cb42fbdc4b 100644 --- a/src/js/extensions/ccf/network.h +++ b/include/ccf/js/extensions/ccf/network.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" #include "node/network_state.h" namespace ccf::js::extensions diff --git a/src/js/extensions/ccf/node.h b/include/ccf/js/extensions/ccf/node.h similarity index 93% rename from src/js/extensions/ccf/node.h rename to include/ccf/js/extensions/ccf/node.h index 790f88f2959c..87272f21dcda 100644 --- a/src/js/extensions/ccf/node.h +++ b/include/ccf/js/extensions/ccf/node.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" #include "node/rpc/gov_effects_interface.h" namespace ccf::js::extensions diff --git a/src/js/extensions/ccf/rpc.h b/include/ccf/js/extensions/ccf/rpc.h similarity index 90% rename from src/js/extensions/ccf/rpc.h rename to include/ccf/js/extensions/ccf/rpc.h index c074938a8602..b37a5caea791 100644 --- a/src/js/extensions/ccf/rpc.h +++ b/include/ccf/js/extensions/ccf/rpc.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/rpc_context.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/extensions/console.h b/include/ccf/js/extensions/console.h similarity index 89% rename from src/js/extensions/console.h rename to include/ccf/js/extensions/console.h index 70beefdadb7a..ca5051f45a95 100644 --- a/src/js/extensions/console.h +++ b/include/ccf/js/extensions/console.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" -#include "js/tx_access.h" +#include "ccf/js/extensions/extension_interface.h" +#include "ccf/js/tx_access.h" #include diff --git a/src/js/extensions/extension_interface.h b/include/ccf/js/extensions/extension_interface.h similarity index 100% rename from src/js/extensions/extension_interface.h rename to include/ccf/js/extensions/extension_interface.h diff --git a/src/js/extensions/math/random.h b/include/ccf/js/extensions/math/random.h similarity index 87% rename from src/js/extensions/math/random.h rename to include/ccf/js/extensions/math/random.h index 073b4678fd90..159816a0ba2c 100644 --- a/src/js/extensions/math/random.h +++ b/include/ccf/js/extensions/math/random.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccf::js::extensions { diff --git a/src/js/tx_access.h b/include/ccf/js/tx_access.h similarity index 100% rename from src/js/tx_access.h rename to include/ccf/js/tx_access.h diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index be6564779e19..2c0d5c37247d 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -20,16 +20,77 @@ #include "endpoint.h" #include "js/interpreter_cache_interface.h" +#include "ccf/js/core/context.h" +#include "ccf/js/extensions/ccf/consensus.h" +#include "ccf/js/extensions/ccf/converters.h" +#include "ccf/js/extensions/ccf/crypto.h" +#include "ccf/js/extensions/ccf/historical.h" +#include "ccf/js/extensions/ccf/host.h" +#include "ccf/js/extensions/ccf/kv.h" +#include "ccf/js/extensions/ccf/rpc.h" +#include "ccf/js/extensions/console.h" +#include "ccf/js/extensions/math/random.h" + using namespace nlohmann; namespace basicapp { class CustomJSEndpointRegistry : public ccf::UserEndpointRegistry { + private: + std::shared_ptr interpreter_cache = nullptr; + public: CustomJSEndpointRegistry(ccfapp::AbstractNodeContext& context) : ccf::UserEndpointRegistry(context) { + interpreter_cache = + context.get_subsystem(); + if (interpreter_cache == nullptr) + { + throw std::logic_error( + "Unexpected: Could not access AbstractInterpreterCache subsytem"); + } + + // Install dependency-less (ie reusable) extensions on interpreters _at + // creation_, rather than on every run + ccf::js::extensions::Extensions extensions; + // override Math.random + extensions.emplace_back( + std::make_shared()); + // add console.[debug|log|...] + extensions.emplace_back( + std::make_shared()); + // add ccf.[strToBuf|bufToStr|...] + extensions.emplace_back( + std::make_shared()); + // add ccf.crypto.* + extensions.emplace_back( + std::make_shared()); + // add ccf.consensus.* + extensions.emplace_back( + std::make_shared(this)); + // add ccf.host.* + extensions.emplace_back( + std::make_shared( + context.get_subsystem().get())); + // add ccf.historical.* + extensions.emplace_back( + std::make_shared( + &context.get_historical_state())); + + interpreter_cache->set_interpreter_factory( + [extensions](ccf::js::TxAccess access) { + auto interpreter = std::make_shared(access); + + for (auto extension : extensions) + { + interpreter->add_extension(extension); + } + + return interpreter; + }); + auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { const auto& caller_identity = ctx.template get_caller(); @@ -143,8 +204,6 @@ namespace basicapp { // TBD: interpreter re-use logic // TBD: runtime options - const auto interpreter_cache = - context.get_subsystem(); // TBD: private headers // const auto rw_access = diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 682c65390f6a..5b33419b0b31 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -11,17 +11,17 @@ #include "ccf/service/tables/jsengine.h" #include "ccf/version.h" #include "enclave/enclave_time.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/core/wrapped_property_enum.h" -#include "js/extensions/ccf/consensus.h" -#include "js/extensions/ccf/converters.h" -#include "js/extensions/ccf/crypto.h" -#include "js/extensions/ccf/historical.h" -#include "js/extensions/ccf/host.h" -#include "js/extensions/ccf/kv.h" -#include "js/extensions/ccf/rpc.h" -#include "js/extensions/console.h" -#include "js/extensions/math/random.h" +#include "ccf/js/extensions/ccf/consensus.h" +#include "ccf/js/extensions/ccf/converters.h" +#include "ccf/js/extensions/ccf/crypto.h" +#include "ccf/js/extensions/ccf/historical.h" +#include "ccf/js/extensions/ccf/host.h" +#include "ccf/js/extensions/ccf/kv.h" +#include "ccf/js/extensions/ccf/rpc.h" +#include "ccf/js/extensions/console.h" +#include "ccf/js/extensions/math/random.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" #include "js/modules.h" diff --git a/src/apps/js_generic/request_extension.cpp b/src/apps/js_generic/request_extension.cpp index 951eae419488..5f109e8f18af 100644 --- a/src/apps/js_generic/request_extension.cpp +++ b/src/apps/js_generic/request_extension.cpp @@ -9,7 +9,7 @@ #include "ccf/endpoints/authentication/cose_auth.h" #include "ccf/endpoints/authentication/empty_auth.h" #include "ccf/endpoints/authentication/jwt_auth.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include diff --git a/src/apps/js_generic/request_extension.h b/src/apps/js_generic/request_extension.h index 5cf1f90caef4..be13f55f17bd 100644 --- a/src/apps/js_generic/request_extension.h +++ b/src/apps/js_generic/request_extension.h @@ -6,7 +6,7 @@ #include "ccf/endpoint_context.h" #include "ccf/rpc_context.h" #include "js/core/wrapped_value.h" -#include "js/extensions/extension_interface.h" +#include "ccf/js/extensions/extension_interface.h" namespace ccfapp { diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index 98ce13ed4ea9..58626efb2fe6 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -13,7 +13,7 @@ #include "indexing/enclave_lfs_access.h" #include "indexing/historical_transaction_fetcher.h" #include "interface.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/ffi_plugins.h" #include "js/interpreter_cache.h" #include "node/acme_challenge_frontend.h" diff --git a/src/js/common_context.h b/src/js/common_context.h index 141e0bb74bc5..e724e27cb994 100644 --- a/src/js/common_context.h +++ b/src/js/common_context.h @@ -2,12 +2,12 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/core/context.h" -#include "js/extensions/ccf/converters.h" -#include "js/extensions/ccf/crypto.h" -#include "js/extensions/ccf/kv.h" -#include "js/extensions/console.h" -#include "js/extensions/math/random.h" +#include "ccf/js/core/context.h" +#include "ccf/js/extensions/ccf/converters.h" +#include "ccf/js/extensions/ccf/crypto.h" +#include "ccf/js/extensions/ccf/kv.h" +#include "ccf/js/extensions/console.h" +#include "ccf/js/extensions/math/random.h" namespace ccf::js { diff --git a/src/js/core/context.cpp b/src/js/core/context.cpp index f038ba2f34cb..452246b0b3b8 100644 --- a/src/js/core/context.cpp +++ b/src/js/core/context.cpp @@ -1,17 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "ccf/ds/hex.h" #include "ccf/pal/locking.h" #include "enclave/enclave_time.h" #include "js/core/runtime.h" #include "js/core/wrapped_value.h" -#include "js/extensions/console.h" +#include "ccf/js/extensions/console.h" #include "js/ffi_plugins.h" #include "js/global_class_ids.h" -#include "js/tx_access.h" +#include "ccf/js/tx_access.h" #include #include diff --git a/src/js/core/runtime.h b/src/js/core/runtime.h index 1906792d889d..a584194a609e 100644 --- a/src/js/core/runtime.h +++ b/src/js/core/runtime.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "kv/kv_types.h" +#include "ccf/tx.h" #include #include diff --git a/src/js/core/wrapped_property_enum.h b/src/js/core/wrapped_property_enum.h index bafa177a1308..3199179694e7 100644 --- a/src/js/core/wrapped_property_enum.h +++ b/src/js/core/wrapped_property_enum.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/core/wrapped_value.h" #include diff --git a/src/js/extensions/ccf/consensus.cpp b/src/js/extensions/ccf/consensus.cpp index 98719c02fd25..5921bbb15a42 100644 --- a/src/js/extensions/ccf/consensus.cpp +++ b/src/js/extensions/ccf/consensus.cpp @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/consensus.h" +#include "ccf/js/extensions/ccf/consensus.h" #include "ccf/base_endpoint_registry.h" #include "js/checks.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include diff --git a/src/js/extensions/ccf/converters.cpp b/src/js/extensions/ccf/converters.cpp index 79c2f891287d..355bba6137fd 100644 --- a/src/js/extensions/ccf/converters.cpp +++ b/src/js/extensions/ccf/converters.cpp @@ -4,11 +4,11 @@ // NB: Despite the naming scheme used elsewhere, this populates functions // directly on the ccf object. -#include "js/extensions/ccf/converters.h" +#include "ccf/js/extensions/ccf/converters.h" #include "ccf/version.h" #include "js/checks.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/modules.h" #include "node/rpc/jwt_management.h" diff --git a/src/js/extensions/ccf/crypto.cpp b/src/js/extensions/ccf/crypto.cpp index 98a91be7336e..6cdfa765a9d8 100644 --- a/src/js/extensions/ccf/crypto.cpp +++ b/src/js/extensions/ccf/crypto.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/crypto.h" +#include "ccf/js/extensions/ccf/crypto.h" #include "ccf/crypto/ecdsa.h" #include "ccf/crypto/eddsa_key_pair.h" @@ -12,7 +12,7 @@ #include "ccf/crypto/sha256.h" #include "ccf/crypto/verifier.h" #include "js/checks.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "tls/ca.h" namespace ccf::js::extensions diff --git a/src/js/extensions/ccf/gov_effects.cpp b/src/js/extensions/ccf/gov_effects.cpp index 9c1ca787a8d5..01b5da2c7065 100644 --- a/src/js/extensions/ccf/gov_effects.cpp +++ b/src/js/extensions/ccf/gov_effects.cpp @@ -4,10 +4,10 @@ // NB: Despite the naming scheme used elsewhere, this populates functions // directly on the ccf object. -#include "js/extensions/ccf/gov_effects.h" +#include "ccf/js/extensions/ccf/gov_effects.h" #include "ccf/version.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include "js/modules.h" #include "node/rpc/jwt_management.h" diff --git a/src/js/extensions/ccf/historical.cpp b/src/js/extensions/ccf/historical.cpp index c57960491bb5..bfb7da36edf1 100644 --- a/src/js/extensions/ccf/historical.cpp +++ b/src/js/extensions/ccf/historical.cpp @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/historical.h" +#include "ccf/js/extensions/ccf/historical.h" #include "ccf/ds/hex.h" #include "ccf/historical_queries_interface.h" #include "js/checks.h" -#include "js/core/context.h" -#include "js/extensions/ccf/kv_helpers.h" +#include "ccf/js/core/context.h" +#include "ccf/js/extensions/ccf/kv_helpers.h" #include "js/global_class_ids.h" #include "kv/untyped_map.h" diff --git a/src/js/extensions/ccf/host.cpp b/src/js/extensions/ccf/host.cpp index 51717967faf5..4b2b9b40da4b 100644 --- a/src/js/extensions/ccf/host.cpp +++ b/src/js/extensions/ccf/host.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/host.h" +#include "ccf/js/extensions/ccf/host.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include diff --git a/src/js/extensions/ccf/kv.cpp b/src/js/extensions/ccf/kv.cpp index 3b49748f507b..9aa396300e2b 100644 --- a/src/js/extensions/ccf/kv.cpp +++ b/src/js/extensions/ccf/kv.cpp @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/kv.h" +#include "ccf/js/extensions/ccf/kv.h" #include "js/checks.h" -#include "js/core/context.h" -#include "js/extensions/ccf/kv_helpers.h" +#include "ccf/js/core/context.h" +#include "ccf/js/extensions/ccf/kv_helpers.h" #include "js/global_class_ids.h" #include "js/map_access_permissions.h" diff --git a/src/js/extensions/ccf/network.cpp b/src/js/extensions/ccf/network.cpp index 1053f1691ee0..cf526845ca0b 100644 --- a/src/js/extensions/ccf/network.cpp +++ b/src/js/extensions/ccf/network.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/network.h" +#include "ccf/js/extensions/ccf/network.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include "node/network_state.h" diff --git a/src/js/extensions/ccf/node.cpp b/src/js/extensions/ccf/node.cpp index 633a5044dcce..0b4c144832f2 100644 --- a/src/js/extensions/ccf/node.cpp +++ b/src/js/extensions/ccf/node.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/node.h" +#include "ccf/js/extensions/ccf/node.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include "node/rpc/gov_logging.h" diff --git a/src/js/extensions/ccf/rpc.cpp b/src/js/extensions/ccf/rpc.cpp index 9333f5c058e9..76feb1b15d8e 100644 --- a/src/js/extensions/ccf/rpc.cpp +++ b/src/js/extensions/ccf/rpc.cpp @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/ccf/rpc.h" +#include "ccf/js/extensions/ccf/rpc.h" #include "ccf/rpc_context.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include diff --git a/src/js/extensions/console.cpp b/src/js/extensions/console.cpp index 0089ba05a223..30bb6f6d5794 100644 --- a/src/js/extensions/console.cpp +++ b/src/js/extensions/console.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/console.h" +#include "ccf/js/extensions/console.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "node/rpc/gov_logging.h" #include diff --git a/src/js/extensions/math/random.cpp b/src/js/extensions/math/random.cpp index dfa1528ad75b..b4714c8b339a 100644 --- a/src/js/extensions/math/random.cpp +++ b/src/js/extensions/math/random.cpp @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/extensions/math/random.h" +#include "ccf/js/extensions/math/random.h" #include "ccf/crypto/entropy.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include diff --git a/src/js/global_class_ids.cpp b/src/js/global_class_ids.cpp index 58a69dde2b70..792d1b04ad51 100644 --- a/src/js/global_class_ids.cpp +++ b/src/js/global_class_ids.cpp @@ -3,7 +3,7 @@ #include "js/global_class_ids.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" namespace ccf::js { diff --git a/src/js/interpreter_cache_interface.h b/src/js/interpreter_cache_interface.h index a4b23803e13e..ae20e6168eb8 100644 --- a/src/js/interpreter_cache_interface.h +++ b/src/js/interpreter_cache_interface.h @@ -4,7 +4,7 @@ #include "ccf/endpoint.h" #include "ccf/node_subsystem_interface.h" -#include "js/tx_access.h" +#include "ccf/js/tx_access.h" namespace ccf::js { diff --git a/src/js/map_access_permissions.h b/src/js/map_access_permissions.h index b9c6a45f046c..689312370ea2 100644 --- a/src/js/map_access_permissions.h +++ b/src/js/map_access_permissions.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/tx_access.h" +#include "ccf/js/tx_access.h" #include "kv/kv_types.h" namespace ccf::js diff --git a/src/js/openenclave.cpp b/src/js/openenclave.cpp index 24fe2831ae04..7643c8d534bf 100644 --- a/src/js/openenclave.cpp +++ b/src/js/openenclave.cpp @@ -6,7 +6,7 @@ #include "ccf/js_plugin.h" #include "ccf/version.h" #include "js/checks.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include #include diff --git a/src/js/snp_attestation.cpp b/src/js/snp_attestation.cpp index 735db5a6319c..250ef6d43acd 100644 --- a/src/js/snp_attestation.cpp +++ b/src/js/snp_attestation.cpp @@ -6,7 +6,7 @@ #include "ccf/pal/attestation.h" #include "ccf/version.h" #include "js/checks.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "node/uvm_endorsements.h" #include diff --git a/src/node/gov/handlers/proposals.h b/src/node/gov/handlers/proposals.h index 2a3e3f36cc54..32d87f5f9e43 100644 --- a/src/node/gov/handlers/proposals.h +++ b/src/node/gov/handlers/proposals.h @@ -4,9 +4,9 @@ #include "ccf/base_endpoint_registry.h" #include "js/common_context.h" -#include "js/extensions/ccf/gov_effects.h" -#include "js/extensions/ccf/network.h" -#include "js/extensions/ccf/node.h" +#include "ccf/js/extensions/ccf/gov_effects.h" +#include "ccf/js/extensions/ccf/network.h" +#include "ccf/js/extensions/ccf/node.h" #include "node/gov/api_version.h" #include "node/gov/handlers/helpers.h" diff --git a/src/node/node_state.h b/src/node/node_state.h index f1d4f0a3787d..9cb3baafa4ac 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -25,7 +25,7 @@ #include "history.h" #include "http/http_parser.h" #include "indexing/indexer.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include "network_state.h" #include "node/hooks.h" diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 287c0bb74d6d..4c618c9b8285 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -16,8 +16,8 @@ #include "ccf/service/tables/nodes.h" #include "frontend.h" #include "js/common_context.h" -#include "js/extensions/ccf/network.h" -#include "js/extensions/ccf/node.h" +#include "ccf/js/extensions/ccf/network.h" +#include "ccf/js/extensions/ccf/node.h" #include "node/gov/gov_endpoint_registry.h" #include "node/rpc/call_types.h" #include "node/rpc/gov_effects_interface.h" diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index aeefd76cc53b..5c767051f157 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -16,7 +16,7 @@ #include "ds/std_formatters.h" #include "enclave/reconfiguration_type.h" #include "frontend.h" -#include "js/core/context.h" +#include "ccf/js/core/context.h" #include "node/network_state.h" #include "node/rpc/jwt_management.h" #include "node/rpc/no_create_tx_claims_digest.cpp" From 6d6fd524c406f1a618b3fc5b7e3427e0e4849c9d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 10:38:42 +0000 Subject: [PATCH 06/25] More exports --- {src => include/ccf}/js/core/constants.h | 0 include/ccf/js/core/context.h | 4 ++-- {src => include/ccf}/js/core/runtime.h | 0 {src => include/ccf}/js/core/wrapped_value.h | 2 +- src/apps/js_generic/request_extension.h | 2 +- src/js/core/context.cpp | 4 ++-- src/js/core/runtime.cpp | 2 +- src/js/core/wrapped_property_enum.h | 2 +- src/js/core/wrapped_value.cpp | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) rename {src => include/ccf}/js/core/constants.h (100%) rename {src => include/ccf}/js/core/runtime.h (100%) rename {src => include/ccf}/js/core/wrapped_value.h (97%) diff --git a/src/js/core/constants.h b/include/ccf/js/core/constants.h similarity index 100% rename from src/js/core/constants.h rename to include/ccf/js/core/constants.h diff --git a/include/ccf/js/core/context.h b/include/ccf/js/core/context.h index 026a3a39f738..75e76f787f02 100644 --- a/include/ccf/js/core/context.h +++ b/include/ccf/js/core/context.h @@ -3,8 +3,8 @@ #pragma once #include "ccf/pal/locking.h" -#include "js/core/runtime.h" -#include "js/core/wrapped_value.h" +#include "ccf/js/core/runtime.h" +#include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/extension_interface.h" #include "ccf/js/tx_access.h" diff --git a/src/js/core/runtime.h b/include/ccf/js/core/runtime.h similarity index 100% rename from src/js/core/runtime.h rename to include/ccf/js/core/runtime.h diff --git a/src/js/core/wrapped_value.h b/include/ccf/js/core/wrapped_value.h similarity index 97% rename from src/js/core/wrapped_value.h rename to include/ccf/js/core/wrapped_value.h index e563edaa6957..7738d7568743 100644 --- a/src/js/core/wrapped_value.h +++ b/include/ccf/js/core/wrapped_value.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "js/core/constants.h" +#include "ccf/js/core/constants.h" #include #include diff --git a/src/apps/js_generic/request_extension.h b/src/apps/js_generic/request_extension.h index be13f55f17bd..0d7f3a5ba4b0 100644 --- a/src/apps/js_generic/request_extension.h +++ b/src/apps/js_generic/request_extension.h @@ -5,7 +5,7 @@ #include "ccf/base_endpoint_registry.h" #include "ccf/endpoint_context.h" #include "ccf/rpc_context.h" -#include "js/core/wrapped_value.h" +#include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/extension_interface.h" namespace ccfapp diff --git a/src/js/core/context.cpp b/src/js/core/context.cpp index 452246b0b3b8..ecf314ae1420 100644 --- a/src/js/core/context.cpp +++ b/src/js/core/context.cpp @@ -6,8 +6,8 @@ #include "ccf/ds/hex.h" #include "ccf/pal/locking.h" #include "enclave/enclave_time.h" -#include "js/core/runtime.h" -#include "js/core/wrapped_value.h" +#include "ccf/js/core/runtime.h" +#include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/console.h" #include "js/ffi_plugins.h" #include "js/global_class_ids.h" diff --git a/src/js/core/runtime.cpp b/src/js/core/runtime.cpp index cb3c69ec5980..085ff2ba4e0c 100644 --- a/src/js/core/runtime.cpp +++ b/src/js/core/runtime.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/core/runtime.h" +#include "ccf/js/core/runtime.h" #include "ccf/service/tables/jsengine.h" #include "ccf/tx.h" diff --git a/src/js/core/wrapped_property_enum.h b/src/js/core/wrapped_property_enum.h index 3199179694e7..9b0aa0cce7ca 100644 --- a/src/js/core/wrapped_property_enum.h +++ b/src/js/core/wrapped_property_enum.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/js/core/context.h" -#include "js/core/wrapped_value.h" +#include "ccf/js/core/wrapped_value.h" #include diff --git a/src/js/core/wrapped_value.cpp b/src/js/core/wrapped_value.cpp index 8e857ad76ea7..7974c01b2fd1 100644 --- a/src/js/core/wrapped_value.cpp +++ b/src/js/core/wrapped_value.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "js/core/wrapped_value.h" +#include "ccf/js/core/wrapped_value.h" -#include "js/core/constants.h" +#include "ccf/js/core/constants.h" namespace ccf::js::core { From e12722f806c65ab4204a1dc4162fee6a6d11d407 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 15:29:05 +0000 Subject: [PATCH 07/25] Some headers not needed in app space --- src/js/extensions/ccf/network.cpp | 2 +- {include/ccf => src}/js/extensions/ccf/network.h | 0 src/js/extensions/ccf/node.cpp | 2 +- {include/ccf => src}/js/extensions/ccf/node.h | 0 src/node/gov/handlers/proposals.h | 4 ++-- src/node/rpc/member_frontend.h | 4 ++-- 6 files changed, 6 insertions(+), 6 deletions(-) rename {include/ccf => src}/js/extensions/ccf/network.h (100%) rename {include/ccf => src}/js/extensions/ccf/node.h (100%) diff --git a/src/js/extensions/ccf/network.cpp b/src/js/extensions/ccf/network.cpp index cf526845ca0b..c7ed1d46d4cc 100644 --- a/src/js/extensions/ccf/network.cpp +++ b/src/js/extensions/ccf/network.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "ccf/js/extensions/ccf/network.h" +#include "js/extensions/ccf/network.h" #include "ccf/js/core/context.h" #include "js/global_class_ids.h" diff --git a/include/ccf/js/extensions/ccf/network.h b/src/js/extensions/ccf/network.h similarity index 100% rename from include/ccf/js/extensions/ccf/network.h rename to src/js/extensions/ccf/network.h diff --git a/src/js/extensions/ccf/node.cpp b/src/js/extensions/ccf/node.cpp index 0b4c144832f2..638a07cd91c4 100644 --- a/src/js/extensions/ccf/node.cpp +++ b/src/js/extensions/ccf/node.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "ccf/js/extensions/ccf/node.h" +#include "js/extensions/ccf/node.h" #include "ccf/js/core/context.h" #include "js/global_class_ids.h" diff --git a/include/ccf/js/extensions/ccf/node.h b/src/js/extensions/ccf/node.h similarity index 100% rename from include/ccf/js/extensions/ccf/node.h rename to src/js/extensions/ccf/node.h diff --git a/src/node/gov/handlers/proposals.h b/src/node/gov/handlers/proposals.h index 32d87f5f9e43..4786b1261c0e 100644 --- a/src/node/gov/handlers/proposals.h +++ b/src/node/gov/handlers/proposals.h @@ -5,8 +5,8 @@ #include "ccf/base_endpoint_registry.h" #include "js/common_context.h" #include "ccf/js/extensions/ccf/gov_effects.h" -#include "ccf/js/extensions/ccf/network.h" -#include "ccf/js/extensions/ccf/node.h" +#include "js/extensions/ccf/network.h" +#include "js/extensions/ccf/node.h" #include "node/gov/api_version.h" #include "node/gov/handlers/helpers.h" diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 4c618c9b8285..287c0bb74d6d 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -16,8 +16,8 @@ #include "ccf/service/tables/nodes.h" #include "frontend.h" #include "js/common_context.h" -#include "ccf/js/extensions/ccf/network.h" -#include "ccf/js/extensions/ccf/node.h" +#include "js/extensions/ccf/network.h" +#include "js/extensions/ccf/node.h" #include "node/gov/gov_endpoint_registry.h" #include "node/rpc/call_types.h" #include "node/rpc/gov_effects_interface.h" From 57665c64028f76f332887c72ee3acd81e7becffb Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 15:40:17 +0000 Subject: [PATCH 08/25] One last file plus format --- include/ccf/js/core/context.h | 2 +- include/ccf/js/extensions/ccf/gov_effects.h | 2 +- include/ccf/js/extensions/ccf/host.h | 2 +- include/ccf/js/extensions/ccf/kv.h | 2 +- include/ccf/js/extensions/ccf/rpc.h | 2 +- samples/apps/basic/custom_endpoints/registry.h | 10 +++++----- src/apps/js_generic/js_generic_base.cpp | 10 +++++----- src/apps/js_generic/request_extension.h | 2 +- src/enclave/enclave.h | 2 +- src/js/core/context.cpp | 6 +++--- src/js/extensions/ccf/consensus.cpp | 2 +- src/js/extensions/ccf/converters.cpp | 2 +- src/js/extensions/ccf/crypto.cpp | 2 +- src/js/extensions/ccf/gov_effects.cpp | 2 +- src/js/extensions/ccf/historical.cpp | 4 ++-- src/js/extensions/ccf/kv.cpp | 4 ++-- {include/ccf => src}/js/extensions/ccf/kv_helpers.h | 0 src/js/extensions/ccf/rpc.cpp | 3 ++- src/js/interpreter_cache_interface.h | 2 +- src/js/openenclave.cpp | 2 +- src/js/snp_attestation.cpp | 2 +- src/node/gov/handlers/proposals.h | 2 +- src/node/node_state.h | 2 +- src/node/rpc/node_frontend.h | 2 +- 24 files changed, 36 insertions(+), 35 deletions(-) rename {include/ccf => src}/js/extensions/ccf/kv_helpers.h (100%) diff --git a/include/ccf/js/core/context.h b/include/ccf/js/core/context.h index 75e76f787f02..28b1fbc556a5 100644 --- a/include/ccf/js/core/context.h +++ b/include/ccf/js/core/context.h @@ -2,11 +2,11 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/pal/locking.h" #include "ccf/js/core/runtime.h" #include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/extension_interface.h" #include "ccf/js/tx_access.h" +#include "ccf/pal/locking.h" #include #include diff --git a/include/ccf/js/extensions/ccf/gov_effects.h b/include/ccf/js/extensions/ccf/gov_effects.h index d22c13608a9b..63fe935e1663 100644 --- a/include/ccf/js/extensions/ccf/gov_effects.h +++ b/include/ccf/js/extensions/ccf/gov_effects.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/tx.h" #include "ccf/js/extensions/extension_interface.h" +#include "ccf/tx.h" namespace ccf::js::extensions { diff --git a/include/ccf/js/extensions/ccf/host.h b/include/ccf/js/extensions/ccf/host.h index 59a9e410c4e3..5211687f8d12 100644 --- a/include/ccf/js/extensions/ccf/host.h +++ b/include/ccf/js/extensions/ccf/host.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/node/host_processes_interface.h" #include "ccf/js/extensions/extension_interface.h" +#include "ccf/node/host_processes_interface.h" namespace ccf::js::extensions { diff --git a/include/ccf/js/extensions/ccf/kv.h b/include/ccf/js/extensions/ccf/kv.h index f6127b3a7cc8..ef912337afb3 100644 --- a/include/ccf/js/extensions/ccf/kv.h +++ b/include/ccf/js/extensions/ccf/kv.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/tx.h" #include "ccf/js/extensions/extension_interface.h" +#include "ccf/tx.h" #include diff --git a/include/ccf/js/extensions/ccf/rpc.h b/include/ccf/js/extensions/ccf/rpc.h index b37a5caea791..b7d8dfdb6b0f 100644 --- a/include/ccf/js/extensions/ccf/rpc.h +++ b/include/ccf/js/extensions/ccf/rpc.h @@ -2,8 +2,8 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/rpc_context.h" #include "ccf/js/extensions/extension_interface.h" +#include "ccf/rpc_context.h" namespace ccf::js::extensions { diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 2c0d5c37247d..a0b1440e9676 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -16,10 +16,6 @@ // Custom Endpoints #include "ccf/bundle.h" #include "ccf/endpoints/authentication/js.h" -#include "ccf/service/tables/modules.h" -#include "endpoint.h" -#include "js/interpreter_cache_interface.h" - #include "ccf/js/core/context.h" #include "ccf/js/extensions/ccf/consensus.h" #include "ccf/js/extensions/ccf/converters.h" @@ -30,6 +26,9 @@ #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" +#include "ccf/service/tables/modules.h" +#include "endpoint.h" +#include "js/interpreter_cache_interface.h" using namespace nlohmann; @@ -38,7 +37,8 @@ namespace basicapp class CustomJSEndpointRegistry : public ccf::UserEndpointRegistry { private: - std::shared_ptr interpreter_cache = nullptr; + std::shared_ptr interpreter_cache = + nullptr; public: CustomJSEndpointRegistry(ccfapp::AbstractNodeContext& context) : diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 5b33419b0b31..28cece1a0f35 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -7,12 +7,7 @@ #include "ccf/crypto/rsa_key_pair.h" #include "ccf/endpoints/authentication/all_of_auth.h" #include "ccf/historical_queries_adapter.h" -#include "ccf/node/host_processes_interface.h" -#include "ccf/service/tables/jsengine.h" -#include "ccf/version.h" -#include "enclave/enclave_time.h" #include "ccf/js/core/context.h" -#include "js/core/wrapped_property_enum.h" #include "ccf/js/extensions/ccf/consensus.h" #include "ccf/js/extensions/ccf/converters.h" #include "ccf/js/extensions/ccf/crypto.h" @@ -22,6 +17,11 @@ #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" +#include "ccf/node/host_processes_interface.h" +#include "ccf/service/tables/jsengine.h" +#include "ccf/version.h" +#include "enclave/enclave_time.h" +#include "js/core/wrapped_property_enum.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" #include "js/modules.h" diff --git a/src/apps/js_generic/request_extension.h b/src/apps/js_generic/request_extension.h index 0d7f3a5ba4b0..ffb9bc8baca5 100644 --- a/src/apps/js_generic/request_extension.h +++ b/src/apps/js_generic/request_extension.h @@ -4,9 +4,9 @@ #include "ccf/base_endpoint_registry.h" #include "ccf/endpoint_context.h" -#include "ccf/rpc_context.h" #include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/extension_interface.h" +#include "ccf/rpc_context.h" namespace ccfapp { diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index 58626efb2fe6..84fc6594cfff 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -3,6 +3,7 @@ #pragma once #include "ccf/app_interface.h" #include "ccf/ds/logger.h" +#include "ccf/js/core/context.h" #include "ccf/node_context.h" #include "ccf/node_subsystem_interface.h" #include "ccf/pal/enclave.h" @@ -13,7 +14,6 @@ #include "indexing/enclave_lfs_access.h" #include "indexing/historical_transaction_fetcher.h" #include "interface.h" -#include "ccf/js/core/context.h" #include "js/ffi_plugins.h" #include "js/interpreter_cache.h" #include "node/acme_challenge_frontend.h" diff --git a/src/js/core/context.cpp b/src/js/core/context.cpp index ecf314ae1420..faf7f83bb018 100644 --- a/src/js/core/context.cpp +++ b/src/js/core/context.cpp @@ -4,14 +4,14 @@ #include "ccf/js/core/context.h" #include "ccf/ds/hex.h" -#include "ccf/pal/locking.h" -#include "enclave/enclave_time.h" #include "ccf/js/core/runtime.h" #include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/console.h" +#include "ccf/js/tx_access.h" +#include "ccf/pal/locking.h" +#include "enclave/enclave_time.h" #include "js/ffi_plugins.h" #include "js/global_class_ids.h" -#include "ccf/js/tx_access.h" #include #include diff --git a/src/js/extensions/ccf/consensus.cpp b/src/js/extensions/ccf/consensus.cpp index 18f2538e5171..2b752939e89b 100644 --- a/src/js/extensions/ccf/consensus.cpp +++ b/src/js/extensions/ccf/consensus.cpp @@ -4,8 +4,8 @@ #include "ccf/js/extensions/ccf/consensus.h" #include "ccf/base_endpoint_registry.h" -#include "js/checks.h" #include "ccf/js/core/context.h" +#include "js/checks.h" #include diff --git a/src/js/extensions/ccf/converters.cpp b/src/js/extensions/ccf/converters.cpp index 355bba6137fd..898a8bba0e4c 100644 --- a/src/js/extensions/ccf/converters.cpp +++ b/src/js/extensions/ccf/converters.cpp @@ -6,9 +6,9 @@ #include "ccf/js/extensions/ccf/converters.h" +#include "ccf/js/core/context.h" #include "ccf/version.h" #include "js/checks.h" -#include "ccf/js/core/context.h" #include "js/modules.h" #include "node/rpc/jwt_management.h" diff --git a/src/js/extensions/ccf/crypto.cpp b/src/js/extensions/ccf/crypto.cpp index 6cdfa765a9d8..9f9c98f87829 100644 --- a/src/js/extensions/ccf/crypto.cpp +++ b/src/js/extensions/ccf/crypto.cpp @@ -11,8 +11,8 @@ #include "ccf/crypto/rsa_key_pair.h" #include "ccf/crypto/sha256.h" #include "ccf/crypto/verifier.h" -#include "js/checks.h" #include "ccf/js/core/context.h" +#include "js/checks.h" #include "tls/ca.h" namespace ccf::js::extensions diff --git a/src/js/extensions/ccf/gov_effects.cpp b/src/js/extensions/ccf/gov_effects.cpp index 3baac7b17f52..c2ba837fafc9 100644 --- a/src/js/extensions/ccf/gov_effects.cpp +++ b/src/js/extensions/ccf/gov_effects.cpp @@ -6,8 +6,8 @@ #include "ccf/js/extensions/ccf/gov_effects.h" -#include "ccf/version.h" #include "ccf/js/core/context.h" +#include "ccf/version.h" #include "js/modules.h" #include "node/rpc/jwt_management.h" diff --git a/src/js/extensions/ccf/historical.cpp b/src/js/extensions/ccf/historical.cpp index 3db00704bb96..03dbb64b8bac 100644 --- a/src/js/extensions/ccf/historical.cpp +++ b/src/js/extensions/ccf/historical.cpp @@ -5,9 +5,9 @@ #include "ccf/ds/hex.h" #include "ccf/historical_queries_interface.h" -#include "js/checks.h" #include "ccf/js/core/context.h" -#include "ccf/js/extensions/ccf/kv_helpers.h" +#include "js/checks.h" +#include "js/extensions/ccf/kv_helpers.h" #include "kv/untyped_map.h" namespace ccf::js::extensions diff --git a/src/js/extensions/ccf/kv.cpp b/src/js/extensions/ccf/kv.cpp index 9aa396300e2b..7b231c700bbc 100644 --- a/src/js/extensions/ccf/kv.cpp +++ b/src/js/extensions/ccf/kv.cpp @@ -3,9 +3,9 @@ #include "ccf/js/extensions/ccf/kv.h" -#include "js/checks.h" #include "ccf/js/core/context.h" -#include "ccf/js/extensions/ccf/kv_helpers.h" +#include "js/checks.h" +#include "js/extensions/ccf/kv_helpers.h" #include "js/global_class_ids.h" #include "js/map_access_permissions.h" diff --git a/include/ccf/js/extensions/ccf/kv_helpers.h b/src/js/extensions/ccf/kv_helpers.h similarity index 100% rename from include/ccf/js/extensions/ccf/kv_helpers.h rename to src/js/extensions/ccf/kv_helpers.h diff --git a/src/js/extensions/ccf/rpc.cpp b/src/js/extensions/ccf/rpc.cpp index d3fdc8852f68..886ae0650607 100644 --- a/src/js/extensions/ccf/rpc.cpp +++ b/src/js/extensions/ccf/rpc.cpp @@ -3,8 +3,9 @@ #include "ccf/js/extensions/ccf/rpc.h" -#include "ccf/rpc_context.h" #include "ccf/js/core/context.h" +#include "ccf/rpc_context.h" + #include namespace ccf::js::extensions diff --git a/src/js/interpreter_cache_interface.h b/src/js/interpreter_cache_interface.h index ae20e6168eb8..d01572219102 100644 --- a/src/js/interpreter_cache_interface.h +++ b/src/js/interpreter_cache_interface.h @@ -3,8 +3,8 @@ #pragma once #include "ccf/endpoint.h" -#include "ccf/node_subsystem_interface.h" #include "ccf/js/tx_access.h" +#include "ccf/node_subsystem_interface.h" namespace ccf::js { diff --git a/src/js/openenclave.cpp b/src/js/openenclave.cpp index 7643c8d534bf..dbaedae4be1a 100644 --- a/src/js/openenclave.cpp +++ b/src/js/openenclave.cpp @@ -2,11 +2,11 @@ // Licensed under the Apache 2.0 License. #include "ccf/ds/hex.h" +#include "ccf/js/core/context.h" #include "ccf/js_openenclave_plugin.h" #include "ccf/js_plugin.h" #include "ccf/version.h" #include "js/checks.h" -#include "ccf/js/core/context.h" #include #include diff --git a/src/js/snp_attestation.cpp b/src/js/snp_attestation.cpp index 250ef6d43acd..267a058d0506 100644 --- a/src/js/snp_attestation.cpp +++ b/src/js/snp_attestation.cpp @@ -1,12 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. +#include "ccf/js/core/context.h" #include "ccf/js_plugin.h" #include "ccf/js_snp_attestation_plugin.h" #include "ccf/pal/attestation.h" #include "ccf/version.h" #include "js/checks.h" -#include "ccf/js/core/context.h" #include "node/uvm_endorsements.h" #include diff --git a/src/node/gov/handlers/proposals.h b/src/node/gov/handlers/proposals.h index 4786b1261c0e..25b1c221c52a 100644 --- a/src/node/gov/handlers/proposals.h +++ b/src/node/gov/handlers/proposals.h @@ -3,8 +3,8 @@ #pragma once #include "ccf/base_endpoint_registry.h" -#include "js/common_context.h" #include "ccf/js/extensions/ccf/gov_effects.h" +#include "js/common_context.h" #include "js/extensions/ccf/network.h" #include "js/extensions/ccf/node.h" #include "node/gov/api_version.h" diff --git a/src/node/node_state.h b/src/node/node_state.h index 9cb3baafa4ac..42ffc444dfad 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -7,6 +7,7 @@ #include "ccf/crypto/symmetric_key.h" #include "ccf/crypto/verifier.h" #include "ccf/ds/logger.h" +#include "ccf/js/core/context.h" #include "ccf/pal/attestation.h" #include "ccf/pal/locking.h" #include "ccf/pal/platform.h" @@ -25,7 +26,6 @@ #include "history.h" #include "http/http_parser.h" #include "indexing/indexer.h" -#include "ccf/js/core/context.h" #include "js/global_class_ids.h" #include "network_state.h" #include "node/hooks.h" diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 5c767051f157..d35a011590fe 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -5,6 +5,7 @@ #include "ccf/common_auth_policies.h" #include "ccf/common_endpoint_registry.h" #include "ccf/http_query.h" +#include "ccf/js/core/context.h" #include "ccf/json_handler.h" #include "ccf/node/quote.h" #include "ccf/odata_error.h" @@ -16,7 +17,6 @@ #include "ds/std_formatters.h" #include "enclave/reconfiguration_type.h" #include "frontend.h" -#include "ccf/js/core/context.h" #include "node/network_state.h" #include "node/rpc/jwt_management.h" #include "node/rpc/no_create_tx_claims_digest.cpp" From 6aa8840ab34db62f4862a9eb71356e289d67e84a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 16:57:17 +0000 Subject: [PATCH 09/25] reuse_policy --- .../apps/basic/custom_endpoints/registry.h | 57 +++++++++++++------ src/apps/js_generic/js_generic_base.cpp | 3 +- src/js/interpreter_cache.h | 9 +-- src/js/interpreter_cache_interface.h | 3 +- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index a0b1440e9676..19045988d6a6 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -15,6 +15,7 @@ // Custom Endpoints #include "ccf/bundle.h" +#include "ccf/endpoint.h" #include "ccf/endpoints/authentication/js.h" #include "ccf/js/core/context.h" #include "ccf/js/extensions/ccf/consensus.h" @@ -202,23 +203,45 @@ namespace basicapp ccf::endpoints::EndpointContext& endpoint_ctx, const std::optional& pre_exec_hook = std::nullopt) { - // TBD: interpreter re-use logic - // TBD: runtime options - - // TBD: private headers - // const auto rw_access = - // endpoint->properties.mode == ccf::endpoints::Mode::ReadWrite ? - // js::TxAccess::APP_RW : - // js::TxAccess::APP_RO; - - // std::shared_ptr interpreter = - // interpreter_cache->get_interpreter(rw_access, *endpoint, - // flush_marker); - // if (interpreter == nullptr) - // { - // throw std::logic_error("Cache failed to produce interpreter"); - // } - // js::core::Context& ctx = *interpreter; + // This KV Value should be updated by any governance actions which modify + // the JS app (including _any_ of its contained modules). We then use the + // version where it was last modified as a safe approximation of when an + // interpreter is unsafe to use. If this value is written to, the + // version_of_previous_write will advance, and all cached interpreters + // will be flushed. + const auto interpreter_flush = endpoint_ctx.tx.ro( + "custom_enpoints.interpreter_flush"); + const auto flush_marker = + interpreter_flush->get_version_of_previous_write().value_or(0); + + const auto rw_access = + endpoint->properties.mode == ccf::endpoints::Mode::ReadWrite ? + ccf::js::TxAccess::APP_RW : + ccf::js::TxAccess::APP_RO; + std::optional reuse_policy = + endpoint->properties.interpreter_reuse; + std::shared_ptr interpreter = + interpreter_cache->get_interpreter( + rw_access, reuse_policy, flush_marker); + if (interpreter == nullptr) + { + throw std::logic_error("Cache failed to produce interpreter"); + } + ccf::js::core::Context& ctx = *interpreter; + + // Prevent any other thread modifying this interpreter, until this + // function completes. We could create interpreters per-thread, but then + // we would get no cross-thread caching benefit (and would need to either + // enforce, or share, caps across per-thread caches). We choose + // instead to allow interpreters to be maximally reused, even across + // threads, at the cost of locking (and potentially stalling another + // thread's request execution) here. + std::lock_guard guard(ctx.lock); + // Update the top of the stack for the current thread, used by the stack + // guard Note this is only active outside SGX + JS_UpdateStackTop(ctx.runtime()); + // Make the heap and stack limits safe while we init the runtime + ctx.runtime().reset_runtime_options(); // TBD: Run fetched endpoint CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 28cece1a0f35..414beebfa3dd 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -123,7 +123,8 @@ namespace ccfapp js::TxAccess::APP_RW : js::TxAccess::APP_RO; std::shared_ptr interpreter = - interpreter_cache->get_interpreter(rw_access, *endpoint, flush_marker); + interpreter_cache->get_interpreter( + rw_access, endpoint->properties.interpreter_reuse, flush_marker); if (interpreter == nullptr) { throw std::logic_error("Cache failed to produce interpreter"); diff --git a/src/js/interpreter_cache.h b/src/js/interpreter_cache.h index 05aac1a50af7..ae9d6299bf63 100644 --- a/src/js/interpreter_cache.h +++ b/src/js/interpreter_cache.h @@ -33,7 +33,8 @@ namespace ccf::js std::shared_ptr get_interpreter( js::TxAccess access, - const JSDynamicEndpoint& endpoint, + const std::optional& + interpreter_reuse, size_t freshness_marker) override { if (access != js::TxAccess::APP_RW && access != js::TxAccess::APP_RO) @@ -55,13 +56,13 @@ namespace ccf::js cache_build_marker = freshness_marker; } - if (endpoint.properties.interpreter_reuse.has_value()) + if (interpreter_reuse.has_value()) { - switch (endpoint.properties.interpreter_reuse->kind) + switch (interpreter_reuse->kind) { case ccf::endpoints::InterpreterReusePolicy::KeyBased: { - auto key = endpoint.properties.interpreter_reuse->key; + auto key = interpreter_reuse->key; if (access == js::TxAccess::APP_RW) { key += " (rw)"; diff --git a/src/js/interpreter_cache_interface.h b/src/js/interpreter_cache_interface.h index d01572219102..c0ee446881fd 100644 --- a/src/js/interpreter_cache_interface.h +++ b/src/js/interpreter_cache_interface.h @@ -38,7 +38,8 @@ namespace ccf::js // execution, where some global initialisation may already be done. virtual std::shared_ptr get_interpreter( js::TxAccess access, - const JSDynamicEndpoint& endpoint, + const std::optional& + interpreter_reuse, size_t freshness_marker) = 0; // Cap the total number of interpreters which will be retained. The From 997cc12abd9d7ada97331ff0c489dd862668f478 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 20:19:22 +0000 Subject: [PATCH 10/25] Move RequestExtension from js_generic to public includes --- CMakeLists.txt | 5 ++- .../js/extensions/ccf}/request_extension.h | 2 +- .../ccf/js}/named_auth_policies.h | 0 .../apps/basic/custom_endpoints/registry.h | 32 +++++++++++++++++++ src/apps/js_generic/js_generic_base.cpp | 6 ++-- .../extensions/ccf}/request_extension.cpp | 6 ++-- 6 files changed, 41 insertions(+), 10 deletions(-) rename {src/apps/js_generic => include/ccf/js/extensions/ccf}/request_extension.h (96%) rename {src/apps/js_generic => include/ccf/js}/named_auth_policies.h (100%) rename src/{apps/js_generic => js/extensions/ccf}/request_extension.cpp (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 94a3e5d5416e..3fa82f7b5fc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,6 +350,7 @@ set(CCF_JS_SOURCES ${CCF_DIR}/src/js/extensions/ccf/network.cpp ${CCF_DIR}/src/js/extensions/ccf/node.cpp ${CCF_DIR}/src/js/extensions/ccf/rpc.cpp + ${CCF_DIR}/src/js/extensions/ccf/request_extension.cpp ) if(COMPILE_TARGET STREQUAL "sgx") @@ -590,9 +591,7 @@ elseif(COMPILE_TARGET STREQUAL "virtual") set(JS_SNP_ATTESTATION_VIRTUAL js_snp_attestation.virtual) endif() -set(JS_GENERIC_SOURCES ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp - ${CCF_DIR}/src/apps/js_generic/request_extension.cpp -) +set(JS_GENERIC_SOURCES ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp) if(COMPILE_TARGET STREQUAL "sgx") add_enclave_library(js_generic_base.enclave ${JS_GENERIC_SOURCES}) target_link_libraries(js_generic_base.enclave PUBLIC ccf.enclave) diff --git a/src/apps/js_generic/request_extension.h b/include/ccf/js/extensions/ccf/request_extension.h similarity index 96% rename from src/apps/js_generic/request_extension.h rename to include/ccf/js/extensions/ccf/request_extension.h index ffb9bc8baca5..62e33d894dc1 100644 --- a/src/apps/js_generic/request_extension.h +++ b/include/ccf/js/extensions/ccf/request_extension.h @@ -8,7 +8,7 @@ #include "ccf/js/extensions/extension_interface.h" #include "ccf/rpc_context.h" -namespace ccfapp +namespace ccf::js::extensions { /** **/ diff --git a/src/apps/js_generic/named_auth_policies.h b/include/ccf/js/named_auth_policies.h similarity index 100% rename from src/apps/js_generic/named_auth_policies.h rename to include/ccf/js/named_auth_policies.h diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 19045988d6a6..aa08335603b2 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -26,6 +26,7 @@ #include "ccf/js/extensions/ccf/kv.h" #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" +#include "ccf/js/extensions/ccf/request_extension.h" #include "ccf/js/extensions/math/random.h" #include "ccf/service/tables/modules.h" #include "endpoint.h" @@ -243,6 +244,37 @@ namespace basicapp // Make the heap and stack limits safe while we init the runtime ctx.runtime().reset_runtime_options(); + // TBD: module loader + // JS_SetModuleLoaderFunc( + // ctx.runtime(), nullptr, js::js_app_module_loader, &endpoint_ctx.tx); + + // Extensions with a dependency on this endpoint context (invocation), + // which must be removed after execution. + ccf::js::extensions::Extensions local_extensions; + + // ccf.kv.* + local_extensions.emplace_back( + std::make_shared(&endpoint_ctx.tx)); + + // ccf.rpc.* + local_extensions.emplace_back( + std::make_shared( + endpoint_ctx.rpc_ctx.get())); + + auto request_extension = + std::make_shared(endpoint_ctx.rpc_ctx.get()); + local_extensions.push_back(request_extension); + + for (auto extension : local_extensions) + { + ctx.add_extension(extension); + } + + if (pre_exec_hook.has_value()) + { + pre_exec_hook.value()(ctx); + } + // TBD: Run fetched endpoint CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); } diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 414beebfa3dd..5dfcfd9fdfa9 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "apps/js_generic/named_auth_policies.h" -#include "apps/js_generic/request_extension.h" +#include "ccf/js/named_auth_policies.h" +#include "ccf/js/extensions/ccf/request_extension.h" #include "ccf/app_interface.h" #include "ccf/crypto/key_wrap.h" #include "ccf/crypto/rsa_key_pair.h" @@ -162,7 +162,7 @@ namespace ccfapp endpoint_ctx.rpc_ctx.get())); auto request_extension = - std::make_shared(endpoint_ctx.rpc_ctx.get()); + std::make_shared(endpoint_ctx.rpc_ctx.get()); local_extensions.push_back(request_extension); for (auto extension : local_extensions) diff --git a/src/apps/js_generic/request_extension.cpp b/src/js/extensions/ccf/request_extension.cpp similarity index 98% rename from src/apps/js_generic/request_extension.cpp rename to src/js/extensions/ccf/request_extension.cpp index 5f109e8f18af..8dddb3a42349 100644 --- a/src/apps/js_generic/request_extension.cpp +++ b/src/js/extensions/ccf/request_extension.cpp @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "apps/js_generic/request_extension.h" +#include "ccf/js/extensions/ccf/request_extension.h" -#include "apps/js_generic/named_auth_policies.h" +#include "ccf/js/named_auth_policies.h" #include "ccf/endpoints/authentication/all_of_auth.h" #include "ccf/endpoints/authentication/cert_auth.h" #include "ccf/endpoints/authentication/cose_auth.h" @@ -13,7 +13,7 @@ #include -namespace ccfapp +namespace ccf::js::extensions { namespace { From 5a372fec61703f140347351210d34176aa060cea Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 21 May 2024 20:25:33 +0000 Subject: [PATCH 11/25] Yes, we know, it's an extension --- CMakeLists.txt | 2 +- .../js/extensions/ccf/{request_extension.h => request.h} | 0 samples/apps/basic/custom_endpoints/registry.h | 5 +++-- src/apps/js_generic/js_generic_base.cpp | 7 ++++--- .../extensions/ccf/{request_extension.cpp => request.cpp} | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) rename include/ccf/js/extensions/ccf/{request_extension.h => request.h} (100%) rename src/js/extensions/ccf/{request_extension.cpp => request.cpp} (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fa82f7b5fc7..c1b3c6b27658 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,7 +350,7 @@ set(CCF_JS_SOURCES ${CCF_DIR}/src/js/extensions/ccf/network.cpp ${CCF_DIR}/src/js/extensions/ccf/node.cpp ${CCF_DIR}/src/js/extensions/ccf/rpc.cpp - ${CCF_DIR}/src/js/extensions/ccf/request_extension.cpp + ${CCF_DIR}/src/js/extensions/ccf/request.cpp ) if(COMPILE_TARGET STREQUAL "sgx") diff --git a/include/ccf/js/extensions/ccf/request_extension.h b/include/ccf/js/extensions/ccf/request.h similarity index 100% rename from include/ccf/js/extensions/ccf/request_extension.h rename to include/ccf/js/extensions/ccf/request.h diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index aa08335603b2..af99b29740c6 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -24,9 +24,9 @@ #include "ccf/js/extensions/ccf/historical.h" #include "ccf/js/extensions/ccf/host.h" #include "ccf/js/extensions/ccf/kv.h" +#include "ccf/js/extensions/ccf/request.h" #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" -#include "ccf/js/extensions/ccf/request_extension.h" #include "ccf/js/extensions/math/random.h" #include "ccf/service/tables/modules.h" #include "endpoint.h" @@ -262,7 +262,8 @@ namespace basicapp endpoint_ctx.rpc_ctx.get())); auto request_extension = - std::make_shared(endpoint_ctx.rpc_ctx.get()); + std::make_shared( + endpoint_ctx.rpc_ctx.get()); local_extensions.push_back(request_extension); for (auto extension : local_extensions) diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 5dfcfd9fdfa9..bdcf59f8fa7a 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -1,7 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "ccf/js/named_auth_policies.h" -#include "ccf/js/extensions/ccf/request_extension.h" #include "ccf/app_interface.h" #include "ccf/crypto/key_wrap.h" #include "ccf/crypto/rsa_key_pair.h" @@ -14,9 +12,11 @@ #include "ccf/js/extensions/ccf/historical.h" #include "ccf/js/extensions/ccf/host.h" #include "ccf/js/extensions/ccf/kv.h" +#include "ccf/js/extensions/ccf/request.h" #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" +#include "ccf/js/named_auth_policies.h" #include "ccf/node/host_processes_interface.h" #include "ccf/service/tables/jsengine.h" #include "ccf/version.h" @@ -162,7 +162,8 @@ namespace ccfapp endpoint_ctx.rpc_ctx.get())); auto request_extension = - std::make_shared(endpoint_ctx.rpc_ctx.get()); + std::make_shared( + endpoint_ctx.rpc_ctx.get()); local_extensions.push_back(request_extension); for (auto extension : local_extensions) diff --git a/src/js/extensions/ccf/request_extension.cpp b/src/js/extensions/ccf/request.cpp similarity index 99% rename from src/js/extensions/ccf/request_extension.cpp rename to src/js/extensions/ccf/request.cpp index 8dddb3a42349..f0ee8dbca59c 100644 --- a/src/js/extensions/ccf/request_extension.cpp +++ b/src/js/extensions/ccf/request.cpp @@ -1,15 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. -#include "ccf/js/extensions/ccf/request_extension.h" +#include "ccf/js/extensions/ccf/request.h" -#include "ccf/js/named_auth_policies.h" #include "ccf/endpoints/authentication/all_of_auth.h" #include "ccf/endpoints/authentication/cert_auth.h" #include "ccf/endpoints/authentication/cose_auth.h" #include "ccf/endpoints/authentication/empty_auth.h" #include "ccf/endpoints/authentication/jwt_auth.h" #include "ccf/js/core/context.h" +#include "ccf/js/named_auth_policies.h" #include From ca437faa5d98b1aa42bc82b5e2b8ee9dbb085729 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 22 May 2024 14:14:20 +0000 Subject: [PATCH 12/25] modules --- {src => include/ccf}/js/modules.h | 8 +++---- .../apps/basic/custom_endpoints/registry.h | 24 ++++++++++++++++--- src/apps/js_generic/js_generic_base.cpp | 2 +- src/js/extensions/ccf/converters.cpp | 2 +- src/js/extensions/ccf/gov_effects.cpp | 2 +- 5 files changed, 28 insertions(+), 10 deletions(-) rename {src => include/ccf}/js/modules.h (94%) diff --git a/src/js/modules.h b/include/ccf/js/modules.h similarity index 94% rename from src/js/modules.h rename to include/ccf/js/modules.h index 18a1ac2f4717..c842eb2c4da7 100644 --- a/src/js/modules.h +++ b/include/ccf/js/modules.h @@ -26,7 +26,7 @@ namespace ccf::js auto loaded_module = jsctx.get_module_from_cache(module_name_quickjs); if (loaded_module.has_value()) { - LOG_TRACE_FMT("Using module from interpreter cache '{}'", module_name_kv); + CCF_APP_TRACE("Using module from interpreter cache '{}'", module_name_kv); return loaded_module.value(); } @@ -48,7 +48,7 @@ namespace ccf::js if (!bytecode) { - LOG_TRACE_FMT("Loading module '{}'", module_name_kv); + CCF_APP_TRACE("Loading module '{}'", module_name_kv); auto module = modules->get(module_name_kv); auto& js = module.value(); @@ -76,7 +76,7 @@ namespace ccf::js } else { - LOG_TRACE_FMT("Loading module from bytecode cache '{}'", module_name_kv); + CCF_APP_TRACE("Loading module from bytecode cache '{}'", module_name_kv); module_val = jsctx.read_object( bytecode->data(), bytecode->size(), JS_READ_OBJ_BYTECODE); @@ -112,7 +112,7 @@ namespace ccf::js } } - LOG_TRACE_FMT("Adding module to interpreter cache '{}'", module_name_kv); + CCF_APP_TRACE("Adding module to interpreter cache '{}'", module_name_kv); jsctx.load_module_to_cache(module_name_quickjs, module_val); return module_val; diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index af99b29740c6..337a8e1d5a1f 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -28,6 +28,7 @@ #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" +#include "ccf/js/modules.h" #include "ccf/service/tables/modules.h" #include "endpoint.h" #include "js/interpreter_cache_interface.h" @@ -244,9 +245,8 @@ namespace basicapp // Make the heap and stack limits safe while we init the runtime ctx.runtime().reset_runtime_options(); - // TBD: module loader - // JS_SetModuleLoaderFunc( - // ctx.runtime(), nullptr, js::js_app_module_loader, &endpoint_ctx.tx); + JS_SetModuleLoaderFunc( + ctx.runtime(), nullptr, ccf::js::js_app_module_loader, &endpoint_ctx.tx); // Extensions with a dependency on this endpoint context (invocation), // which must be removed after execution. @@ -276,6 +276,24 @@ namespace basicapp pre_exec_hook.value()(ctx); } + ccf::js::core::JSWrappedValue export_func; + try + { + const auto& props = endpoint->properties; + auto module_val = + ccf::js::load_app_module(ctx, props.js_module.c_str(), &endpoint_ctx.tx); + export_func = ctx.get_exported_function( + module_val, props.js_function, props.js_module); + } + catch (const std::exception& exc) + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + exc.what()); + return; + } + // TBD: Run fetched endpoint CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); } diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index bdcf59f8fa7a..bce4e651d03d 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -24,7 +24,7 @@ #include "js/core/wrapped_property_enum.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" -#include "js/modules.h" +#include "ccf/js/modules.h" #include "node/rpc/rpc_context_impl.h" #include "service/tables/endpoints.h" diff --git a/src/js/extensions/ccf/converters.cpp b/src/js/extensions/ccf/converters.cpp index 898a8bba0e4c..5c86001e0d32 100644 --- a/src/js/extensions/ccf/converters.cpp +++ b/src/js/extensions/ccf/converters.cpp @@ -9,7 +9,7 @@ #include "ccf/js/core/context.h" #include "ccf/version.h" #include "js/checks.h" -#include "js/modules.h" +#include "ccf/js/modules.h" #include "node/rpc/jwt_management.h" #include diff --git a/src/js/extensions/ccf/gov_effects.cpp b/src/js/extensions/ccf/gov_effects.cpp index c2ba837fafc9..9d747a1b6ccb 100644 --- a/src/js/extensions/ccf/gov_effects.cpp +++ b/src/js/extensions/ccf/gov_effects.cpp @@ -8,7 +8,7 @@ #include "ccf/js/core/context.h" #include "ccf/version.h" -#include "js/modules.h" +#include "ccf/js/modules.h" #include "node/rpc/jwt_management.h" #include From fbe0654bb2e51f0577a6e54cd8f9d70491b58b03 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 22 May 2024 16:10:49 +0000 Subject: [PATCH 13/25] fmt --- samples/apps/basic/custom_endpoints/registry.h | 9 ++++++--- src/apps/js_generic/js_generic_base.cpp | 2 +- src/js/extensions/ccf/converters.cpp | 2 +- src/js/extensions/ccf/gov_effects.cpp | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 337a8e1d5a1f..09fdeff79cb8 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -246,7 +246,10 @@ namespace basicapp ctx.runtime().reset_runtime_options(); JS_SetModuleLoaderFunc( - ctx.runtime(), nullptr, ccf::js::js_app_module_loader, &endpoint_ctx.tx); + ctx.runtime(), + nullptr, + ccf::js::js_app_module_loader, + &endpoint_ctx.tx); // Extensions with a dependency on this endpoint context (invocation), // which must be removed after execution. @@ -280,8 +283,8 @@ namespace basicapp try { const auto& props = endpoint->properties; - auto module_val = - ccf::js::load_app_module(ctx, props.js_module.c_str(), &endpoint_ctx.tx); + auto module_val = ccf::js::load_app_module( + ctx, props.js_module.c_str(), &endpoint_ctx.tx); export_func = ctx.get_exported_function( module_val, props.js_function, props.js_module); } diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index bce4e651d03d..3c5416d72672 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -16,6 +16,7 @@ #include "ccf/js/extensions/ccf/rpc.h" #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" +#include "ccf/js/modules.h" #include "ccf/js/named_auth_policies.h" #include "ccf/node/host_processes_interface.h" #include "ccf/service/tables/jsengine.h" @@ -24,7 +25,6 @@ #include "js/core/wrapped_property_enum.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" -#include "ccf/js/modules.h" #include "node/rpc/rpc_context_impl.h" #include "service/tables/endpoints.h" diff --git a/src/js/extensions/ccf/converters.cpp b/src/js/extensions/ccf/converters.cpp index 5c86001e0d32..b8d058ca5781 100644 --- a/src/js/extensions/ccf/converters.cpp +++ b/src/js/extensions/ccf/converters.cpp @@ -7,9 +7,9 @@ #include "ccf/js/extensions/ccf/converters.h" #include "ccf/js/core/context.h" +#include "ccf/js/modules.h" #include "ccf/version.h" #include "js/checks.h" -#include "ccf/js/modules.h" #include "node/rpc/jwt_management.h" #include diff --git a/src/js/extensions/ccf/gov_effects.cpp b/src/js/extensions/ccf/gov_effects.cpp index 9d747a1b6ccb..2f4e1d3ea3fe 100644 --- a/src/js/extensions/ccf/gov_effects.cpp +++ b/src/js/extensions/ccf/gov_effects.cpp @@ -7,8 +7,8 @@ #include "ccf/js/extensions/ccf/gov_effects.h" #include "ccf/js/core/context.h" -#include "ccf/version.h" #include "ccf/js/modules.h" +#include "ccf/version.h" #include "node/rpc/jwt_management.h" #include From 0dcdbc5d63c468ed8c59e848fc8f8c0b94892c6e Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 22 May 2024 17:02:36 +0000 Subject: [PATCH 14/25] . --- include/ccf/js/modules.h | 2 ++ samples/apps/basic/custom_endpoints/registry.h | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/ccf/js/modules.h b/include/ccf/js/modules.h index c842eb2c4da7..04fa6f397454 100644 --- a/include/ccf/js/modules.h +++ b/include/ccf/js/modules.h @@ -6,6 +6,8 @@ #include "ccf/service/tables/modules.h" #include "ccf/tx.h" +#include "ccf/version.h" + #include namespace ccf::js diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 09fdeff79cb8..3d136430589a 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -283,10 +283,11 @@ namespace basicapp try { const auto& props = endpoint->properties; - auto module_val = ccf::js::load_app_module( - ctx, props.js_module.c_str(), &endpoint_ctx.tx); - export_func = ctx.get_exported_function( - module_val, props.js_function, props.js_module); + // Needs #6199 + // auto module_val = ccf::js::load_app_module( + // ctx, props.js_module.c_str(), &endpoint_ctx.tx); + // export_func = ctx.get_exported_function( + // module_val, props.js_function, props.js_module); } catch (const std::exception& exc) { From 2fd1e56795c5f884a66ede2b5116858d224b91b1 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 22 May 2024 19:30:06 +0000 Subject: [PATCH 15/25] Base module resolution --- include/ccf/js/modules.h | 11 ++++++---- .../apps/basic/custom_endpoints/registry.h | 20 +++++++++++-------- tests/programmability.py | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/include/ccf/js/modules.h b/include/ccf/js/modules.h index 04fa6f397454..556396e7c07d 100644 --- a/include/ccf/js/modules.h +++ b/include/ccf/js/modules.h @@ -13,7 +13,10 @@ namespace ccf::js { static inline js::core::JSWrappedValue load_app_module( - JSContext* ctx, const char* module_name, kv::Tx* tx) + JSContext* ctx, const char* module_name, kv::Tx* tx, + const std::string& modules_map = ccf::Tables::MODULES, + const std::string& modules_quickjs_bytecode_map = ccf::Tables::MODULES_QUICKJS_BYTECODE, + const std::string& modules_quickjs_version_map = ccf::Tables::MODULES_QUICKJS_VERSION) { js::core::Context& jsctx = *(js::core::Context*)JS_GetContextOpaque(ctx); @@ -32,16 +35,16 @@ namespace ccf::js return loaded_module.value(); } - const auto modules = tx->ro(ccf::Tables::MODULES); + const auto modules = tx->ro(modules_map); std::optional> bytecode; const auto modules_quickjs_bytecode = tx->ro( - ccf::Tables::MODULES_QUICKJS_BYTECODE); + modules_quickjs_bytecode_map); bytecode = modules_quickjs_bytecode->get(module_name_kv); if (bytecode) { auto modules_quickjs_version = tx->ro( - ccf::Tables::MODULES_QUICKJS_VERSION); + modules_quickjs_version_map); if (modules_quickjs_version->get() != std::string(ccf::quickjs_version)) bytecode = std::nullopt; } diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 3d136430589a..993471c8b802 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -134,7 +134,7 @@ namespace basicapp const auto wrapper = j.get(); auto endpoints = ctx.tx.template rw( - "custom_endpoints.metadata"); + "public:custom_endpoints.metadata"); // Similar to set_js_app for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) { @@ -148,7 +148,7 @@ namespace basicapp } auto modules = - ctx.tx.template rw("custom_endpoints.modules"); + ctx.tx.template rw("public:custom_endpoints.modules"); for (const auto& [name, module] : wrapper.bundle.modules) { modules->put(name, module); @@ -178,7 +178,7 @@ namespace basicapp const auto verb = rpc_ctx.get_request_verb(); auto endpoints = - tx.ro("custom_endpoints.metadata"); + tx.ro("public:custom_endpoints.metadata"); const auto key = ccf::endpoints::EndpointKey{method, verb}; // Look for a direct match of the given path @@ -212,7 +212,7 @@ namespace basicapp // version_of_previous_write will advance, and all cached interpreters // will be flushed. const auto interpreter_flush = endpoint_ctx.tx.ro( - "custom_enpoints.interpreter_flush"); + "public:custom_enpoints.interpreter_flush"); const auto flush_marker = interpreter_flush->get_version_of_previous_write().value_or(0); @@ -284,10 +284,14 @@ namespace basicapp { const auto& props = endpoint->properties; // Needs #6199 - // auto module_val = ccf::js::load_app_module( - // ctx, props.js_module.c_str(), &endpoint_ctx.tx); - // export_func = ctx.get_exported_function( - // module_val, props.js_function, props.js_module); + auto module_val = ccf::js::load_app_module( + ctx, props.js_module.c_str(), &endpoint_ctx.tx, + "public:custom_endpoints.modules", + "public:custom_endpoints.modules_quickjs_bytecode", + "public:custom_endpoints.modules_quickjs_version" + ); + export_func = ctx.get_exported_function( + module_val, props.js_function, props.js_module); } catch (const std::exception& exc) { diff --git a/tests/programmability.py b/tests/programmability.py index 56d5c20ee822..1c8e723c89a1 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -46,7 +46,7 @@ def test_custom_endpoints(network, args): }, } }, - "modules": {"test.js": TESTJS}, + "modules": {"/test.js": TESTJS}, } with primary.client(None, None, user.local_id) as c: From 325deba3f28ea34c5d8a5c389d2c44e73d045230 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 23 May 2024 09:02:31 +0000 Subject: [PATCH 16/25] fmt --- include/ccf/js/modules.h | 19 +++++++++++-------- .../apps/basic/custom_endpoints/registry.h | 7 ++++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/ccf/js/modules.h b/include/ccf/js/modules.h index 556396e7c07d..4e0685d71615 100644 --- a/include/ccf/js/modules.h +++ b/include/ccf/js/modules.h @@ -5,7 +5,6 @@ #include "ccf/ds/logger.h" #include "ccf/service/tables/modules.h" #include "ccf/tx.h" - #include "ccf/version.h" #include @@ -13,10 +12,14 @@ namespace ccf::js { static inline js::core::JSWrappedValue load_app_module( - JSContext* ctx, const char* module_name, kv::Tx* tx, + JSContext* ctx, + const char* module_name, + kv::Tx* tx, const std::string& modules_map = ccf::Tables::MODULES, - const std::string& modules_quickjs_bytecode_map = ccf::Tables::MODULES_QUICKJS_BYTECODE, - const std::string& modules_quickjs_version_map = ccf::Tables::MODULES_QUICKJS_VERSION) + const std::string& modules_quickjs_bytecode_map = + ccf::Tables::MODULES_QUICKJS_BYTECODE, + const std::string& modules_quickjs_version_map = + ccf::Tables::MODULES_QUICKJS_VERSION) { js::core::Context& jsctx = *(js::core::Context*)JS_GetContextOpaque(ctx); @@ -38,13 +41,13 @@ namespace ccf::js const auto modules = tx->ro(modules_map); std::optional> bytecode; - const auto modules_quickjs_bytecode = tx->ro( - modules_quickjs_bytecode_map); + const auto modules_quickjs_bytecode = + tx->ro(modules_quickjs_bytecode_map); bytecode = modules_quickjs_bytecode->get(module_name_kv); if (bytecode) { - auto modules_quickjs_version = tx->ro( - modules_quickjs_version_map); + auto modules_quickjs_version = + tx->ro(modules_quickjs_version_map); if (modules_quickjs_version->get() != std::string(ccf::quickjs_version)) bytecode = std::nullopt; } diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 993471c8b802..0f3e95d0e10e 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -285,11 +285,12 @@ namespace basicapp const auto& props = endpoint->properties; // Needs #6199 auto module_val = ccf::js::load_app_module( - ctx, props.js_module.c_str(), &endpoint_ctx.tx, + ctx, + props.js_module.c_str(), + &endpoint_ctx.tx, "public:custom_endpoints.modules", "public:custom_endpoints.modules_quickjs_bytecode", - "public:custom_endpoints.modules_quickjs_version" - ); + "public:custom_endpoints.modules_quickjs_version"); export_func = ctx.get_exported_function( module_val, props.js_function, props.js_module); } From 7c550d51dfe46e29d2f796a3fbb2394c0a45cfd1 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 23 May 2024 18:11:11 +0000 Subject: [PATCH 17/25] Wrapped enum --- .../ccf}/js/core/wrapped_property_enum.h | 0 .../apps/basic/custom_endpoints/registry.h | 244 +++++++++++++++++- src/apps/js_generic/js_generic_base.cpp | 2 +- tests/programmability.py | 3 +- 4 files changed, 246 insertions(+), 3 deletions(-) rename {src => include/ccf}/js/core/wrapped_property_enum.h (100%) diff --git a/src/js/core/wrapped_property_enum.h b/include/ccf/js/core/wrapped_property_enum.h similarity index 100% rename from src/js/core/wrapped_property_enum.h rename to include/ccf/js/core/wrapped_property_enum.h diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 0f3e95d0e10e..030129531894 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -18,6 +18,7 @@ #include "ccf/endpoint.h" #include "ccf/endpoints/authentication/js.h" #include "ccf/js/core/context.h" +#include "ccf/js/core/wrapped_property_enum.h" #include "ccf/js/extensions/ccf/consensus.h" #include "ccf/js/extensions/ccf/converters.h" #include "ccf/js/extensions/ccf/crypto.h" @@ -303,8 +304,249 @@ namespace basicapp return; } - // TBD: Run fetched endpoint CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); + + // Call exported function; + auto request = request_extension->create_request_obj( + ctx, endpoint->full_uri_path, endpoint_ctx, this); + + auto val = ctx.call_with_rt_options( + export_func, + {request}, + &endpoint_ctx.tx, + ccf::js::core::RuntimeLimitsPolicy::NONE); + + for (auto extension : local_extensions) + { + ctx.remove_extension(extension); + } + + const auto& rt = ctx.runtime(); + + if (val.is_exception()) + { + bool time_out = ctx.interrupt_data.request_timed_out; + std::string error_msg = "Exception thrown while executing."; + if (time_out) + { + error_msg = "Operation took too long to complete."; + } + + auto [reason, trace] = ctx.error_message(); + + if (rt.log_exception_details) + { + CCF_APP_FAIL("{}: {}", reason, trace.value_or("")); + } + + if (rt.return_exception_details) + { + std::vector details = {ccf::ODataJSExceptionDetails{ + ccf::errors::JSException, reason, trace}}; + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + std::move(error_msg), + std::move(details)); + } + else + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + std::move(error_msg)); + } + + return; + } + + // Handle return value: {body, headers, statusCode} + if (!val.is_obj()) + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (not an object)."); + return; + } + + // Response body (also sets a default response content-type header) + { + auto response_body_js = val["body"]; + if (!response_body_js.is_undefined()) + { + std::vector response_body; + size_t buf_size; + size_t buf_offset; + auto typed_array_buffer = ctx.get_typed_array_buffer( + response_body_js, &buf_offset, &buf_size, nullptr); + uint8_t* array_buffer; + if (!typed_array_buffer.is_exception()) + { + size_t buf_size_total; + array_buffer = + JS_GetArrayBuffer(ctx, &buf_size_total, typed_array_buffer.val); + array_buffer += buf_offset; + } + else + { + array_buffer = + JS_GetArrayBuffer(ctx, &buf_size, response_body_js.val); + } + if (array_buffer) + { + endpoint_ctx.rpc_ctx->set_response_header( + http::headers::CONTENT_TYPE, + http::headervalues::contenttype::OCTET_STREAM); + response_body = + std::vector(array_buffer, array_buffer + buf_size); + } + else + { + std::optional str; + if (response_body_js.is_str()) + { + endpoint_ctx.rpc_ctx->set_response_header( + http::headers::CONTENT_TYPE, + http::headervalues::contenttype::TEXT); + str = ctx.to_str(response_body_js); + } + else + { + endpoint_ctx.rpc_ctx->set_response_header( + http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); + auto rval = ctx.json_stringify(response_body_js); + if (rval.is_exception()) + { + auto [reason, trace] = ctx.error_message(); + + if (rt.log_exception_details) + { + CCF_APP_FAIL( + "Failed to convert return value to JSON:{} {}", + reason, + trace.value_or("")); + } + + if (rt.return_exception_details) + { + std::vector details = { + ccf::ODataJSExceptionDetails{ + ccf::errors::JSException, reason, trace}}; + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (error during JSON " + "conversion of body)", + std::move(details)); + } + else + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (error during JSON " + "conversion of body)."); + } + return; + } + str = ctx.to_str(rval); + } + + if (!str) + { + auto [reason, trace] = ctx.error_message(); + + if (rt.log_exception_details) + { + CCF_APP_FAIL( + "Failed to convert return value to JSON:{} {}", + reason, + trace.value_or("")); + } + + if (rt.return_exception_details) + { + std::vector details = { + ccf::ODataJSExceptionDetails{ + ccf::errors::JSException, reason, trace}}; + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (error during string " + "conversion of body).", + std::move(details)); + } + else + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (error during string " + "conversion of body)."); + } + return; + } + + response_body = std::vector(str->begin(), str->end()); + } + endpoint_ctx.rpc_ctx->set_response_body(std::move(response_body)); + } + } + + // Response headers + { + auto response_headers_js = val["headers"]; + if (response_headers_js.is_obj()) + { + ccf::js::core::JSWrappedPropertyEnum prop_enum( + ctx, response_headers_js); + for (size_t i = 0; i < prop_enum.size(); i++) + { + auto prop_name = ctx.to_str(prop_enum[i]); + if (!prop_name) + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (header type)."); + return; + } + auto prop_val = response_headers_js[*prop_name]; + auto prop_val_str = ctx.to_str(prop_val); + if (!prop_val_str) + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (header value type)."); + return; + } + endpoint_ctx.rpc_ctx->set_response_header( + *prop_name, *prop_val_str); + } + } + } + + // Response status code + int response_status_code = HTTP_STATUS_OK; + { + auto status_code_js = val["statusCode"]; + if (!status_code_js.is_undefined() && !JS_IsNull(status_code_js.val)) + { + if (JS_VALUE_GET_TAG(status_code_js.val) != JS_TAG_INT) + { + endpoint_ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Invalid endpoint function return value (status code type)."); + return; + } + response_status_code = JS_VALUE_GET_INT(status_code_js.val); + } + endpoint_ctx.rpc_ctx->set_response_status(response_status_code); + } } void execute_request( diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 3c5416d72672..fdc38cc22b70 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -6,6 +6,7 @@ #include "ccf/endpoints/authentication/all_of_auth.h" #include "ccf/historical_queries_adapter.h" #include "ccf/js/core/context.h" +#include "ccf/js/core/wrapped_property_enum.h" #include "ccf/js/extensions/ccf/consensus.h" #include "ccf/js/extensions/ccf/converters.h" #include "ccf/js/extensions/ccf/crypto.h" @@ -22,7 +23,6 @@ #include "ccf/service/tables/jsengine.h" #include "ccf/version.h" #include "enclave/enclave_time.h" -#include "js/core/wrapped_property_enum.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" #include "node/rpc/rpc_context_impl.h" diff --git a/tests/programmability.py b/tests/programmability.py index 1c8e723c89a1..ed6687af65c8 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -14,7 +14,7 @@ return { statusCode: 200, body: { - error: "Test content", + payload: "Test content", }, }; } @@ -60,6 +60,7 @@ def test_custom_endpoints(network, args): with primary.client() as c: r = c.get("/app/content") assert r.status_code == http.HTTPStatus.OK.value, r.status_code + assert r.body.json()["payload"] == "Test content", r.body.json() return network From b62e535349bb38759733f9c41f909d295942b3c5 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 10:19:36 +0000 Subject: [PATCH 18/25] Interpreter flush, module prefixing --- samples/apps/basic/custom_endpoints/registry.h | 8 +++++++- tests/programmability.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 030129531894..c7d50a8d9e21 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -137,6 +137,7 @@ namespace basicapp auto endpoints = ctx.tx.template rw( "public:custom_endpoints.metadata"); // Similar to set_js_app + endpoints->clear(); for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) { for (const auto& [method, metadata] : methods) @@ -150,12 +151,17 @@ namespace basicapp auto modules = ctx.tx.template rw("public:custom_endpoints.modules"); + modules->clear(); for (const auto& [name, module] : wrapper.bundle.modules) { - modules->put(name, module); + modules->put(fmt::format("/{}", name), module); } // TBD: Bytecode compilation support + auto interpreter_flush = ctx.tx.template rw( + "public:custom_endpoints.interpreter_flush"); + interpreter_flush->put(true); + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); }; diff --git a/tests/programmability.py b/tests/programmability.py index ed6687af65c8..0e07105cdc7a 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -46,7 +46,7 @@ def test_custom_endpoints(network, args): }, } }, - "modules": {"/test.js": TESTJS}, + "modules": {"test.js": TESTJS}, } with primary.client(None, None, user.local_id) as c: From ec54bdc3857e997bcb30e773a8330d600c094c2f Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 10:34:26 +0000 Subject: [PATCH 19/25] Bytecode gen --- .../apps/basic/custom_endpoints/registry.h | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index c7d50a8d9e21..b6d1bac012eb 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -31,6 +31,7 @@ #include "ccf/js/extensions/math/random.h" #include "ccf/js/modules.h" #include "ccf/service/tables/modules.h" +// samples/apps/basic/custom_endpoints/registry.h #include "endpoint.h" #include "js/interpreter_cache_interface.h" @@ -156,12 +157,53 @@ namespace basicapp { modules->put(fmt::format("/{}", name), module); } - // TBD: Bytecode compilation support + // Trigger interpreter flush, in case interpreter reuse + // is enabled for some endpoints auto interpreter_flush = ctx.tx.template rw( "public:custom_endpoints.interpreter_flush"); interpreter_flush->put(true); + // Refresh app bytecode + ccf::js::core::Context jsctx(ccf::js::TxAccess::APP_RW); + jsctx.runtime().set_runtime_options( + &ctx.tx, ccf::js::core::RuntimeLimitsPolicy::NO_LOWER_THAN_DEFAULTS); + JS_SetModuleLoaderFunc( + jsctx.runtime(), nullptr, ccf::js::js_app_module_loader, &ctx.tx); + + auto quickjs_version = ctx.tx.wo( + "public:custom_endpoints.modules_quickjs_version"); + auto quickjs_bytecode = ctx.tx.wo( + "public:custom_endpoints.modules_quickjs_bytecode"); + + quickjs_version->put(ccf::quickjs_version); + quickjs_bytecode->clear(); + + modules->foreach([&](const auto& name, const auto& src) { + auto module_val = ccf::js::load_app_module( + jsctx, + name.c_str(), + &ctx.tx, + "public:custom_endpoints.modules", + "public:custom_endpoints.modules_quickjs_bytecode", + "public:custom_endpoints.modules_quickjs_version"); + + uint8_t* out_buf; + size_t out_buf_len; + int flags = JS_WRITE_OBJ_BYTECODE; + out_buf = JS_WriteObject(jsctx, &out_buf_len, module_val.val, flags); + if (!out_buf) + { + throw std::runtime_error(fmt::format( + "Unable to serialize bytecode for JS module '{}'", name)); + } + + quickjs_bytecode->put(name, {out_buf, out_buf + out_buf_len}); + js_free(jsctx, out_buf); + + return true; + }); + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); }; From 50b29c64a2e7d8e2df94033b9b5741f857016297 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 10:40:03 +0000 Subject: [PATCH 20/25] That was just pointlessly confusing, let's not do that --- samples/apps/basic/custom_endpoints/endpoint.h | 10 ---------- samples/apps/basic/custom_endpoints/registry.h | 6 +++--- 2 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 samples/apps/basic/custom_endpoints/endpoint.h diff --git a/samples/apps/basic/custom_endpoints/endpoint.h b/samples/apps/basic/custom_endpoints/endpoint.h deleted file mode 100644 index 180d53e00c52..000000000000 --- a/samples/apps/basic/custom_endpoints/endpoint.h +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. - -#include "ccf/app_interface.h" - -namespace basicapp -{ - struct CustomJSEndpoint : public ccf::endpoints::Endpoint - {}; -} \ No newline at end of file diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index b6d1bac012eb..727a3d26f21c 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -30,15 +30,15 @@ #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" #include "ccf/js/modules.h" -#include "ccf/service/tables/modules.h" -// samples/apps/basic/custom_endpoints/registry.h -#include "endpoint.h" #include "js/interpreter_cache_interface.h" using namespace nlohmann; namespace basicapp { + struct CustomJSEndpoint : public ccf::endpoints::Endpoint + {}; + class CustomJSEndpointRegistry : public ccf::UserEndpointRegistry { private: From 7f9f55c1e41d47b0bac2c0f70b8778407cbae601 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 12:27:59 +0000 Subject: [PATCH 21/25] namespace --- samples/apps/basic/basic.cpp | 8 ++- .../apps/basic/custom_endpoints/registry.h | 61 ++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 88d34ff0e391..e47e48e23c97 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -27,7 +27,13 @@ namespace basicapp { public: BasicHandlers(ccfapp::AbstractNodeContext& context) : - basicapp::CustomJSEndpointRegistry(context) + basicapp::CustomJSEndpointRegistry( + context, + "custom_endpoints", // Custom JS endpoints can be install with PUT + // /app/custom_endpoints + "public:custom_endpoints" // Internal KV space will be under + // public:custom_endpoints.* + ) { openapi_info.title = "CCF Basic App"; openapi_info.description = diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 727a3d26f21c..fb300b7ddacb 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -44,10 +44,27 @@ namespace basicapp private: std::shared_ptr interpreter_cache = nullptr; + std::string install_endpoint_name; + std::string modules_map; + std::string metadata_map; + std::string interpreter_flush_map; + std::string modules_quickjs_version_map; + std::string modules_quickjs_bytecode_map; public: - CustomJSEndpointRegistry(ccfapp::AbstractNodeContext& context) : - ccf::UserEndpointRegistry(context) + CustomJSEndpointRegistry( + ccfapp::AbstractNodeContext& context, + const std::string& install_endpoint_name_ = "custom_endpoints", + const std::string& kv_prefix_ = "public:custom_endpoints") : + ccf::UserEndpointRegistry(context), + install_endpoint_name(install_endpoint_name_), + modules_map(fmt::format("{}.modules", kv_prefix_)), + metadata_map(fmt::format("{}.metadata", kv_prefix_)), + interpreter_flush_map(fmt::format("{}.interpreter_flush", kv_prefix_)), + modules_quickjs_version_map( + fmt::format("{}.modules_quickjs_version", kv_prefix_)), + modules_quickjs_bytecode_map( + fmt::format("{}.modules_quickjs_bytecode", kv_prefix_)) { interpreter_cache = context.get_subsystem(); @@ -135,8 +152,8 @@ namespace basicapp caller_identity.content.begin(), caller_identity.content.end()); const auto wrapper = j.get(); - auto endpoints = ctx.tx.template rw( - "public:custom_endpoints.metadata"); + auto endpoints = + ctx.tx.template rw(metadata_map); // Similar to set_js_app endpoints->clear(); for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) @@ -150,8 +167,7 @@ namespace basicapp } } - auto modules = - ctx.tx.template rw("public:custom_endpoints.modules"); + auto modules = ctx.tx.template rw(modules_map); modules->clear(); for (const auto& [name, module] : wrapper.bundle.modules) { @@ -160,8 +176,8 @@ namespace basicapp // Trigger interpreter flush, in case interpreter reuse // is enabled for some endpoints - auto interpreter_flush = ctx.tx.template rw( - "public:custom_endpoints.interpreter_flush"); + auto interpreter_flush = + ctx.tx.template rw(interpreter_flush_map); interpreter_flush->put(true); // Refresh app bytecode @@ -171,10 +187,10 @@ namespace basicapp JS_SetModuleLoaderFunc( jsctx.runtime(), nullptr, ccf::js::js_app_module_loader, &ctx.tx); - auto quickjs_version = ctx.tx.wo( - "public:custom_endpoints.modules_quickjs_version"); - auto quickjs_bytecode = ctx.tx.wo( - "public:custom_endpoints.modules_quickjs_bytecode"); + auto quickjs_version = + ctx.tx.wo(modules_quickjs_version_map); + auto quickjs_bytecode = + ctx.tx.wo(modules_quickjs_bytecode_map); quickjs_version->put(ccf::quickjs_version); quickjs_bytecode->clear(); @@ -184,9 +200,9 @@ namespace basicapp jsctx, name.c_str(), &ctx.tx, - "public:custom_endpoints.modules", - "public:custom_endpoints.modules_quickjs_bytecode", - "public:custom_endpoints.modules_quickjs_version"); + modules_map, + modules_quickjs_bytecode_map, + modules_quickjs_version_map); uint8_t* out_buf; size_t out_buf_len; @@ -208,7 +224,7 @@ namespace basicapp }; make_endpoint( - "custom_endpoints", + install_endpoint_name, HTTP_PUT, put_custom_endpoints, {ccf::user_cose_sign1_auth_policy}) @@ -226,8 +242,7 @@ namespace basicapp const auto method = rpc_ctx.get_method(); const auto verb = rpc_ctx.get_request_verb(); - auto endpoints = - tx.ro("public:custom_endpoints.metadata"); + auto endpoints = tx.ro(metadata_map); const auto key = ccf::endpoints::EndpointKey{method, verb}; // Look for a direct match of the given path @@ -260,8 +275,8 @@ namespace basicapp // interpreter is unsafe to use. If this value is written to, the // version_of_previous_write will advance, and all cached interpreters // will be flushed. - const auto interpreter_flush = endpoint_ctx.tx.ro( - "public:custom_enpoints.interpreter_flush"); + const auto interpreter_flush = + endpoint_ctx.tx.ro(interpreter_flush_map); const auto flush_marker = interpreter_flush->get_version_of_previous_write().value_or(0); @@ -337,9 +352,9 @@ namespace basicapp ctx, props.js_module.c_str(), &endpoint_ctx.tx, - "public:custom_endpoints.modules", - "public:custom_endpoints.modules_quickjs_bytecode", - "public:custom_endpoints.modules_quickjs_version"); + modules_map, + modules_quickjs_bytecode_map, + modules_quickjs_version_map); export_func = ctx.get_exported_function( module_val, props.js_function, props.js_module); } From 523f80426be4a76449a79b590cc2f7058f3f4b59 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 12:47:11 +0000 Subject: [PATCH 22/25] Templated paths and doc --- include/ccf/claims_digest.h | 7 ++ .../ccf/node}/rpc_context_impl.h | 4 +- .../ccf/research}/grpc_status.h | 0 samples/apps/basic/basic.cpp | 19 ++++++ .../apps/basic/custom_endpoints/registry.h | 68 ++++++++++++++++++- .../external_executor/external_executor.cpp | 2 +- src/apps/js_generic/js_generic_base.cpp | 2 +- src/endpoints/endpoint_registry.cpp | 2 +- src/endpoints/grpc/grpc.h | 2 +- src/http/http_rpc_context.h | 2 +- src/node/rpc/claims.h | 7 -- src/node/rpc/frontend.h | 2 +- 12 files changed, 101 insertions(+), 16 deletions(-) rename {src/node/rpc => include/ccf/node}/rpc_context_impl.h (98%) rename {src/endpoints/grpc => include/ccf/research}/grpc_status.h (100%) diff --git a/include/ccf/claims_digest.h b/include/ccf/claims_digest.h index 9b55aa7b9ae1..6eb6de405f4f 100644 --- a/include/ccf/claims_digest.h +++ b/include/ccf/claims_digest.h @@ -65,4 +65,11 @@ namespace ccf { ds::json::fill_schema(schema); } + + static ClaimsDigest empty_claims() + { + ClaimsDigest cd; + cd.set(ClaimsDigest::Digest::Representation()); + return cd; + } } \ No newline at end of file diff --git a/src/node/rpc/rpc_context_impl.h b/include/ccf/node/rpc_context_impl.h similarity index 98% rename from src/node/rpc/rpc_context_impl.h rename to include/ccf/node/rpc_context_impl.h index 56f448214bdc..e5b6e244c4e1 100644 --- a/src/node/rpc/rpc_context_impl.h +++ b/include/ccf/node/rpc_context_impl.h @@ -2,9 +2,9 @@ // Licensed under the Apache 2.0 License. #pragma once +#include "ccf/claims_digest.h" +#include "ccf/research/grpc_status.h" #include "ccf/rpc_context.h" -#include "endpoints/grpc/grpc_status.h" -#include "node/rpc/claims.h" namespace ccf { diff --git a/src/endpoints/grpc/grpc_status.h b/include/ccf/research/grpc_status.h similarity index 100% rename from src/endpoints/grpc/grpc_status.h rename to include/ccf/research/grpc_status.h diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index e47e48e23c97..866b9ebec529 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -23,6 +23,25 @@ namespace basicapp using RecordsMap = kv::Map>; static constexpr auto PRIVATE_RECORDS = "records"; + // By subclassing CustomJSEndpointRegistry, this application gains the ability + // to install and execute custom JavaScript endpoints. + // CustomJSEndpointRegistry adds a single built-in C++ endpoint installed at a + // parametrable path (second argument to the constructor) that enables a user + // for which with user_data["isAdmin"] is true to install custom JavaScript + // endpoints. The JavaScript code for these endpoints is stored in the + // internal KV store under a namespace configured in the second argument to + // the constructor. PUT /app/custom_endpoints (in this case) is logically + // equivalent to passing a set_js_app proposal in governance, except the + // application resides in the application space. + // + // Known limitations: + // + // No auditability yet, COSE Sign1 auth is mandated, but the signature is not + // stored. No support for historical endpoints. + // + // Additional functionality compared to set_js_app: + // The KV namespace can be private, to keep the application confidential if + // desired. class BasicHandlers : public basicapp::CustomJSEndpointRegistry { public: diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index fb300b7ddacb..60cd913555d0 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -30,6 +30,7 @@ #include "ccf/js/extensions/console.h" #include "ccf/js/extensions/math/random.h" #include "ccf/js/modules.h" +#include "ccf/node/rpc_context_impl.h" #include "js/interpreter_cache_interface.h" using namespace nlohmann; @@ -258,7 +259,72 @@ namespace basicapp return endpoint_def; } - // TBD: templated endpoints + // If that doesn't exist, look through _all_ the endpoints to find + // templated matches. If there is one, that's a match. More is an error, + // none means delegate to the base class. + { + std::vector matches; + + endpoints->foreach_key([this, &endpoints, &matches, &key, &rpc_ctx]( + const auto& other_key) { + if (key.verb == other_key.verb) + { + const auto opt_spec = + ccf::endpoints::PathTemplateSpec::parse(other_key.uri_path); + if (opt_spec.has_value()) + { + const auto& template_spec = opt_spec.value(); + // This endpoint has templates in its path, and the correct verb + // - now check if template matches the current request's path + std::smatch match; + if (std::regex_match( + key.uri_path, match, template_spec.template_regex)) + { + if (matches.empty()) + { + auto ctx_impl = static_cast(&rpc_ctx); + if (ctx_impl == nullptr) + { + throw std::logic_error("Unexpected type of RpcContext"); + } + // Populate the request_path_params while we have the match, + // though this will be discarded on error if we later find + // multiple matches + auto& path_params = ctx_impl->path_params; + for (size_t i = 0; + i < template_spec.template_component_names.size(); + ++i) + { + const auto& template_name = + template_spec.template_component_names[i]; + const auto& template_value = match[i + 1].str(); + path_params[template_name] = template_value; + } + } + + auto endpoint = std::make_shared(); + endpoint->dispatch = other_key; + endpoint->full_uri_path = fmt::format( + "/{}{}", method_prefix, endpoint->dispatch.uri_path); + endpoint->properties = endpoints->get(other_key).value(); + ccf::instantiate_authn_policies(*endpoint); + matches.push_back(endpoint); + } + } + } + return true; + }); + + if (matches.size() > 1) + { + report_ambiguous_templated_path(key.uri_path, matches); + } + else if (matches.size() == 1) + { + return matches[0]; + } + } + return ccf::endpoints::EndpointRegistry::find_endpoint(tx, rpc_ctx); } diff --git a/src/apps/external_executor/external_executor.cpp b/src/apps/external_executor/external_executor.cpp index ad989776d0bd..f4cfde3b5e8b 100644 --- a/src/apps/external_executor/external_executor.cpp +++ b/src/apps/external_executor/external_executor.cpp @@ -10,6 +10,7 @@ #include "ccf/http_consts.h" #include "ccf/http_responder.h" #include "ccf/json_handler.h" +#include "ccf/node/rpc_context_impl.h" #include "ccf/service/tables/nodes.h" #include "executor_auth_policy.h" #include "executor_code_id.h" @@ -22,7 +23,6 @@ #include "misc.pb.h" #include "node/endpoint_context_impl.h" #include "node/rpc/network_identity_subsystem.h" -#include "node/rpc/rpc_context_impl.h" #define FMT_HEADER_ONLY #include diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index fdc38cc22b70..746ee4decdd1 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -20,12 +20,12 @@ #include "ccf/js/modules.h" #include "ccf/js/named_auth_policies.h" #include "ccf/node/host_processes_interface.h" +#include "ccf/node/rpc_context_impl.h" #include "ccf/service/tables/jsengine.h" #include "ccf/version.h" #include "enclave/enclave_time.h" #include "js/global_class_ids.h" #include "js/interpreter_cache_interface.h" -#include "node/rpc/rpc_context_impl.h" #include "service/tables/endpoints.h" #include diff --git a/src/endpoints/endpoint_registry.cpp b/src/endpoints/endpoint_registry.cpp index 8e45ad9e0ab5..a0d42c1b81b5 100644 --- a/src/endpoints/endpoint_registry.cpp +++ b/src/endpoints/endpoint_registry.cpp @@ -4,9 +4,9 @@ #include "ccf/endpoint_registry.h" #include "ccf/common_auth_policies.h" +#include "ccf/node/rpc_context_impl.h" #include "ccf/pal/locking.h" #include "http/http_parser.h" -#include "node/rpc/rpc_context_impl.h" namespace ccf::endpoints { diff --git a/src/endpoints/grpc/grpc.h b/src/endpoints/grpc/grpc.h index bf7829febfe6..3c6cd57d3a19 100644 --- a/src/endpoints/grpc/grpc.h +++ b/src/endpoints/grpc/grpc.h @@ -3,9 +3,9 @@ #pragma once #include "ccf/endpoint_context.h" +#include "ccf/node/rpc_context_impl.h" #include "ccf/odata_error.h" #include "message.h" -#include "node/rpc/rpc_context_impl.h" #include "node/rpc/rpc_exception.h" #include "stream.h" #include "types.h" diff --git a/src/http/http_rpc_context.h b/src/http/http_rpc_context.h index 5d84975cd068..2e98e303463d 100644 --- a/src/http/http_rpc_context.h +++ b/src/http/http_rpc_context.h @@ -4,10 +4,10 @@ #include "ccf/actors.h" #include "ccf/http_responder.h" +#include "ccf/node/rpc_context_impl.h" #include "ccf/odata_error.h" #include "ccf/rpc_context.h" #include "http_parser.h" -#include "node/rpc/rpc_context_impl.h" namespace http { diff --git a/src/node/rpc/claims.h b/src/node/rpc/claims.h index e735cee0501d..49a8cb450c57 100644 --- a/src/node/rpc/claims.h +++ b/src/node/rpc/claims.h @@ -12,13 +12,6 @@ namespace ccf return ClaimsDigest(); } - static ClaimsDigest empty_claims() - { - ClaimsDigest cd; - cd.set(ClaimsDigest::Digest::Representation()); - return cd; - } - static crypto::Sha256Hash entry_leaf( const std::vector& write_set, const std::optional& commit_evidence_digest, diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index 918ea077eaae..996cc322bb3a 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -6,6 +6,7 @@ #include "ccf/http_status.h" #include "ccf/node_context.h" #include "ccf/pal/locking.h" +#include "ccf/research/grpc_status.h" #include "ccf/service/node_info_network.h" #include "ccf/service/signed_req.h" #include "ccf/service/tables/jwt.h" @@ -13,7 +14,6 @@ #include "ccf/service/tables/service.h" #include "common/configuration.h" #include "enclave/rpc_handler.h" -#include "endpoints/grpc/grpc_status.h" #include "forwarder.h" #include "http/http_jwt.h" #include "kv/compacted_version_conflict.h" From cfbf401d87e0baf6ae7e43c29fe5423722a1646b Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 12:49:22 +0000 Subject: [PATCH 23/25] growth mindset --- samples/apps/basic/basic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 866b9ebec529..f69deaccd152 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -37,7 +37,7 @@ namespace basicapp // Known limitations: // // No auditability yet, COSE Sign1 auth is mandated, but the signature is not - // stored. No support for historical endpoints. + // stored. No support for historical endpoints yet. // // Additional functionality compared to set_js_app: // The KV namespace can be private, to keep the application confidential if From 48c5bdcce71ab62046b83e91213902ad47266dec Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 13:47:18 +0000 Subject: [PATCH 24/25] Split endpoint from install API --- samples/apps/basic/basic.cpp | 64 ++++++- .../apps/basic/custom_endpoints/registry.h | 175 ++++++------------ 2 files changed, 117 insertions(+), 122 deletions(-) diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index f69deaccd152..8a781f7908a0 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -24,13 +24,13 @@ namespace basicapp static constexpr auto PRIVATE_RECORDS = "records"; // By subclassing CustomJSEndpointRegistry, this application gains the ability - // to install and execute custom JavaScript endpoints. - // CustomJSEndpointRegistry adds a single built-in C++ endpoint installed at a - // parametrable path (second argument to the constructor) that enables a user - // for which with user_data["isAdmin"] is true to install custom JavaScript + // execute custom JavaScript endpoints, and exposes the ability to install + // them via install_custom_endpoints(). + // This sample also adds a PUT /app/custom_endpoints that enables a user + // for which user_data["isAdmin"] is true to install custom JavaScript // endpoints. The JavaScript code for these endpoints is stored in the // internal KV store under a namespace configured in the second argument to - // the constructor. PUT /app/custom_endpoints (in this case) is logically + // the constructor. PUT /app/custom_endpoints is logically // equivalent to passing a set_js_app proposal in governance, except the // application resides in the application space. // @@ -40,6 +40,7 @@ namespace basicapp // stored. No support for historical endpoints yet. // // Additional functionality compared to set_js_app: + // // The KV namespace can be private, to keep the application confidential if // desired. class BasicHandlers : public basicapp::CustomJSEndpointRegistry @@ -48,8 +49,6 @@ namespace basicapp BasicHandlers(ccfapp::AbstractNodeContext& context) : basicapp::CustomJSEndpointRegistry( context, - "custom_endpoints", // Custom JS endpoints can be install with PUT - // /app/custom_endpoints "public:custom_endpoints" // Internal KV space will be under // public:custom_endpoints.* ) @@ -132,6 +131,57 @@ namespace basicapp }; make_endpoint("/records", HTTP_POST, post, {ccf::user_cert_auth_policy}) .install(); + + auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { + const auto& caller_identity = + ctx.template get_caller(); + + // Authorization Check + nlohmann::json user_data = nullptr; + auto result = + get_user_data_v1(ctx.tx, caller_identity.user_id, user_data); + if (result == ccf::ApiResult::InternalError) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + fmt::format( + "Failed to get user data for user {}: {}", + caller_identity.user_id, + ccf::api_result_to_str(result))); + return; + } + const auto is_admin_it = user_data.find("isAdmin"); + + // Not every user gets to define custom endpoints, only users with + // isAdmin + if ( + !user_data.is_object() || is_admin_it == user_data.end() || + !is_admin_it.value().get()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_FORBIDDEN, + ccf::errors::AuthorizationFailed, + "Only admins may access this endpoint."); + return; + } + // End of Authorization Check + + const auto j = nlohmann::json::parse( + caller_identity.content.begin(), caller_identity.content.end()); + const auto wrapper = j.get(); + + install_custom_endpoints(ctx, wrapper); + ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); + }; + + make_endpoint( + "/custom_endpoints", + HTTP_PUT, + put_custom_endpoints, + {ccf::user_cose_sign1_auth_policy}) + .set_auto_schema() + .install(); } }; } diff --git a/samples/apps/basic/custom_endpoints/registry.h b/samples/apps/basic/custom_endpoints/registry.h index 60cd913555d0..3bd0be10717f 100644 --- a/samples/apps/basic/custom_endpoints/registry.h +++ b/samples/apps/basic/custom_endpoints/registry.h @@ -45,7 +45,6 @@ namespace basicapp private: std::shared_ptr interpreter_cache = nullptr; - std::string install_endpoint_name; std::string modules_map; std::string metadata_map; std::string interpreter_flush_map; @@ -55,10 +54,8 @@ namespace basicapp public: CustomJSEndpointRegistry( ccfapp::AbstractNodeContext& context, - const std::string& install_endpoint_name_ = "custom_endpoints", const std::string& kv_prefix_ = "public:custom_endpoints") : ccf::UserEndpointRegistry(context), - install_endpoint_name(install_endpoint_name_), modules_map(fmt::format("{}.modules", kv_prefix_)), metadata_map(fmt::format("{}.metadata", kv_prefix_)), interpreter_flush_map(fmt::format("{}.interpreter_flush", kv_prefix_)), @@ -113,128 +110,80 @@ namespace basicapp return interpreter; }); + } - auto put_custom_endpoints = [this](ccf::endpoints::EndpointContext& ctx) { - const auto& caller_identity = - ctx.template get_caller(); - - // Authorization Check - nlohmann::json user_data = nullptr; - auto result = - get_user_data_v1(ctx.tx, caller_identity.user_id, user_data); - if (result == ccf::ApiResult::InternalError) + void install_custom_endpoints( + ccf::endpoints::EndpointContext& ctx, + const ccf::js::BundleWrapper& wrapper) + { + auto endpoints = + ctx.tx.template rw(metadata_map); + endpoints->clear(); + for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) + { + for (const auto& [method, metadata] : methods) { - ctx.rpc_ctx->set_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - ccf::errors::InternalError, - fmt::format( - "Failed to get user data for user {}: {}", - caller_identity.user_id, - ccf::api_result_to_str(result))); - return; + std::string method_upper = method; + nonstd::to_upper(method_upper); + const auto key = ccf::endpoints::EndpointKey{url, method_upper}; + endpoints->put(key, metadata); } - const auto is_admin_it = user_data.find("isAdmin"); + } - // Not every user gets to define custom endpoints, only users with - // isAdmin - if ( - !user_data.is_object() || is_admin_it == user_data.end() || - !is_admin_it.value().get()) - { - ctx.rpc_ctx->set_error( - HTTP_STATUS_FORBIDDEN, - ccf::errors::AuthorizationFailed, - "Only admins may access this endpoint."); - return; - } - // End of Authorization Check + auto modules = ctx.tx.template rw(modules_map); + modules->clear(); + for (const auto& [name, module] : wrapper.bundle.modules) + { + modules->put(fmt::format("/{}", name), module); + } - const auto j = nlohmann::json::parse( - caller_identity.content.begin(), caller_identity.content.end()); - const auto wrapper = j.get(); + // Trigger interpreter flush, in case interpreter reuse + // is enabled for some endpoints + auto interpreter_flush = + ctx.tx.template rw(interpreter_flush_map); + interpreter_flush->put(true); - auto endpoints = - ctx.tx.template rw(metadata_map); - // Similar to set_js_app - endpoints->clear(); - for (const auto& [url, methods] : wrapper.bundle.metadata.endpoints) - { - for (const auto& [method, metadata] : methods) - { - std::string method_upper = method; - nonstd::to_upper(method_upper); - const auto key = ccf::endpoints::EndpointKey{url, method_upper}; - endpoints->put(key, metadata); - } - } + // Refresh app bytecode + ccf::js::core::Context jsctx(ccf::js::TxAccess::APP_RW); + jsctx.runtime().set_runtime_options( + &ctx.tx, ccf::js::core::RuntimeLimitsPolicy::NO_LOWER_THAN_DEFAULTS); + JS_SetModuleLoaderFunc( + jsctx.runtime(), nullptr, ccf::js::js_app_module_loader, &ctx.tx); - auto modules = ctx.tx.template rw(modules_map); - modules->clear(); - for (const auto& [name, module] : wrapper.bundle.modules) - { - modules->put(fmt::format("/{}", name), module); - } + auto quickjs_version = + ctx.tx.wo(modules_quickjs_version_map); + auto quickjs_bytecode = + ctx.tx.wo(modules_quickjs_bytecode_map); - // Trigger interpreter flush, in case interpreter reuse - // is enabled for some endpoints - auto interpreter_flush = - ctx.tx.template rw(interpreter_flush_map); - interpreter_flush->put(true); - - // Refresh app bytecode - ccf::js::core::Context jsctx(ccf::js::TxAccess::APP_RW); - jsctx.runtime().set_runtime_options( - &ctx.tx, ccf::js::core::RuntimeLimitsPolicy::NO_LOWER_THAN_DEFAULTS); - JS_SetModuleLoaderFunc( - jsctx.runtime(), nullptr, ccf::js::js_app_module_loader, &ctx.tx); - - auto quickjs_version = - ctx.tx.wo(modules_quickjs_version_map); - auto quickjs_bytecode = - ctx.tx.wo(modules_quickjs_bytecode_map); - - quickjs_version->put(ccf::quickjs_version); - quickjs_bytecode->clear(); - - modules->foreach([&](const auto& name, const auto& src) { - auto module_val = ccf::js::load_app_module( - jsctx, - name.c_str(), - &ctx.tx, - modules_map, - modules_quickjs_bytecode_map, - modules_quickjs_version_map); - - uint8_t* out_buf; - size_t out_buf_len; - int flags = JS_WRITE_OBJ_BYTECODE; - out_buf = JS_WriteObject(jsctx, &out_buf_len, module_val.val, flags); - if (!out_buf) - { - throw std::runtime_error(fmt::format( - "Unable to serialize bytecode for JS module '{}'", name)); - } + quickjs_version->put(ccf::quickjs_version); + quickjs_bytecode->clear(); - quickjs_bytecode->put(name, {out_buf, out_buf + out_buf_len}); - js_free(jsctx, out_buf); + modules->foreach([&](const auto& name, const auto& src) { + auto module_val = ccf::js::load_app_module( + jsctx, + name.c_str(), + &ctx.tx, + modules_map, + modules_quickjs_bytecode_map, + modules_quickjs_version_map); - return true; - }); + uint8_t* out_buf; + size_t out_buf_len; + int flags = JS_WRITE_OBJ_BYTECODE; + out_buf = JS_WriteObject(jsctx, &out_buf_len, module_val.val, flags); + if (!out_buf) + { + throw std::runtime_error(fmt::format( + "Unable to serialize bytecode for JS module '{}'", name)); + } - ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT); - }; + quickjs_bytecode->put(name, {out_buf, out_buf + out_buf_len}); + js_free(jsctx, out_buf); - make_endpoint( - install_endpoint_name, - HTTP_PUT, - put_custom_endpoints, - {ccf::user_cose_sign1_auth_policy}) - .set_auto_schema() - .install(); + return true; + }); } - // Custom Endpoints - ccf::endpoints::EndpointDefinitionPtr find_endpoint( kv::Tx& tx, ccf::RpcContext& rpc_ctx) override { @@ -413,7 +362,6 @@ namespace basicapp try { const auto& props = endpoint->properties; - // Needs #6199 auto module_val = ccf::js::load_app_module( ctx, props.js_module.c_str(), @@ -433,8 +381,6 @@ namespace basicapp return; } - CCF_APP_INFO("CUSTOM ENDPOINT: {}", endpoint->dispatch.uri_path); - // Call exported function; auto request = request_extension->create_request_obj( ctx, endpoint->full_uri_path, endpoint_ctx, this); @@ -682,7 +628,6 @@ namespace basicapp const CustomJSEndpoint* endpoint, ccf::endpoints::EndpointContext& endpoint_ctx) { - // TBD: historical queries do_execute_request(endpoint, endpoint_ctx); } From edf8ba5096ade2810ad96a7285ceb2124e276ddc Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 24 May 2024 13:56:19 +0000 Subject: [PATCH 25/25] Test update --- samples/apps/basic/basic.cpp | 4 ++- tests/programmability.py | 50 +++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 8a781f7908a0..e1bd36894041 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -37,7 +37,9 @@ namespace basicapp // Known limitations: // // No auditability yet, COSE Sign1 auth is mandated, but the signature is not - // stored. No support for historical endpoints yet. + // stored. + // No support for historical endpoints yet. + // No support for import from external modules. // // Additional functionality compared to set_js_app: // diff --git a/tests/programmability.py b/tests/programmability.py index 0e07105cdc7a..34084aaded23 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -30,38 +30,52 @@ def test_custom_endpoints(network, args): primary, user.service_id, user_data={"isAdmin": True} ) - bundle = { - "metadata": { - "endpoints": { - "/content": { - "get": { - "js_module": "test.js", - "js_function": "content", - "forwarding_required": "never", - "redirection_strategy": "none", - "authn_policies": ["no_auth"], - "mode": "readonly", - "openapi": {}, - } - }, - } - }, + content_endpoint_def = { + "get": { + "js_module": "test.js", + "js_function": "content", + "forwarding_required": "never", + "redirection_strategy": "none", + "authn_policies": ["no_auth"], + "mode": "readonly", + "openapi": {}, + } + } + + bundle_with_content = { + "metadata": {"endpoints": {"/content": content_endpoint_def}}, + "modules": {"test.js": TESTJS}, + } + + bundle_with_other_content = { + "metadata": {"endpoints": {"/other_content": content_endpoint_def}}, "modules": {"test.js": TESTJS}, } with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body={"bundle": bundle}) + r = c.put("/app/custom_endpoints", body={"bundle": bundle_with_content}) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code with primary.client() as c: r = c.get("/app/not_content") assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code - with primary.client() as c: r = c.get("/app/content") assert r.status_code == http.HTTPStatus.OK.value, r.status_code assert r.body.json()["payload"] == "Test content", r.body.json() + with primary.client(None, None, user.local_id) as c: + r = c.put("/app/custom_endpoints", body={"bundle": bundle_with_other_content}) + assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code + + with primary.client() as c: + r = c.get("/app/other_content") + assert r.status_code == http.HTTPStatus.OK.value, r.status_code + assert r.body.json()["payload"] == "Test content", r.body.json() + + r = c.get("/app/content") + assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code + return network