Implement Azure-compliant governance interface (microsoft#5660)

(cherry picked from commit 88cb1ea)

# Conflicts:
#	CMakeLists.txt
#	src/node/rpc/member_frontend.h
#	src/node/share_manager.h
#	src/service/genesis_gen.h
#	tests/infra/clients.py
#	tests/requirements.txt

CMakeLists.txt

@@ -1162,10 +1162,16 @@ if(BUILD_TESTS)
     endif()
 
     add_e2e_test(
-      NAME recovery_test_cft
+      NAME recovery_test_cft_api_0
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
-      CONSENSUS cft
-      ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS}
+      ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} --gov-api-version "classic"
+    )
+
+    add_e2e_test(
+      NAME recovery_test_cft_api_1
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
+      ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} --gov-api-version
+                      "2023-06-01-preview"
     )
 
     add_e2e_test(
@@ -1337,10 +1343,17 @@ if(BUILD_TESTS)
     )
 
     add_e2e_test(
-      NAME membership
+      NAME membership_api_0
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/membership.py
-      CONSENSUS cft
+      ADDITIONAL_ARGS --gov-api-version "classic"
     )
+
+    add_e2e_test(
+      NAME membership_api_1
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/membership.py
+      ADDITIONAL_ARGS --gov-api-version "2023-06-01-preview"
+    )
+
     set(PARTITIONS_TEST_ARGS
         # Higher snapshot interval as the test currently assumes that no
         # transactions

include/ccf/endpoint_registry.h

@@ -42,6 +42,51 @@ namespace ccf::endpoints
   void default_locally_committed_func(
     CommandEndpointContext& ctx, const TxID& tx_id);
 
+  template <typename T>
+  inline bool get_path_param(
+    const ccf::PathParams& params,
+    const std::string& param_name,
+    T& value,
+    std::string& error)
+  {
+    const auto it = params.find(param_name);
+    if (it == params.end())
+    {
+      error = fmt::format("No parameter named '{}' in path", param_name);
+      return false;
+    }
+
+    const auto param_s = it->second;
+    const auto [p, ec] =
+      std::from_chars(param_s.data(), param_s.data() + param_s.size(), value);
+    if (ec != std::errc())
+    {
+      error = fmt::format(
+        "Unable to parse path parameter '{}' as a {}", param_s, param_name);
+      return false;
+    }
+
+    return true;
+  }
+
+  template <>
+  inline bool get_path_param(
+    const ccf::PathParams& params,
+    const std::string& param_name,
+    std::string& value,
+    std::string& error)
+  {
+    const auto it = params.find(param_name);
+    if (it == params.end())
+    {
+      error = fmt::format("No parameter named '{}' in path", param_name);
+      return false;
+    }
+
+    value = it->second;
+    return true;
+  }
+
   /** The EndpointRegistry records the user-defined endpoints for a given
    * CCF application.
    *
@@ -83,24 +128,8 @@ namespace ccf::endpoints
       T& value,
       std::string& error)
     {
-      const auto it = params.find(param_name);
-      if (it == params.end())
-      {
-        error = fmt::format("No parameter named '{}' in path", param_name);
-        return false;
-      }
-
-      const auto param_s = it->second;
-      const auto [p, ec] =
-        std::from_chars(param_s.data(), param_s.data() + param_s.size(), value);
-      if (ec != std::errc())
-      {
-        error = fmt::format(
-          "Unable to parse path parameter '{}' as a {}", param_s, param_name);
-        return false;
-      }
-
-      return true;
+      return ccf::endpoints::get_path_param<T>(
+        params, param_name, value, error);
     }
 
     template <>
@@ -110,15 +139,7 @@ namespace ccf::endpoints
       std::string& value,
       std::string& error)
     {
-      const auto it = params.find(param_name);
-      if (it == params.end())
-      {
-        error = fmt::format("No parameter named '{}' in path", param_name);
-        return false;
-      }
-
-      value = it->second;
-      return true;
+      return ccf::endpoints::get_path_param(params, param_name, value, error);
     }
 
   protected:
@@ -245,6 +266,8 @@ namespace ccf::endpoints
     virtual std::set<RESTVerb> get_allowed_verbs(
       kv::Tx&, const ccf::RpcContext& rpc_ctx);
 
+    virtual bool request_needs_root(const ccf::RpcContext& rpc_ctx);
+
     virtual void report_ambiguous_templated_path(
       const std::string& path,
       const std::vector<EndpointDefinitionPtr>& matches);

include/ccf/endpoints/authentication/cose_auth.h

@@ -102,6 +102,29 @@ namespace ccf
     }
   };
 
+  /** Active Member COSE Sign1 Authentication Policy
+   *
+   * Extends MemberCOSESign1AuthPolicy, to also require that the signer's state
+   * is Active
+   */
+  class ActiveMemberCOSESign1AuthnPolicy : public MemberCOSESign1AuthnPolicy
+  {
+  public:
+    static constexpr auto SECURITY_SCHEME_NAME = "active_member_cose_sign1";
+
+    using MemberCOSESign1AuthnPolicy::MemberCOSESign1AuthnPolicy;
+
+    std::unique_ptr<AuthnIdentity> authenticate(
+      kv::ReadOnlyTx& tx,
+      const std::shared_ptr<ccf::RpcContext>& ctx,
+      std::string& error_reason) override;
+
+    std::string get_security_scheme_name() override
+    {
+      return SECURITY_SCHEME_NAME;
+    }
+  };
+
   /** User COSE Sign1 Authentication Policy
    */
   class UserCOSESign1AuthnPolicy : public AuthnPolicy

include/ccf/http_consts.h

@@ -34,6 +34,7 @@ namespace http
       static constexpr auto OCTET_STREAM = "application/octet-stream";
       static constexpr auto GRPC = "application/grpc";
       static constexpr auto COSE = "application/cose";
+      static constexpr auto JAVASCRIPT = "text/javascript";
     }
   }
 

include/ccf/odata_error.h

@@ -112,6 +112,8 @@ namespace ccf
     ERROR(ProposalCreatedTooLongAgo)
     ERROR(InvalidCreatedAt)
     ERROR(JSException)
+    ERROR(MissingApiVersionParameter)
+    ERROR(UnsupportedApiVersionValue)
 
     // node-to-node (/join and /create):
     ERROR(ConsensusTypeMismatch)

include/ccf/rpc_context.h

@@ -151,7 +151,7 @@ namespace ccf
     }
 
     virtual void set_response_json(
-      nlohmann::json& body, http_status status) = 0;
+      const nlohmann::json& body, http_status status) = 0;
 
     /// Construct error response, formatted according to the request content
     /// type (either JSON OData-formatted or gRPC error)

python/utils/submit_recovery_share.sh

@@ -7,7 +7,7 @@ set -e
 function usage()
 {
     echo "Usage:"""
-    echo "  $0 https://<node-address> --member-enc-privk /path/to/member_enc_privk.pem --cert /path/to/member_cert.pem [CURL_OPTIONS]"
+    echo "  $0 https://<node-address> --member-enc-privk /path/to/member_enc_privk.pem --api-version api_version --cert /path/to/member_cert.pem [CURL_OPTIONS]"
     echo "Retrieves the encrypted recovery share for a given member, decrypts the share and submits it for recovery."
     echo ""
     echo "A sufficient number of recovery shares must be submitted by members to initiate the end of recovery procedure."
@@ -25,6 +25,7 @@ fi
 node_rpc_address=$1
 shift
 
+api_version="classic"
 while [ "$1" != "" ]; do
     case $1 in
         -h|-\?|--help)
@@ -34,6 +35,9 @@ while [ "$1" != "" ]; do
         --member-enc-privk)
             member_enc_privk="$2"
             ;;
+        --api-version)
+            api_version="$2"
+            ;;
         *)
             break
     esac
@@ -67,9 +71,23 @@ fi
 # Compute member ID, as the SHA-256 fingerprint of the signing certificate
 member_id=$(openssl x509 -in "$cert" -noout -fingerprint -sha256 | cut -d "=" -f 2 | sed 's/://g' | awk '{print tolower($0)}')
 
+if [ "${api_version}" == "classic" ]; then
+    get_share_path="gov/encrypted_recovery_share/${member_id}"
+    share_field="encrypted_share"
+    submit_share_path="gov/recovery_share"
+else
+    get_share_path="gov/recovery/encrypted-shares/${member_id}?api-version=${api_version}"
+    share_field="encryptedShare"
+    submit_share_path="gov/recovery/members/${member_id}:recover?api-version=${api_version}"
+fi
+
 # First, retrieve the encrypted recovery share
-encrypted_share=$(curl -sS --fail -X GET "${node_rpc_address}"/gov/encrypted_recovery_share/"${member_id}" "${@}" | jq -r '.encrypted_share')
+encrypted_share=$(curl -sS --fail -X GET "${node_rpc_address}/${get_share_path}" "${@}" | jq -r ".${share_field}")
 
 # Then, decrypt encrypted share with member private key submit decrypted recovery share
 # Note: all in one line so that the decrypted recovery share is not exposed
-echo "${encrypted_share}" | openssl base64 -d | openssl pkeyutl -inkey "${member_enc_privk}" -decrypt -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 | openssl base64 -A | jq -R '{share: (.)}' | curl -i -sS --fail -H "Content-Type: application/json" -X POST "${node_rpc_address}"/gov/recovery_share "${@}" -d @-
+echo "${encrypted_share}" \
+    | openssl base64 -d \
+    | openssl pkeyutl -inkey "${member_enc_privk}" -decrypt -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \
+    | openssl base64 -A | jq -R '{share: (.)}' \
+    | curl -i -sS --fail -H "Content-Type: application/json" -X POST "${node_rpc_address}/${submit_share_path}" "${@}" -d @-

src/endpoints/authentication/cose_auth.cpp

@@ -373,6 +373,38 @@ namespace ccf
          "protected headers. "
          "Signer must be a member identity registered with this service."}});
 
+  std::unique_ptr<AuthnIdentity> ActiveMemberCOSESign1AuthnPolicy::authenticate(
+    kv::ReadOnlyTx& tx,
+    const std::shared_ptr<ccf::RpcContext>& ctx,
+    std::string& error_reason)
+  {
+    auto ident =
+      MemberCOSESign1AuthnPolicy::authenticate(tx, ctx, error_reason);
+    if (ident != nullptr)
+    {
+      auto cose_ident =
+        dynamic_cast<const MemberCOSESign1AuthnIdentity*>(ident.get());
+      if (cose_ident == nullptr)
+      {
+        error_reason = "Unexpected Identity type";
+        return nullptr;
+      }
+
+      const auto member_id = cose_ident->member_id;
+
+      auto member_info_handle =
+        tx.template ro<ccf::MemberInfo>(ccf::Tables::MEMBER_INFO);
+      const auto member = member_info_handle->get(member_id);
+      if (!member.has_value() || member->status != ccf::MemberStatus::ACTIVE)
+      {
+        error_reason = "Signer is not an ACTIVE member";
+        return nullptr;
+      }
+    }
+
+    return ident;
+  }
+
   UserCOSESign1AuthnPolicy::UserCOSESign1AuthnPolicy() = default;
   UserCOSESign1AuthnPolicy::~UserCOSESign1AuthnPolicy() = default;
 

src/endpoints/endpoint_registry.cpp

@@ -535,6 +535,11 @@ namespace ccf::endpoints
     return verbs;
   }
 
+  bool EndpointRegistry::request_needs_root(const ccf::RpcContext& rpc_ctx)
+  {
+    return false;
+  }
+
   void EndpointRegistry::report_ambiguous_templated_path(
     const std::string& path, const std::vector<EndpointDefinitionPtr>& matches)
   {

src/node/gov/README.md

@@ -0,0 +1,12 @@
+This folder implements a new governance API.
+This API is defined in the root-level `typespec-ccf` folder, with a TypeSpec source generating an OpenAPI definition.
+
+It aims to expose all of the functionality of the existing API, defined in `src/node/rpc/member_frontend.h`, but with naming and schemas compliant with Azure API standards.
+
+For a transition period (at least the 4.x release), both APIs will be offered. The plan is to eventually deprecate the old API.
+
+Implementation notes:
+
+- All endpoints validate and process an `api-version` parameter, modifying their behaviour accordingly.
+- To present both under the `/gov` prefix, the old frontend is a subclass of the new frontend.
+- The frontend implementation is split into distinct components which can be more easily moved around, rather than a single monolithic frontend implementation.