diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml index b7477eb501ff..742c347bc18a 100644 --- a/.azure-pipelines-gh-pages.yml +++ b/.azure-pipelines-gh-pages.yml @@ -11,7 +11,7 @@ jobs: variables: Codeql.SkipTaskAutoInjection: true skipComponentGovernanceDetection: true - container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 pool: vmImage: ubuntu-20.04 diff --git a/.azure-pipelines-model-checking.yml b/.azure-pipelines-model-checking.yml index 7ffe0658c1ae..d28f4495f347 100644 --- a/.azure-pipelines-model-checking.yml +++ b/.azure-pipelines-model-checking.yml @@ -21,7 +21,7 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE jobs: diff --git a/.azure-pipelines-release.yml b/.azure-pipelines-release.yml index 03d5e5778bb9..d6785a332921 100644 --- a/.azure-pipelines-release.yml +++ b/.azure-pipelines-release.yml @@ -8,15 +8,15 @@ pr: none resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 0a1dd04cbba4..79b8f435ea7d 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -54,7 +54,7 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.6-0-snp-clang15 + docker pull $ACR_REGISTRY/ccf/ci:2024-06-26-snp-clang15 docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image @@ -63,7 +63,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15 + BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15 - script: | set -ex diff --git a/.azure-pipelines-templates/matrix.yml b/.azure-pipelines-templates/matrix.yml index 4950e65bab0f..01bcf56f8aba 100644 --- a/.azure-pipelines-templates/matrix.yml +++ b/.azure-pipelines-templates/matrix.yml @@ -69,20 +69,6 @@ parameters: ctest_args: '-L "raft_scenario"' jobs: - # Debug - - ${{ each target in parameters.target }}: - - template: common.yml - parameters: - target: ${{ target }} - env: ${{ parameters.env[target] }} - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build[target].cmake_args }}" - cmake_env: "${{ parameters.build[target].cmake_env }}" - suffix: "Debug" - artifact_name: "${{ target }}_Debug" - ctest_filter: "${{ parameters.test[target].ctest_args }}" - depends_on: configure - installExtendedTestingTools: false - # Tracing - template: common.yml parameters: @@ -97,57 +83,6 @@ jobs: installExtendedTestingTools: false ninja_targets: "${{ parameters.build.tracing.ninja_targets }}" - # Performance - - ${{ if eq(parameters.perf_tests, 'run') }}: - - template: common.yml - parameters: - target: SGX - env: ${{ parameters.env.SGX }} - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }}" - suffix: "Perf" - artifact_name: "SGX_Perf" - ctest_filter: "${{ parameters.test.perf.ctest_args }}" - depends_on: configure - installExtendedTestingTools: false - - - ${{ if eq(parameters.perf_tests, 'run') }}: - - template: common.yml - parameters: - target: Virtual - env: ${{ parameters.env.Virtual }} - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" - cmake_env: "${{ parameters.build.Virtual.cmake_env }}" - suffix: "Perf" - artifact_name: "Virtual_Perf" - ctest_filter: "${{ parameters.test.virtual_perf.ctest_args }}" - depends_on: configure - installExtendedTestingTools: false - - - ${{ if eq(parameters.perf_tests, 'run') }}: - - template: common.yml - parameters: - target: Virtual - env: ${{ parameters.env.Virtual }} - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DWORKER_THREADS=2" - cmake_env: "${{ parameters.build.Virtual.cmake_env }}" - suffix: "Perf_MultiThreaded" - artifact_name: "Virtual_Perf_MultiThreaded" - ctest_filter: "-R pi_basic_mt" - depends_on: configure - installExtendedTestingTools: false - - - ${{ if eq(parameters.perf_tests, 'run') }}: - - template: common.yml - parameters: - target: SGX - env: ${{ parameters.env.SGX }} - cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DWORKER_THREADS=2" - suffix: "Perf_MultiThreaded" - artifact_name: "SGX_Perf_MultiThreaded" - ctest_filter: "-R pi_basic_mt" - depends_on: configure - installExtendedTestingTools: false - - ${{ if eq(parameters.perf_tests, 'run') }}: - template: simulation.yml parameters: diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index c8ff90eb3064..5ca38bc59176 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -30,15 +30,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro variables: diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 3708df528b85..1ffb3ec69635 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -32,7 +32,7 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.daily.yml b/.daily.yml index 378a37159c58..6977470ac796 100644 --- a/.daily.yml +++ b/.daily.yml @@ -27,15 +27,15 @@ schedules: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - container: snp - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/.daily_canary b/.daily_canary index a02a115d02e2..b0c8ecafff33 100644 --- a/.daily_canary +++ b/.daily_canary @@ -3,4 +3,4 @@ ( V ) / . \ | +---=---' /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---.. 2024__ -!..! \ No newline at end of file +!..! diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1057f18e21a8..c20457d9c709 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "CCF Development Environment", - "image": "ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15", + "image": "ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15", "runArgs": [], "extensions": [ "eamodio.gitlens", diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index 2096a34220e3..41ad6e61d681 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -11,7 +11,7 @@ jobs: name: Continuous Benchmarking with Bencher runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..d7cb0d6a1835 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,91 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + actions: read + checks: write + +jobs: + build_and_test: + name: CI + strategy: + matrix: + platform: + - name: virtual + image: default + nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + - name: snp + image: default + nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + - name: sgx + image: sgx + nodes: [self-hosted, 1ES.Pool=gha-sgx-ccf-sub] + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro + runs-on: ${{ matrix.platform.nodes }} + container: + image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-26-06-2024 + options: ${{ matrix.platform.options }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build Debug ${{ matrix.platform.name }}" + run: | + set -ex + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DVERBOSE_LOGGING=ON .. + ninja + shell: bash + + - name: "Platform SGX" + run: | + sudo groupadd -fg $(/usr/bin/stat -Lc '%g' /dev/sgx/provision) sgx_prv + sudo usermod -a -G sgx_prv $(whoami) + samples/scripts/sgxinfo.sh + cat /proc/cpuinfo | grep flags | uniq + shell: bash + if: "${{ matrix.platform.name == 'sgx' }}" + + - name: "Platform SNP" + run: | + samples/scripts/snpinfo.sh + shell: bash + if: "${{ matrix.platform.name == 'snp' }}" + + - name: "Test ${{ matrix.platform.name }}" + run: | + set -ex + cd build + rm -rf /github/home/.cache + mkdir -p /github/home/.cache + export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15) + # Unit tests + ./tests.sh --output-on-failure -L unit -j$(nproc --all) + # All other acceptably fast tests, mostly end-to-end + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite|unit" + # Partitions tests + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite" + shell: bash + if: "${{ matrix.platform.name != 'snp' }}" # Needs 1ES Pool support + + - name: "Upload logs for ${{ matrix.platform.name }}" + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.platform.name }} + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + if-no-files-found: ignore + if: success() || failure() diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index b99186c23065..10c322b4851d 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -12,7 +12,7 @@ permissions: read-all jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + container: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b56a06200817..53b2a077c54c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -21,6 +21,9 @@ jobs: name: Analyze # Insufficient space to run on public runner, so use custom pool runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 + options: --user root permissions: security-events: write @@ -28,11 +31,7 @@ jobs: strategy: fail-fast: false matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ["cpp"] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository @@ -40,48 +39,26 @@ jobs: with: fetch-depth: 0 - # Initializes the CodeQL tools for scanning. + # Done before CodeQL init to let it find the commit successfully + - name: Work around git warning + run: git config --global --add safe.directory /__w/CCF/CCF + - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: security-extended - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - #- name: Autobuild - # uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - run: | - cd getting_started/setup_vm - sudo apt update - sudo apt install -y ansible software-properties-common bsdmainutils dnsutils - sudo ansible-playbook ccf-dev.yml --extra-vars "platform=virtual" --extra-vars "require_open_enclave=false" - name: Install dependencies - run: | + set -ex mkdir build cd build - cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-11` -DCMAKE_CXX_COMPILER=`which clang++-11` .. + cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF .. name: Run CMake - run: | cd build - make + make -j16 name: Run Make - name: Perform CodeQL Analysis diff --git a/.github/workflows/tlaplus.yml b/.github/workflows/tlaplus.yml index 546d9396b812..f561f76a7e07 100644 --- a/.github/workflows/tlaplus.yml +++ b/.github/workflows/tlaplus.yml @@ -2,8 +2,7 @@ name: "TLA+ Spec Verification" on: push: - paths: - - "tla/**" + branches: [main] pull_request: paths: - "tla/**" @@ -19,7 +18,7 @@ jobs: name: Model Checking - Consistency runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 steps: - uses: actions/checkout@v4 diff --git a/.multi-thread.yml b/.multi-thread.yml index d8ceba878f3c..40ed8ec0801b 100644 --- a/.multi-thread.yml +++ b/.multi-thread.yml @@ -20,7 +20,7 @@ pr: resources: containers: - container: virtual - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15 + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15 options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: diff --git a/.stress.yml b/.stress.yml index 5fdf14ce6b42..9ad58f2451e8 100644 --- a/.stress.yml +++ b/.stress.yml @@ -24,7 +24,7 @@ schedules: resources: containers: - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx + image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx jobs: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7257a126720d..392bfa76c680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `::threading` is now `ccf::threading`, and `ccf/ds/thread_ids.h` has moved to `ccf/threading/thread_ids.h` - `::consensus` is now `ccf::consensus` - `::tls` is now `ccf::tls` + - `::http` is now `ccf::http` + - `::nonstd` is now `ccf::nonstd` + - `::crypto` is now `ccf::crypto` + - `::kv` is now `ccf::kv` + - `::logger` is now `ccf::logger` + - `::ccfapp` is now `::ccf` +- The `programmability` sample app now demonstrates how applications can define their own extensions, creating bindings between C++ and JS state, and allowing JS endpoints to call functions implemented in C++. +- Introduce `DynamicJSEndpointRegistry::record_action_for_audit_v1` and `DynamicJSEndpointRegistry::check_action_not_replayed_v1` to allow an application making use of the programmability feature to easily implement auditability, and protect users allowed to update the application against replay attacks (#6285). +- Endpoints now support a `ToBackup` redirection strategy, for requests which should never be executed on a primary. These must also be read-only. These are configured similar to `ToPrimary` endpoints, with a `to_backup` object (specifying by-role or statically-addressed targets) in each node's configuration. + +## Changed + +- Updated Open Enclave to [0.19.7](https://github.com/openenclave/openenclave/releases/tag/v0.19.7). ### Changed @@ -25,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Removed - Removed the existing metrics endpoint and API (`GET /api/metrics`, `get_metrics_v1`). Stats for request execution can instead be gathered by overriding the `EndpointRegistry::handle_event_request_completed()` method. +- Removed automatic msgpack support from JSON endpoint adapters, and related `include/ccf/serdes.h` file. ## [5.0.0-dev18] diff --git a/README.md b/README.md index f9c38fec8be2..795eda54591e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ccf -- Continuous Build: [![Build Status](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20Github%20CI?branchName=main)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=3&branchName=main) +- Continuous Build: [![CI](https://github.com/microsoft/CCF/actions/workflows/build.yml/badge.svg)](https://github.com/microsoft/CCF/actions/workflows/build.yml) - Daily Build: [![Build Status](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20GitHub%20Daily?branchName=main)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=7&branchName=main) - Doc Build: [![docs](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20GitHub%20Pages?branchName=main)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=4&branchName=main) - Containers: [![Build and Publish Release Containers](https://github.com/microsoft/CCF/actions/workflows/containers.yml/badge.svg)](https://github.com/microsoft/CCF/actions/workflows/containers.yml) diff --git a/cmake/common.cmake b/cmake/common.cmake index eebb8bb66933..c2f98c0cb74c 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -17,7 +17,7 @@ function(add_unit_test name) set_property( TEST ${name} APPEND - PROPERTY LABELS unit_test + PROPERTY LABELS unit ) set_property( diff --git a/cmake/cpack_settings.cmake b/cmake/cpack_settings.cmake index c51e8485ac7d..a56dff24babd 100644 --- a/cmake/cpack_settings.cmake +++ b/cmake/cpack_settings.cmake @@ -24,7 +24,7 @@ message(STATUS "Debian package version: ${CPACK_DEBIAN_PACKAGE_VERSION}") set(CCF_DEB_BASE_DEPENDENCIES "libuv1 (>= 1.34.2);openssl (>=1.1.1f)") set(CCF_DEB_DEPENDENCIES ${CCF_DEB_BASE_DEPENDENCIES}) -set(OE_VERSION "0.19.6") +set(OE_VERSION "0.19.7") if(COMPILE_TARGET STREQUAL "sgx") list(APPEND CCF_DEB_DEPENDENCIES "libc++1-11;libc++abi1-11;open-enclave (>=${OE_VERSION})" diff --git a/cmake/open_enclave.cmake b/cmake/open_enclave.cmake index 4bb7b8fc0040..9ccb11b38dfa 100644 --- a/cmake/open_enclave.cmake +++ b/cmake/open_enclave.cmake @@ -13,7 +13,7 @@ if(REQUIRE_OPENENCLAVE) endif() # Find OpenEnclave package - find_package(OpenEnclave 0.19.6 CONFIG REQUIRED) + find_package(OpenEnclave 0.19.7 CONFIG REQUIRED) option(USE_OPENSSL_3 "Use OpenSSL 3.x for Open Enclave builds" ON) if(USE_OPENSSL_3) diff --git a/doc/architecture/ledger.rst b/doc/architecture/ledger.rst index 1f4172bfd4ab..9bb636892be5 100644 --- a/doc/architecture/ledger.rst +++ b/doc/architecture/ledger.rst @@ -35,21 +35,21 @@ The following table describes the structure of a serialised transaction as it is + +------------------------------------------+-------------------------------------------------------------------------+ | | uint64_t | Length of serialised public domain | +----------+------------------------------------------+-------------------------------------------------------------------------+ -| | :cpp:type:`kv::EntryType` | Snapshot, or a WriteSet variant | +| | :cpp:type:`ccf::kv::EntryType` | Snapshot, or a WriteSet variant | + +------------------------------------------+-------------------------------------------------------------------------+ -| | :cpp:type:`kv::Version` | Transaction version | +| | :cpp:type:`ccf::kv::Version` | Transaction version | + +------------------------------------------+-------------------------------------------------------------------------+ -| | :cpp:type:`crypto::Sha256Hash` | User-defined claims digest, when entry type is WriteSetWith.*Claims | +| | :cpp:type:`ccf::crypto::Sha256Hash` | User-defined claims digest, when entry type is WriteSetWith.*Claims | + +------------------------------------------+-------------------------------------------------------------------------+ -| | :cpp:type:`crypto::Sha256Hash` | Commit evidence digest, when entry type is WriteSetWithCommitEvidence.* | +| | :cpp:type:`ccf::crypto::Sha256Hash` | Commit evidence digest, when entry type is WriteSetWithCommitEvidence.* | + +------------------------------------------+-------------------------------------------------------------------------+ -| | :cpp:type:`kv::Version` | Unused, reserved for compatibility | +| | :cpp:type:`ccf::kv::Version` | Unused, reserved for compatibility | + +------------------------------------------+-------------------------------------------------------------------------+ | | **Repeating [0..n]** | With ``n`` the number of maps in the transaction | + +-----+------------------------------------+-------------------------------------------------------------------------+ -| | | std::string | Name of the serialised :cpp:type:`kv::Map` | +| | | std::string | Name of the serialised :cpp:type:`ccf::kv::Map` | | +-----+------------------------------------+-------------------------------------------------------------------------+ -| | | :cpp:type:`kv::Version` | Read version | +| | | :cpp:type:`ccf::kv::Version` | Read version | | +-----+------------------------------------+-------------------------------------------------------------------------+ | | | uint64_t | Read count | | | +------------------------------------+-------------------------------------------------------------------------+ diff --git a/doc/build_apps/api.rst b/doc/build_apps/api.rst index f89b5ef8068e..7b944c5ede28 100644 --- a/doc/build_apps/api.rst +++ b/doc/build_apps/api.rst @@ -10,7 +10,7 @@ A CCF application is composed of the following: Application Entry Point ----------------------- -.. doxygenfunction:: ccfapp::make_user_endpoints +.. doxygenfunction:: ccf::make_user_endpoints :project: CCF @@ -72,6 +72,9 @@ Policies .. doxygenvariable:: ccf::jwt_auth_policy :project: CCF +.. doxygenvariable:: ccf::TypedUserCOSESign1AuthnPolicy + :project: CCF + Identities ~~~~~~~~~~ @@ -122,7 +125,7 @@ Historical Queries .. doxygenclass:: ccf::historical::AbstractStateCache :project: CCF - :members: set_default_expiry_duration, get_state_at, get_store_at, get_store_range, drop_cached_states + :members: set_default_expiry_duration, set_soft_cache_limit, get_state_at, get_store_at, get_store_range, drop_cached_states .. doxygenstruct:: ccf::historical::State :project: CCF @@ -146,7 +149,7 @@ Indexing JavaScript FFI Plugins ---------------------- -.. doxygenfunction:: ccfapp::get_js_plugins +.. doxygenfunction:: ccf::get_js_plugins :project: CCF HTTP Entity Tags Matching diff --git a/doc/build_apps/crypto.rst b/doc/build_apps/crypto.rst index 8d6b4512ab8c..16229e8f4ea1 100644 --- a/doc/build_apps/crypto.rst +++ b/doc/build_apps/crypto.rst @@ -8,17 +8,17 @@ For convenience, CCF provides access to commonly used cryptographic primitives t Hashing ------- -.. doxygenfunction:: crypto::sha256(const std::vector &data) +.. doxygenfunction:: ccf::crypto::sha256(const std::vector &data) :project: CCF -.. doxygenfunction:: crypto::hmac(MDType, const std::vector&, const std::vector&) +.. doxygenfunction:: ccf::crypto::hmac(MDType, const std::vector&, const std::vector&) :project: CCF -.. doxygenClass:: crypto::HashProvider +.. doxygenClass:: ccf::crypto::HashProvider :project: CCF :members: -.. doxygenfunction:: crypto::make_hash_provider +.. doxygenfunction:: ccf::crypto::make_hash_provider :project: CCF @@ -27,49 +27,49 @@ Asymmetric Keys CCF supports EC and RSA keys; public keys are held in (RSA)PublicKey objects and private keys in (RSA)KeyPair objects. (RSA)KeyPairs automatically generate random -keys when constructed via :cpp:func:`KeyPairPtr crypto::make_key_pair(CurveID)` or -:cpp:func:`RSAKeyPairPtr crypto::make_rsa_key_pair(size_t, size_t)`. +keys when constructed via :cpp:func:`KeyPairPtr ccf::crypto::make_key_pair(CurveID)` or +:cpp:func:`RSAKeyPairPtr ccf::crypto::make_rsa_key_pair(size_t, size_t)`. -.. doxygenclass:: crypto::PublicKey +.. doxygenclass:: ccf::crypto::PublicKey :project: CCF :members: -.. doxygenclass:: crypto::KeyPair +.. doxygenclass:: ccf::crypto::KeyPair :project: CCF :members: -.. doxygenclass:: crypto::RSAPublicKey +.. doxygenclass:: ccf::crypto::RSAPublicKey :project: CCF :members: -.. doxygenclass:: crypto::RSAKeyPair +.. doxygenclass:: ccf::crypto::RSAKeyPair :project: CCF :members: -.. doxygenenum:: crypto::CurveID +.. doxygenenum:: ccf::crypto::CurveID :project: CCF -.. doxygenfunction:: crypto::make_key_pair(CurveID) +.. doxygenfunction:: ccf::crypto::make_key_pair(CurveID) :project: CCF -.. doxygenfunction:: crypto::make_key_pair(const Pem&) +.. doxygenfunction:: ccf::crypto::make_key_pair(const Pem&) :project: CCF -.. doxygenfunction:: crypto::make_rsa_key_pair(size_t, size_t) +.. doxygenfunction:: ccf::crypto::make_rsa_key_pair(size_t, size_t) :project: CCF Symmetric Keys -------------------- -Currently, only AES-GCM is supported for symmetric encryption. New keys are generated via :cpp:func:`crypto::Entropy::random` +Currently, only AES-GCM is supported for symmetric encryption. New keys are generated via :cpp:func:`ccf::crypto::Entropy::random` -.. doxygenfunction:: crypto::aes_gcm_encrypt +.. doxygenfunction:: ccf::crypto::aes_gcm_encrypt :project: CCF -.. doxygenfunction:: crypto::aes_gcm_decrypt +.. doxygenfunction:: ccf::crypto::aes_gcm_decrypt :project: CCF -.. doxygenclass:: crypto::Entropy +.. doxygenclass:: ccf::crypto::Entropy :project: CCF :members: @@ -78,7 +78,7 @@ Signatures Verification of signatures is supported via the :cpp:class:`Verifier` class. -.. doxygenclass:: crypto::Verifier +.. doxygenclass:: ccf::crypto::Verifier :project: CCF :members: @@ -88,36 +88,36 @@ Key Wrapping PKCS11 2.1.8 CKM_RSA_PKCS_OAEP -.. doxygenfunction:: crypto::ckm_rsa_pkcs_oaep_wrap(RSAPublicKeyPtr, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_pkcs_oaep_wrap(RSAPublicKeyPtr, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_pkcs_oaep_wrap(const Pem&, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_pkcs_oaep_wrap(const Pem&, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_pkcs_oaep_unwrap(RSAKeyPairPtr, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_pkcs_oaep_unwrap(RSAKeyPairPtr, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_pkcs_oaep_unwrap(const Pem&, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_pkcs_oaep_unwrap(const Pem&, const std::vector&, const std::optional>&) :project: CCF PKCS11 2.14.3 CKM_AES_KEY_WRAP_PAD (RFC 5649) -.. doxygenfunction:: crypto::ckm_aes_key_wrap_pad +.. doxygenfunction:: ccf::crypto::ckm_aes_key_wrap_pad :project: CCF -.. doxygenfunction:: crypto::ckm_aes_key_unwrap_pad +.. doxygenfunction:: ccf::crypto::ckm_aes_key_unwrap_pad :project: CCF PKCS11 2.1.21 CKM_RSA_AES_KEY_WRAP -.. doxygenfunction:: crypto::ckm_rsa_aes_key_wrap(size_t, RSAPublicKeyPtr, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_aes_key_wrap(size_t, RSAPublicKeyPtr, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_aes_key_wrap(size_t, const Pem&, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_aes_key_wrap(size_t, const Pem&, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_aes_key_unwrap(RSAKeyPairPtr, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_aes_key_unwrap(RSAKeyPairPtr, const std::vector&, const std::optional>&) :project: CCF -.. doxygenfunction:: crypto::ckm_rsa_aes_key_unwrap(const Pem&, const std::vector&, const std::optional>&) +.. doxygenfunction:: ccf::crypto::ckm_rsa_aes_key_unwrap(const Pem&, const std::vector&, const std::optional>&) :project: CCF \ No newline at end of file diff --git a/doc/build_apps/example_cpp.rst b/doc/build_apps/example_cpp.rst index 9cc5f748bde2..cf06f4b352d0 100644 --- a/doc/build_apps/example_cpp.rst +++ b/doc/build_apps/example_cpp.rst @@ -19,7 +19,7 @@ The Logging example application simply has: .. note:: - :cpp:type:`kv::Map` tables are the only interface between CCF and the replicated application, and the sole mechanism for it to have distributed state. + :cpp:type:`ccf::kv::Map` tables are the only interface between CCF and the replicated application, and the sole mechanism for it to have distributed state. The Logging application keeps its state in a pair of tables, one containing private encrypted logs and the other containing public unencrypted logs. Their type is defined as: @@ -46,7 +46,7 @@ The Logging example application simply has: Application Endpoints ~~~~~~~~~~~~~~~~~~~~~ -The implementation of :cpp:func:`ccfapp::make_user_endpoints()` should return a subclass of :cpp:class:`ccf::endpoints::EndpointRegistry`, containing the endpoints that constitute the app. +The implementation of :cpp:func:`ccf::make_user_endpoints()` should return a subclass of :cpp:class:`ccf::endpoints::EndpointRegistry`, containing the endpoints that constitute the app. .. literalinclude:: ../../samples/apps/logging/logging.cpp :language: cpp @@ -54,7 +54,7 @@ The implementation of :cpp:func:`ccfapp::make_user_endpoints()` should return a :lines: 1 :dedent: -The logging app defines :cpp:class:`ccfapp::LoggerHandlers`, which creates and installs handler functions or lambdas for several different HTTP endpoints. Each of these functions takes as input the details of the current request (such as the URI which was called, the query string, the request body), interacts with the KV tables using the given :cpp:class:`kv::Tx` object, and returns a result: +The logging app defines :cpp:class:`ccf::LoggerHandlers`, which creates and installs handler functions or lambdas for several different HTTP endpoints. Each of these functions takes as input the details of the current request (such as the URI which was called, the query string, the request body), interacts with the KV tables using the given :cpp:class:`ccf::kv::Tx` object, and returns a result: .. literalinclude:: ../../samples/apps/logging/logging.cpp :language: cpp diff --git a/doc/build_apps/kv/api.rst b/doc/build_apps/kv/api.rst index e498bcb6d4fc..7bcfba652fbb 100644 --- a/doc/build_apps/kv/api.rst +++ b/doc/build_apps/kv/api.rst @@ -3,84 +3,84 @@ Key-Value Store API This page presents the API that a CCF application must use to access and mutate the replicated key-value store. -A CCF application should store its data in one or more :cpp:type:`kv::Map`. The name, type, and serialisation of these maps is under the application's control. Each invocation of an :cpp:class:`ccf::EndpointRegistry::Endpoint` is given a :cpp:class:`kv::Tx` transaction object, through which it can read and write to its :cpp:type:`kv::Map`. +A CCF application should store its data in one or more :cpp:type:`ccf::kv::Map`. The name, type, and serialisation of these maps is under the application's control. Each invocation of an :cpp:class:`ccf::EndpointRegistry::Endpoint` is given a :cpp:class:`ccf::kv::Tx` transaction object, through which it can read and write to its :cpp:type:`ccf::kv::Map`. Map --- -.. doxygentypedef:: kv::Version +.. doxygentypedef:: ccf::kv::Version :project: CCF -.. doxygenvariable:: kv::NoVersion +.. doxygenvariable:: ccf::kv::NoVersion :project: CCF -.. doxygenclass:: kv::TypedMap +.. doxygenclass:: ccf::kv::TypedMap :project: CCF -.. doxygentypedef:: kv::Map +.. doxygentypedef:: ccf::kv::Map :project: CCF -.. doxygenclass:: kv::TypedValue +.. doxygenclass:: ccf::kv::TypedValue :project: CCF -.. doxygentypedef:: kv::Value +.. doxygentypedef:: ccf::kv::Value :project: CCF -.. doxygenclass:: kv::TypedSet +.. doxygenclass:: ccf::kv::TypedSet :project: CCF -.. doxygentypedef:: kv::Set +.. doxygentypedef:: ccf::kv::Set :project: CCF Transaction ----------- -.. doxygenclass:: kv::ReadOnlyTx +.. doxygenclass:: ccf::kv::ReadOnlyTx :project: CCF :members: ro -.. doxygenclass:: kv::Tx +.. doxygenclass:: ccf::kv::Tx :project: CCF :members: rw, wo Handles ------- -.. doxygenclass:: kv::ReadableMapHandle +.. doxygenclass:: ccf::kv::ReadableMapHandle :project: CCF :members: -.. doxygenclass:: kv::WriteableMapHandle +.. doxygenclass:: ccf::kv::WriteableMapHandle :project: CCF :members: -.. doxygenclass:: kv::MapHandle +.. doxygenclass:: ccf::kv::MapHandle :project: CCF -.. doxygenclass:: kv::ReadableValueHandle +.. doxygenclass:: ccf::kv::ReadableValueHandle :project: CCF :members: -.. doxygenclass:: kv::WriteableValueHandle +.. doxygenclass:: ccf::kv::WriteableValueHandle :project: CCF :members: -.. doxygenclass:: kv::ValueHandle +.. doxygenclass:: ccf::kv::ValueHandle :project: CCF -.. doxygenclass:: kv::ReadableSetHandle +.. doxygenclass:: ccf::kv::ReadableSetHandle :project: CCF :members: -.. doxygenclass:: kv::WriteableSetHandle +.. doxygenclass:: ccf::kv::WriteableSetHandle :project: CCF :members: -.. doxygenclass:: kv::SetHandle +.. doxygenclass:: ccf::kv::SetHandle :project: CCF Serialisation ------------- -.. doxygenenum:: kv::EntryType +.. doxygenenum:: ccf::kv::EntryType :project: CCF \ No newline at end of file diff --git a/doc/build_apps/kv/kv_how_to.rst b/doc/build_apps/kv/kv_how_to.rst index f97570189602..1667454fac06 100644 --- a/doc/build_apps/kv/kv_how_to.rst +++ b/doc/build_apps/kv/kv_how_to.rst @@ -1,70 +1,70 @@ Key-Value Store How-To ====================== -The `Key-Value Store` (KV) consists of a set of :cpp:type:`kv::Map` objects that are available to the endpoints of an application. Endpoint handlers create handles which allow them to read and write from these :cpp:type:`kv::Map` objects. The framework handles conflicts between concurrent execution of multiple transactions, and produces a consistent order of transactions which is replicated between nodes, allowing the entries to be read from multiple nodes. This page outlines the core concepts and C++ APIs used to interact with the KV. +The `Key-Value Store` (KV) consists of a set of :cpp:type:`ccf::kv::Map` objects that are available to the endpoints of an application. Endpoint handlers create handles which allow them to read and write from these :cpp:type:`ccf::kv::Map` objects. The framework handles conflicts between concurrent execution of multiple transactions, and produces a consistent order of transactions which is replicated between nodes, allowing the entries to be read from multiple nodes. This page outlines the core concepts and C++ APIs used to interact with the KV. Map Naming ---------- -A :cpp:type:`kv::Map` (often referred to as a `Table`) is a collection of key-value pairs of a given type. The :cpp:type:`kv::Map` itself is identified by its name, which is used to lookup the map :cpp:type:`kv::Map` from the local store during a transaction. +A :cpp:type:`ccf::kv::Map` (often referred to as a `Table`) is a collection of key-value pairs of a given type. The :cpp:type:`ccf::kv::Map` itself is identified by its name, which is used to lookup the map :cpp:type:`ccf::kv::Map` from the local store during a transaction. -If a :cpp:type:`kv::Map` with the given name did not previously exist, it will be created in this transaction. +If a :cpp:type:`ccf::kv::Map` with the given name did not previously exist, it will be created in this transaction. -A :cpp:type:`kv::Map` can either be created as private (default) or public. Public map's names begin with a ``public:`` prefix, any any other name indicates a private map. For instance the name ``public:foo`` to a public map, while ``foo`` refers to a private map. Transactions on private maps are written to the ledger in encrypted form and can only be decrypted in the enclave of nodes that have joined the network. Transactions on public maps are written to the ledger as plaintext and can be read from outside the enclave; only their integrity is protected. The security domain of a map (public or private) cannot be changed after its creation, since this is encoded in the map's name. Public and private maps with similar names in different domains are distinct; writes to ``public:foo`` have no impact on ``foo``, and vice versa. +A :cpp:type:`ccf::kv::Map` can either be created as private (default) or public. Public map's names begin with a ``public:`` prefix, any any other name indicates a private map. For instance the name ``public:foo`` to a public map, while ``foo`` refers to a private map. Transactions on private maps are written to the ledger in encrypted form and can only be decrypted in the enclave of nodes that have joined the network. Transactions on public maps are written to the ledger as plaintext and can be read from outside the enclave; only their integrity is protected. The security domain of a map (public or private) cannot be changed after its creation, since this is encoded in the map's name. Public and private maps with similar names in different domains are distinct; writes to ``public:foo`` have no impact on ``foo``, and vice versa. Some table names are reserved for governance of a CCF service and cannot be modified by application code. For more information, see :doc:`/audit/read_write_restrictions`. Transaction Semantics --------------------- -A transaction (:cpp:class:`kv::Tx`) encapsulates an individual endpoint invocation's atomic interaction with the KV. Transactions may read from and write to multiple :cpp:type:`kv::Map`, and each is automatically ordered, applied, serialised, and committed by the framework. +A transaction (:cpp:class:`ccf::kv::Tx`) encapsulates an individual endpoint invocation's atomic interaction with the KV. Transactions may read from and write to multiple :cpp:type:`ccf::kv::Map`, and each is automatically ordered, applied, serialised, and committed by the framework. -A reference to a new :cpp:class:`kv::Tx` is passed to each endpoint handler, and used to interact with the KV. +A reference to a new :cpp:class:`ccf::kv::Tx` is passed to each endpoint handler, and used to interact with the KV. -Each :cpp:class:`kv::Tx` gets a consistent, opaque view of the KV, including the values which have been left by previous writes. Any writes produced by this transaction will be visible to all future transactions. +Each :cpp:class:`ccf::kv::Tx` gets a consistent, opaque view of the KV, including the values which have been left by previous writes. Any writes produced by this transaction will be visible to all future transactions. -When the endpoint handler indicates that its :cpp:class:`kv::Tx` should be applied (see :ref:`this section ` for details), the executing node attempts to apply the changes to its local KV. If this produces conflicts with concurrently executing transactions, it will be automatically re-executed. Once the transaction is applied successfully, it is automatically replicated to other nodes and will, if the network is healthy, eventually be committed. +When the endpoint handler indicates that its :cpp:class:`ccf::kv::Tx` should be applied (see :ref:`this section ` for details), the executing node attempts to apply the changes to its local KV. If this produces conflicts with concurrently executing transactions, it will be automatically re-executed. Once the transaction is applied successfully, it is automatically replicated to other nodes and will, if the network is healthy, eventually be committed. -For each :cpp:type:`kv::Map` that a transaction wants to write to or read from, a :cpp:class:`kv::MapHandle` must first be acquired. These are acquired from the :cpp:func:`kv::Tx::rw` (`read-write`) method. These may be acquired either by name (in which case the desired type must be explicitly specified as a template parameter), or by using a :cpp:type:`kv::Map` instance which defines both the map's name and key-value types. +For each :cpp:type:`ccf::kv::Map` that a transaction wants to write to or read from, a :cpp:class:`ccf::kv::MapHandle` must first be acquired. These are acquired from the :cpp:func:`ccf::kv::Tx::rw` (`read-write`) method. These may be acquired either by name (in which case the desired type must be explicitly specified as a template parameter), or by using a :cpp:type:`ccf::kv::Map` instance which defines both the map's name and key-value types. By name: .. code-block:: cpp // Handle for map1 - auto map1_handle = tx.rw>("map1"); + auto map1_handle = tx.rw>("map1"); // Handles for 2 other maps, one public and one private, with different types - auto map2_handle = tx.rw>("public:map2"); - auto map3_handle = tx.rw>("map3"); + auto map2_handle = tx.rw>("public:map2"); + auto map3_handle = tx.rw>("map3"); -By :cpp:type:`kv::Map`: +By :cpp:type:`ccf::kv::Map`: .. code-block:: cpp - kv::Map map_priv("map1"); + ccf::kv::Map map_priv("map1"); auto map1_handle = tx.rw(map_priv); - kv::Map map_pub("public:map2"); + ccf::kv::Map map_pub("public:map2"); auto map2_handle = tx.rw(map_pub); - kv::Map map_priv_int("map3"); + ccf::kv::Map map_priv_int("map3"); auto map3_handle = tx.rw(map_priv_int); The latter approach introduces a named binding between the map's name and the types of its keys and values, reducing the chance for errors where code attempts to read a map with the wrong type. -.. note:: As mentioned above, there is no need to explicitly declare a :cpp:type:`kv::Map` before it is used. The first write to a :cpp:type:`kv::Map` implicitly creates it in the underlying KV. Within a transaction, a newly created :cpp:type:`kv::Map` behaves exactly the same as an existing :cpp:type:`kv::Map` with no keys - the framework views these as semantically identical, and offers no way for the application logic to tell them apart. Any writes to a newly created :cpp:type:`kv::Map` will be persisted when the transaction commits, and future transactions will be able to access this :cpp:type:`kv::Map` by name to read those writes. +.. note:: As mentioned above, there is no need to explicitly declare a :cpp:type:`ccf::kv::Map` before it is used. The first write to a :cpp:type:`ccf::kv::Map` implicitly creates it in the underlying KV. Within a transaction, a newly created :cpp:type:`ccf::kv::Map` behaves exactly the same as an existing :cpp:type:`ccf::kv::Map` with no keys - the framework views these as semantically identical, and offers no way for the application logic to tell them apart. Any writes to a newly created :cpp:type:`ccf::kv::Map` will be persisted when the transaction commits, and future transactions will be able to access this :cpp:type:`ccf::kv::Map` by name to read those writes. Accessing Map content via a Handle ---------------------------------- -Once a :cpp:class:`kv::MapHandle` on a specific :cpp:type:`kv::Map` has been obtained, it is possible to: +Once a :cpp:class:`ccf::kv::MapHandle` on a specific :cpp:type:`ccf::kv::Map` has been obtained, it is possible to: -- test (:cpp:func:`kv::ReadableMapHandle::has`) whether a key has any associated value; -- read (:cpp:func:`kv::ReadableMapHandle::get`) the value associated with a key; -- write (:cpp:func:`kv::WriteableMapHandle::put`) a new value for a key; -- delete (:cpp:func:`kv::WriteableMapHandle::remove`) a key and its current value; -- iterate (:cpp:func:`kv::ReadableMapHandle::foreach`) through all key-value pairs. +- test (:cpp:func:`ccf::kv::ReadableMapHandle::has`) whether a key has any associated value; +- read (:cpp:func:`ccf::kv::ReadableMapHandle::get`) the value associated with a key; +- write (:cpp:func:`ccf::kv::WriteableMapHandle::put`) a new value for a key; +- delete (:cpp:func:`ccf::kv::WriteableMapHandle::remove`) a key and its current value; +- iterate (:cpp:func:`ccf::kv::ReadableMapHandle::foreach`) through all key-value pairs. .. code-block:: cpp @@ -91,7 +91,7 @@ Once a :cpp:class:`kv::MapHandle` on a specific :cpp:type:`kv::Map` has been obt Read/Write safety ----------------- -If you are only reading from or only writing to a given :cpp:type:`kv::Map` you can retrieve a `read-only` or `write-only` handle for it. This will turn unexpected reads/writes (which would introduce unintended dependencies between transactions) into compile-time errors. Instead of calling :cpp:func:`kv::Tx::rw` to get a handle which can both read and write, you can call :cpp:func:`kv::ReadOnlyTx::ro` to acquire a `read-only` handle or :cpp:func:`kv::Tx::wo` to acquire a `write-only` handle. +If you are only reading from or only writing to a given :cpp:type:`ccf::kv::Map` you can retrieve a `read-only` or `write-only` handle for it. This will turn unexpected reads/writes (which would introduce unintended dependencies between transactions) into compile-time errors. Instead of calling :cpp:func:`ccf::kv::Tx::rw` to get a handle which can both read and write, you can call :cpp:func:`ccf::kv::ReadOnlyTx::ro` to acquire a `read-only` handle or :cpp:func:`ccf::kv::Tx::wo` to acquire a `write-only` handle. .. code-block:: cpp @@ -122,7 +122,7 @@ Note that, as in the sample above, it is possible to acquire different kinds of Removing a key -------------- -If a Key-Value pair was written to a :cpp:type:`kv::Map` by a previous :cpp:class:`kv::Tx`, it is possible to delete this key. Because of the append-only nature of the KV, this Key-Value pair is not actually removed from the :cpp:type:`kv::Map` but instead explicitly marked as deleted in the version that the deleting :cpp:class:`kv::Tx` is applied at. +If a Key-Value pair was written to a :cpp:type:`ccf::kv::Map` by a previous :cpp:class:`ccf::kv::Tx`, it is possible to delete this key. Because of the append-only nature of the KV, this Key-Value pair is not actually removed from the :cpp:type:`ccf::kv::Map` but instead explicitly marked as deleted in the version that the deleting :cpp:class:`ccf::kv::Tx` is applied at. .. code-block:: cpp @@ -141,7 +141,7 @@ Global commit A transaction is automatically (globally) committed once the consensus protocol has established that a majority of nodes in the CCF network have successfully received and acknowledged that transaction. To operate on durable state, an application may want to query the globally committed state rather than the *current* state of the KV. -The :cpp:func:`kv::MapHandle::get_globally_committed` member function returns the value of a key that we know has been globally committed. +The :cpp:func:`ccf::kv::MapHandle::get_globally_committed` member function returns the value of a key that we know has been globally committed. .. code-block:: cpp @@ -166,9 +166,9 @@ Miscellaneous ``foreach()`` ~~~~~~~~~~~~~ -Values can only be retrieved directly (:cpp:func:`kv::MapHandle::get`) for a given target key. However, it is sometimes necessary to access unknown keys, or to iterate through all Key-Value pairs. +Values can only be retrieved directly (:cpp:func:`ccf::kv::MapHandle::get`) for a given target key. However, it is sometimes necessary to access unknown keys, or to iterate through all Key-Value pairs. -CCF offers a member function :cpp:func:`kv::MapHandle::foreach` to iterate over all the elements written to that :cpp:type:`kv::Map` so far, and run a lambda function for each Key-Value pair. Note that a :cpp:class:`kv::MapHandle::foreach` loop can be ended early by returning ``false`` from this lambda, while ``true`` should be returned to continue iteration. +CCF offers a member function :cpp:func:`ccf::kv::MapHandle::foreach` to iterate over all the elements written to that :cpp:type:`ccf::kv::Map` so far, and run a lambda function for each Key-Value pair. Note that a :cpp:class:`ccf::kv::MapHandle::foreach` loop can be ended early by returning ``false`` from this lambda, while ``true`` should be returned to continue iteration. .. code-block:: cpp @@ -192,7 +192,7 @@ CCF offers a member function :cpp:func:`kv::MapHandle::foreach` to iterate over Applying and reverting writes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Changes to the KV are made by atomic transactions. For a given :cpp:class:`kv::Tx`, either all of its writes are applied, or none are. Only applied writes are replicated and may be globally committed. Transactions may be abandoned without applying their writes - their changes will never be seen by other transactions. +Changes to the KV are made by atomic transactions. For a given :cpp:class:`ccf::kv::Tx`, either all of its writes are applied, or none are. Only applied writes are replicated and may be globally committed. Transactions may be abandoned without applying their writes - their changes will never be seen by other transactions. By default CCF decides which transactions are successful (so should be applied to the persistent store) by looking at the status code contained in the response: all transactions producing ``2xx`` status codes will be applied, while any other status code will be treated as an error and will `not` be applied to the persistent store. If this behaviour is not desired, for instance when an app wants to log incoming requests even though they produce an error, then it can be dynamically overridden by explicitly telling CCF whether it should apply a given transaction: diff --git a/doc/build_apps/kv/kv_serialisation.rst b/doc/build_apps/kv/kv_serialisation.rst index ace926c4d064..9f2c0aa69cd8 100644 --- a/doc/build_apps/kv/kv_serialisation.rst +++ b/doc/build_apps/kv/kv_serialisation.rst @@ -1,14 +1,14 @@ Key-Value Serialisation ======================= -Every transaction executed by the primary on its Key-Value store is serialised before being replicated to all backups of the CCF network and written to the ledger. The serialisation format is defined per :cpp:type:`kv::Map` and distinctly for the key and value types. +Every transaction executed by the primary on its Key-Value store is serialised before being replicated to all backups of the CCF network and written to the ledger. The serialisation format is defined per :cpp:type:`ccf::kv::Map` and distinctly for the key and value types. -.. tip:: Selecting the right serialision format for a KV map depends on the application logic but is generally a trade-off between performance and auditability of the ledger. For example, the default serialisation format for :cpp:type:`kv::Map` is JSON and allows for easy parsing of transactions in the `public` ledger. For more performance sensitive use cases, apps may define or use their own serialisers. +.. tip:: Selecting the right serialision format for a KV map depends on the application logic but is generally a trade-off between performance and auditability of the ledger. For example, the default serialisation format for :cpp:type:`ccf::kv::Map` is JSON and allows for easy parsing of transactions in the `public` ledger. For more performance sensitive use cases, apps may define or use their own serialisers. Custom key and value types -------------------------- -User-defined types can be used for both the key and value types of a :cpp:type:`kv::Map`. It must be possible to use the key type as the key of an ``std::map`` (so it must be copyable, assignable, and less-comparable), and both types must be serialisable. By default, when using a :cpp:type:`kv::Map`, serialisation converts to JSON. To add support to your custom types, it should usually be possible to use the ``DECLARE_JSON_...`` macros: +User-defined types can be used for both the key and value types of a :cpp:type:`ccf::kv::Map`. It must be possible to use the key type as the key of an ``std::map`` (so it must be copyable, assignable, and less-comparable), and both types must be serialisable. By default, when using a :cpp:type:`ccf::kv::Map`, serialisation converts to JSON. To add support to your custom types, it should usually be possible to use the ``DECLARE_JSON_...`` macros: .. literalinclude:: ../../../src/kv/test/kv_serialisation.cpp :language: cpp @@ -22,7 +22,7 @@ Custom serialisers can also be defined. The serialiser itself must be a type imp :start-after: SNIPPET_START: CustomSerialiser definition :end-before: SNIPPET_END: CustomSerialiser definition -To use these serialised for a specific map declare the map as a :cpp:class:`kv::TypedMap`, adding the appropriate serialiser types for the key and value types: +To use these serialised for a specific map declare the map as a :cpp:class:`ccf::kv::TypedMap`, adding the appropriate serialiser types for the key and value types: .. literalinclude:: ../../../src/kv/test/kv_serialisation.cpp :language: cpp diff --git a/doc/build_apps/migration_4_x_to_5_0.rst b/doc/build_apps/migration_4_x_to_5_0.rst index 23e8d467265f..5fe99db9c543 100644 --- a/doc/build_apps/migration_4_x_to_5_0.rst +++ b/doc/build_apps/migration_4_x_to_5_0.rst @@ -117,3 +117,48 @@ The default value for both is ``ToPrimary``/``"to_primary"``, meaning that all r - ``ToPrimary`` / ``"to_primary"`` While ``Never`` and ``Always`` have clear analogs in redirection, the session consistency-preserving ``Sometimes`` value is more complicated. All writes should be redirected to a primary, as attempting to execute them on a backup will result in an error. For reads, you may choose to redirect to retain simple consistency, but to support scaling (by reading on backups), we recommend you choose ``None`` for redirections. Where between-request consistency is a strong requirement, we recommend you enforce it at the application level (eg - ETags, request IDs, etc). + +ToBackup +~~~~~~~~ + +A third ``RedirectionStrategy`` exists named ``ToBackup`` (represented by ``"redirection_strategy": "to_backup"`` in ``app.json``). This is a mirror of the ``ToPrimary`` strategy - if such a request is processed by a node which is currently a primary, that node will produce a HTTP redirect response directing to a backup node. The choice of backup is arbitrary. The redirection address which is inserted can be configured per-node by the operator, in the ``redirections.to_backup`` object. + +For example, to redirect directly to a backup by their unique accessible hostname: + +.. code-block:: json + + { + "network": { + "rpc_interfaces": { + "interface_name": { + "redirections": { + "to_backup": { + "kind": "NodeByRole", + "target": { "role": "backup" } + } + } + } + } + } + } + +To redirect to a static address (such as a load balancer): + +.. code-block:: json + + { + "network": { + "rpc_interfaces": { + "interface_name": { + "redirections": { + "to_backup": { + "kind": "StaticAddress", + "target": { + "address": "backup.ccf.example.com" + } + } + } + } + } + } + } diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json index d273433048eb..c9993ee7d1f0 100644 --- a/doc/host_config_schema/cchost_config.json +++ b/doc/host_config_schema/cchost_config.json @@ -169,6 +169,10 @@ "to_primary": { "$ref": "#/$defs/RedirectionResolver", "description": "Configures how the Location header should be populated, when requests arrive on this interface that must be served by a primary while the receiving node is not a primary" + }, + "to_backup": { + "$ref": "#/$defs/RedirectionResolver", + "description": "Configures how the Location header should be populated, when requests arrive on this interface that must be served by a backup while the receiving node is currently a primary" } }, "additionalProperties": false @@ -724,7 +728,7 @@ "type": "object", "properties": { "role": { - "enum": ["primary"], + "enum": ["primary", "backup"], "default": "primary" } }, diff --git a/doc/overview/glossary.rst b/doc/overview/glossary.rst index ebdc7e015329..72016a58b34a 100644 --- a/doc/overview/glossary.rst +++ b/doc/overview/glossary.rst @@ -79,7 +79,7 @@ Glossary `Transport Layer Security `_ is an IETF cryptographic protocol standard designed to secure communications between a client and a server over a computer network. Transaction ID - Unique transaction identifier in CCF, composed of a View and a Sequence Number separated by a period. Sequence Numbers start from 1, and are contiguous. Views are monotonic. E.g. The transaction ID ``2.15`` indicates the View is ``2`` and the Sequence Number is ``15``. Sequence Numbers are also referred to as a :cpp:type:`kv::Version` in the context of the Key-Value store. + Unique transaction identifier in CCF, composed of a View and a Sequence Number separated by a period. Sequence Numbers start from 1, and are contiguous. Views are monotonic. E.g. The transaction ID ``2.15`` indicates the View is ``2`` and the Sequence Number is ``15``. Sequence Numbers are also referred to as a :cpp:type:`ccf::kv::Version` in the context of the Key-Value store. Users Directly interact with the application running in CCF. Their public identity should be voted in by members before they are allowed to issue requests. diff --git a/doc/schemas/app_openapi.json b/doc/schemas/app_openapi.json index e711e6655b81..f87cd88370f6 100644 --- a/doc/schemas/app_openapi.json +++ b/doc/schemas/app_openapi.json @@ -648,6 +648,46 @@ } } }, + "/app/log/private/backup": { + "get": { + "parameters": [ + { + "in": "query", + "name": "id", + "required": true, + "schema": { + "$ref": "#/components/schemas/uint64" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LoggingGet__Out" + } + } + }, + "description": "Default response description" + }, + "default": { + "$ref": "#/components/responses/default" + } + }, + "security": [ + { + "jwt": [] + }, + { + "user_cose_sign1": [] + } + ], + "x-ccf-forwarding": { + "$ref": "#/components/x-ccf-forwarding/sometimes" + } + } + }, "/app/log/private/committed": { "get": { "parameters": [ @@ -1079,6 +1119,46 @@ } } }, + "/app/log/public/backup": { + "get": { + "parameters": [ + { + "in": "query", + "name": "id", + "required": true, + "schema": { + "$ref": "#/components/schemas/uint64" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LoggingGet__Out" + } + } + }, + "description": "Default response description" + }, + "default": { + "$ref": "#/components/responses/default" + } + }, + "security": [ + { + "jwt": [] + }, + { + "user_cose_sign1": [] + } + ], + "x-ccf-forwarding": { + "$ref": "#/components/x-ccf-forwarding/sometimes" + } + } + }, "/app/log/public/count": { "get": { "responses": { diff --git a/doc/schemas/gov_openapi.json b/doc/schemas/gov_openapi.json index 9ce989aa261c..855144deeb4a 100644 --- a/doc/schemas/gov_openapi.json +++ b/doc/schemas/gov_openapi.json @@ -645,6 +645,22 @@ }, "redirections": { "properties": { + "to_backup": { + "properties": { + "kind": { + "enum": [ + "NodeByRole", + "StaticAddress" + ], + "type": "string" + }, + "target": {} + }, + "required": [ + "kind" + ], + "type": "object" + }, "to_primary": { "properties": { "kind": { @@ -768,6 +784,22 @@ }, "redirections": { "properties": { + "to_backup": { + "properties": { + "kind": { + "enum": [ + "NodeByRole", + "StaticAddress" + ], + "type": "string" + }, + "target": {} + }, + "required": [ + "kind" + ], + "type": "object" + }, "to_primary": { "properties": { "kind": { @@ -968,7 +1000,8 @@ "RedirectionStrategy": { "enum": [ "none", - "to_primary" + "to_primary", + "to_backup" ], "type": "string" }, diff --git a/doc/schemas/node_openapi.json b/doc/schemas/node_openapi.json index 6ad3058a4009..53602354c9d4 100644 --- a/doc/schemas/node_openapi.json +++ b/doc/schemas/node_openapi.json @@ -552,6 +552,9 @@ }, "NodeInfoNetwork_v2__NetInterface__Redirections": { "properties": { + "to_backup": { + "$ref": "#/components/schemas/RedirectionResolverConfig" + }, "to_primary": { "$ref": "#/components/schemas/RedirectionResolverConfig" } diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 95dd5807741f..4362f14b0428 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -4,7 +4,7 @@ # Latest image as of this change ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang-15 +ARG base=ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang-15 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead diff --git a/getting_started/setup_vm/roles/openenclave/vars/common.yml b/getting_started/setup_vm/roles/openenclave/vars/common.yml index b02747bb80b9..1d7f01c20215 100644 --- a/getting_started/setup_vm/roles/openenclave/vars/common.yml +++ b/getting_started/setup_vm/roles/openenclave/vars/common.yml @@ -1,6 +1,6 @@ -oe_ver: "0.19.6" +oe_ver: "0.19.7" # Usually the same, except for rc, where ver is -rc and ver_ is _rc -oe_ver_: "0.19.6" +oe_ver_: "0.19.7" # Source install workspace: "/tmp/" diff --git a/include/ccf/app_interface.h b/include/ccf/app_interface.h index f5c9ed702f9e..03498766c1ce 100644 --- a/include/ccf/app_interface.h +++ b/include/ccf/app_interface.h @@ -17,7 +17,7 @@ namespace ccf class RpcFrontend; } -namespace kv +namespace ccf::kv { class Store; } @@ -27,7 +27,7 @@ namespace ccf class UserEndpointRegistry : public CommonEndpointRegistry { public: - UserEndpointRegistry(ccfapp::AbstractNodeContext& context) : + UserEndpointRegistry(ccf::AbstractNodeContext& context) : CommonEndpointRegistry(get_actor_prefix(ActorsType::users), context) {} @@ -48,7 +48,7 @@ namespace ccf }; } -namespace ccfapp +namespace ccf { // SNIPPET_START: app_interface /** To be implemented by the application. Creates a collection of endpoints @@ -59,7 +59,7 @@ namespace ccfapp * @return Unique pointer to the endpoint registry instance */ std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context); + ccf::AbstractNodeContext& context); /** To be implemented by the application. * diff --git a/include/ccf/base_endpoint_registry.h b/include/ccf/base_endpoint_registry.h index 17487562b145..a1999d251167 100644 --- a/include/ccf/base_endpoint_registry.h +++ b/include/ccf/base_endpoint_registry.h @@ -63,6 +63,47 @@ namespace ccf } } + /** Lists possible reasons for an ApiResult::InvalidArgs being return in @c + * ccf::BaseEndpointRegistry + */ + enum class InvalidArgsReason + { + NoReason = 0, + /** Views start at 1 (one) in CCF */ + ViewSmallerThanOne, + /** Action has already been applied on this instance */ + ActionAlreadyApplied, + /** Action created_at is older than the median of recent action */ + StaleActionCreatedTimestamp, + }; + + constexpr char const* invalid_args_reason_to_str(InvalidArgsReason reason) + { + switch (reason) + { + case InvalidArgsReason::NoReason: + { + return "NoReason"; + } + case InvalidArgsReason::ViewSmallerThanOne: + { + return "ViewSmallerThanOne"; + } + case InvalidArgsReason::ActionAlreadyApplied: + { + return "ActionAlreadyApplied"; + } + case InvalidArgsReason::StaleActionCreatedTimestamp: + { + return "StaleActionCreatedTimestamp"; + } + default: + { + return "Unhandled InvalidArgsReason"; + } + } + } + /** Extends the basic @ref ccf::endpoints::EndpointRegistry with helper API * methods for retrieving core CCF properties. * @@ -79,11 +120,11 @@ namespace ccf class BaseEndpointRegistry : public ccf::endpoints::EndpointRegistry { protected: - ccfapp::AbstractNodeContext& context; + ccf::AbstractNodeContext& context; public: BaseEndpointRegistry( - const std::string& method_prefix_, ccfapp::AbstractNodeContext& context_); + const std::string& method_prefix_, ccf::AbstractNodeContext& context_); /** Get the history of the consensus view changes. * @@ -96,6 +137,19 @@ namespace ccf ApiResult get_view_history_v1( std::vector& history, ccf::View since = 1); + /** Get the history of the consensus view changes. + * + * Returns the history of view changes since the given view, which defaults + * to the start of time. + * + * A view change is characterised by the first sequence number in the new + * view. + */ + ApiResult get_view_history_v2( + std::vector& history, + ccf::View since, + ccf::InvalidArgsReason& reason); + /** Get the status of a transaction by ID, provided as a view+seqno pair. * * Note that this value is the node's local understanding of the status @@ -134,7 +188,7 @@ namespace ccf * does not affect the OpenAPI version of the format of the document. */ ApiResult generate_openapi_document_v1( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::string& title, const std::string& description, const std::string& document_version, @@ -143,7 +197,7 @@ namespace ccf /** Get a quote attesting to the hardware this node is running on. */ ApiResult get_quote_for_this_node_v1( - kv::ReadOnlyTx& tx, QuoteInfo& quote_info); + ccf::kv::ReadOnlyTx& tx, QuoteInfo& quote_info); /** Get the id of the currently executing node. */ @@ -153,7 +207,7 @@ namespace ccf * running on. */ ApiResult get_quotes_for_all_trusted_nodes_v1( - kv::ReadOnlyTx& tx, std::map& quotes); + ccf::kv::ReadOnlyTx& tx, std::map& quotes); /** Get the view associated with a given seqno, to construct a valid TxID. */ @@ -162,26 +216,30 @@ namespace ccf /** Get the user data associated with a given user id. */ ApiResult get_user_data_v1( - kv::ReadOnlyTx& tx, const UserId& user_id, nlohmann::json& user_data); + ccf::kv::ReadOnlyTx& tx, + const UserId& user_id, + nlohmann::json& user_data); /** Get the member data associated with a given member id. */ ApiResult get_member_data_v1( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const MemberId& member_id, nlohmann::json& member_data); /** Get the certificate (PEM) of a given user id. */ ApiResult get_user_cert_v1( - kv::ReadOnlyTx& tx, const UserId& user_id, crypto::Pem& user_cert_pem); + ccf::kv::ReadOnlyTx& tx, + const UserId& user_id, + ccf::crypto::Pem& user_cert_pem); /** Get the certificate (PEM) of a given member id. */ ApiResult get_member_cert_v1( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const MemberId& member_id, - crypto::Pem& member_cert_pem); + ccf::crypto::Pem& member_cert_pem); /** Get untrusted time from the host of the currently executing node. */ diff --git a/include/ccf/claims_digest.h b/include/ccf/claims_digest.h index 6eb6de405f4f..005c9707f1ba 100644 --- a/include/ccf/claims_digest.h +++ b/include/ccf/claims_digest.h @@ -9,7 +9,7 @@ namespace ccf class ClaimsDigest { public: - using Digest = crypto::Sha256Hash; + using Digest = ccf::crypto::Sha256Hash; private: bool is_set = false; diff --git a/include/ccf/common_endpoint_registry.h b/include/ccf/common_endpoint_registry.h index a9dffa4c7feb..377baf311628 100644 --- a/include/ccf/common_endpoint_registry.h +++ b/include/ccf/common_endpoint_registry.h @@ -16,7 +16,7 @@ namespace ccf { public: CommonEndpointRegistry( - const std::string& method_prefix_, ccfapp::AbstractNodeContext& context_); + const std::string& method_prefix_, ccf::AbstractNodeContext& context_); void init_handlers() override; diff --git a/include/ccf/crypto/base64.h b/include/ccf/crypto/base64.h index 19175df8106f..f145ee770772 100644 --- a/include/ccf/crypto/base64.h +++ b/include/ccf/crypto/base64.h @@ -5,7 +5,7 @@ #include #include -namespace crypto +namespace ccf::crypto { std::vector raw_from_b64(const std::string_view& b64_string); diff --git a/include/ccf/crypto/cose_verifier.h b/include/ccf/crypto/cose_verifier.h index 0d30c270a413..e4d403661912 100644 --- a/include/ccf/crypto/cose_verifier.h +++ b/include/ccf/crypto/cose_verifier.h @@ -9,7 +9,7 @@ #include -namespace crypto +namespace ccf::crypto { class COSEVerifier { diff --git a/include/ccf/crypto/curve.h b/include/ccf/crypto/curve.h index 77151db53bba..088dd2c6946d 100644 --- a/include/ccf/crypto/curve.h +++ b/include/ccf/crypto/curve.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { // SNIPPET_START: supported_curves enum class CurveID diff --git a/include/ccf/crypto/ecdsa.h b/include/ccf/crypto/ecdsa.h index 57066a9510e7..e61b3ad24b74 100644 --- a/include/ccf/crypto/ecdsa.h +++ b/include/ccf/crypto/ecdsa.h @@ -6,7 +6,7 @@ #include -namespace crypto +namespace ccf::crypto { /** Converts R and S signature parameters to RFC 3279 DER * encoding. diff --git a/include/ccf/crypto/eddsa_key_pair.h b/include/ccf/crypto/eddsa_key_pair.h index 3eab20ee690f..d1e7008a4000 100644 --- a/include/ccf/crypto/eddsa_key_pair.h +++ b/include/ccf/crypto/eddsa_key_pair.h @@ -16,7 +16,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class EdDSAKeyPair { diff --git a/include/ccf/crypto/eddsa_public_key.h b/include/ccf/crypto/eddsa_public_key.h index 588777aa433d..628fae63f7b6 100644 --- a/include/ccf/crypto/eddsa_public_key.h +++ b/include/ccf/crypto/eddsa_public_key.h @@ -10,7 +10,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class EdDSAPublicKey { diff --git a/include/ccf/crypto/entropy.h b/include/ccf/crypto/entropy.h index 7641627d941f..33d03ca638a4 100644 --- a/include/ccf/crypto/entropy.h +++ b/include/ccf/crypto/entropy.h @@ -27,7 +27,7 @@ // issue.` #define RDRAND_RETRIES 10 -namespace crypto +namespace ccf::crypto { using rng_func_t = int (*)(void* ctx, unsigned char* output, size_t len); diff --git a/include/ccf/crypto/hash_bytes.h b/include/ccf/crypto/hash_bytes.h index 015156a5f075..939ee3a765df 100644 --- a/include/ccf/crypto/hash_bytes.h +++ b/include/ccf/crypto/hash_bytes.h @@ -5,7 +5,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using HashBytes = std::vector; } \ No newline at end of file diff --git a/include/ccf/crypto/hash_provider.h b/include/ccf/crypto/hash_provider.h index 55c5f425e0c3..c767fe8438d7 100644 --- a/include/ccf/crypto/hash_provider.h +++ b/include/ccf/crypto/hash_provider.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class HashProvider { diff --git a/include/ccf/crypto/hkdf.h b/include/ccf/crypto/hkdf.h index d2b2a438c86a..47fcb8da00a2 100644 --- a/include/ccf/crypto/hkdf.h +++ b/include/ccf/crypto/hkdf.h @@ -7,7 +7,7 @@ #include #include -namespace crypto +namespace ccf::crypto { /** Perform HKDF key derivation */ std::vector hkdf( diff --git a/include/ccf/crypto/hmac.h b/include/ccf/crypto/hmac.h index d96da1aec1d5..8efe0896c506 100644 --- a/include/ccf/crypto/hmac.h +++ b/include/ccf/crypto/hmac.h @@ -5,7 +5,7 @@ #include "ccf/crypto/hash_bytes.h" #include "ccf/crypto/md_type.h" -namespace crypto +namespace ccf::crypto { /** Compute the HMAC of @p key and @p data */ diff --git a/include/ccf/crypto/jwk.h b/include/ccf/crypto/jwk.h index 42556572a4d0..926740a4eb4e 100644 --- a/include/ccf/crypto/jwk.h +++ b/include/ccf/crypto/jwk.h @@ -8,7 +8,7 @@ #include -namespace crypto +namespace ccf::crypto { enum class JsonWebKeyType { diff --git a/include/ccf/crypto/key_pair.h b/include/ccf/crypto/key_pair.h index c8dae8e27a12..cda9710be53e 100644 --- a/include/ccf/crypto/key_pair.h +++ b/include/ccf/crypto/key_pair.h @@ -13,7 +13,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class KeyPair { diff --git a/include/ccf/crypto/key_wrap.h b/include/ccf/crypto/key_wrap.h index b6acc0cfb6ef..bda51b22cc4b 100644 --- a/include/ccf/crypto/key_wrap.h +++ b/include/ccf/crypto/key_wrap.h @@ -9,7 +9,7 @@ #include #include -namespace crypto +namespace ccf::crypto { /// PKCS11 2.1.8 CKM_RSA_PKCS_OAEP wrap /// @param wrapping_key The wrapping (encryption) key diff --git a/include/ccf/crypto/md_type.h b/include/ccf/crypto/md_type.h index fd7008ea91bc..8462cb35926e 100644 --- a/include/ccf/crypto/md_type.h +++ b/include/ccf/crypto/md_type.h @@ -4,7 +4,7 @@ #include "ccf/ds/json.h" -namespace crypto +namespace ccf::crypto { enum class MDType { diff --git a/include/ccf/crypto/pem.h b/include/ccf/crypto/pem.h index ef7ba7a6c8b6..f0bd6debef68 100644 --- a/include/ccf/crypto/pem.h +++ b/include/ccf/crypto/pem.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { // Convenience class ensuring null termination of PEM-encoded certificates class Pem @@ -139,11 +139,11 @@ namespace crypto return "Pem"; } - static std::vector split_x509_cert_bundle( + static std::vector split_x509_cert_bundle( const std::string_view& pem) { std::string separator("-----END CERTIFICATE-----"); - std::vector pems; + std::vector pems; auto separator_end = 0; auto next_separator_start = pem.find(separator); while (next_separator_start != std::string_view::npos) @@ -165,9 +165,9 @@ namespace crypto namespace std { template <> - struct hash + struct hash { - size_t operator()(const crypto::Pem& pem) const + size_t operator()(const ccf::crypto::Pem& pem) const { return std::hash()(pem.str()); } diff --git a/include/ccf/crypto/public_key.h b/include/ccf/crypto/public_key.h index 6a205be67938..a2116c862a4c 100644 --- a/include/ccf/crypto/public_key.h +++ b/include/ccf/crypto/public_key.h @@ -13,7 +13,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class PublicKey { diff --git a/include/ccf/crypto/rsa_key_pair.h b/include/ccf/crypto/rsa_key_pair.h index e3ba6b93f60f..ac0f9eca6cf1 100644 --- a/include/ccf/crypto/rsa_key_pair.h +++ b/include/ccf/crypto/rsa_key_pair.h @@ -12,7 +12,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class RSAKeyPair { diff --git a/include/ccf/crypto/rsa_public_key.h b/include/ccf/crypto/rsa_public_key.h index fe265dd0e983..d37463da9447 100644 --- a/include/ccf/crypto/rsa_public_key.h +++ b/include/ccf/crypto/rsa_public_key.h @@ -10,7 +10,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class RSAPublicKey { diff --git a/include/ccf/crypto/san.h b/include/ccf/crypto/san.h index ee10739ba964..aa20a48a970d 100644 --- a/include/ccf/crypto/san.h +++ b/include/ccf/crypto/san.h @@ -9,7 +9,7 @@ #include #include -namespace crypto +namespace ccf::crypto { static const std::string IP_ADDRESS_PREFIX = "iPAddress:"; static const std::string DNS_NAME_PREFIX = "dNSName:"; @@ -56,7 +56,7 @@ namespace crypto FMT_BEGIN_NAMESPACE template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -65,7 +65,7 @@ struct formatter } template - auto format(const crypto::SubjectAltName& san, FormatContext& ctx) const + auto format(const ccf::crypto::SubjectAltName& san, FormatContext& ctx) const -> decltype(ctx.out()) { std::string prefix; diff --git a/include/ccf/crypto/sha256.h b/include/ccf/crypto/sha256.h index 97fac504517f..eb6f4b89654d 100644 --- a/include/ccf/crypto/sha256.h +++ b/include/ccf/crypto/sha256.h @@ -6,7 +6,7 @@ #include -namespace crypto +namespace ccf::crypto { /** Compute the SHA256 hash of @p data * @param data The data to compute the hash of diff --git a/include/ccf/crypto/sha256_hash.h b/include/ccf/crypto/sha256_hash.h index 40ae2ddc7388..cf4cde1a0f15 100644 --- a/include/ccf/crypto/sha256_hash.h +++ b/include/ccf/crypto/sha256_hash.h @@ -10,7 +10,7 @@ #define FMT_HEADER_ONLY #include -namespace crypto +namespace ccf::crypto { class Sha256Hash { @@ -36,7 +36,7 @@ namespace crypto const Sha256Hash& third); friend std::ostream& operator<<( - std::ostream& os, const crypto::Sha256Hash& h); + std::ostream& os, const ccf::crypto::Sha256Hash& h); std::string hex_str() const; @@ -60,7 +60,7 @@ namespace crypto FMT_BEGIN_NAMESPACE template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -69,28 +69,28 @@ struct formatter } template - auto format(const crypto::Sha256Hash& p, FormatContext& ctx) const + auto format(const ccf::crypto::Sha256Hash& p, FormatContext& ctx) const { return format_to(ctx.out(), "", fmt::join(p.h, "")); } }; FMT_END_NAMESPACE -namespace kv::serialisers +namespace ccf::kv::serialisers { template <> - struct BlitSerialiser + struct BlitSerialiser { - static SerialisedEntry to_serialised(const crypto::Sha256Hash& h) + static SerialisedEntry to_serialised(const ccf::crypto::Sha256Hash& h) { auto hex_str = h.hex_str(); return SerialisedEntry(hex_str.begin(), hex_str.end()); } - static crypto::Sha256Hash from_serialised(const SerialisedEntry& data) + static ccf::crypto::Sha256Hash from_serialised(const SerialisedEntry& data) { auto data_str = std::string{data.begin(), data.end()}; - crypto::Sha256Hash ret; + ccf::crypto::Sha256Hash ret; return ret.from_hex_string(data_str); } }; diff --git a/include/ccf/crypto/symmetric_key.h b/include/ccf/crypto/symmetric_key.h index fcf25ba8e606..367a580d309b 100644 --- a/include/ccf/crypto/symmetric_key.h +++ b/include/ccf/crypto/symmetric_key.h @@ -7,7 +7,7 @@ #include #include -namespace crypto +namespace ccf::crypto { constexpr size_t GCM_DEFAULT_KEY_SIZE = 32; @@ -44,7 +44,7 @@ namespace crypto return GCM_SIZE_TAG + IV_SIZE; } - void set_random_iv(EntropyPtr entropy = crypto::get_entropy()) + void set_random_iv(EntropyPtr entropy = ccf::crypto::get_entropy()) { iv = entropy->random(IV_SIZE); } diff --git a/include/ccf/crypto/verifier.h b/include/ccf/crypto/verifier.h index ed74d3b3623b..b57017514ac9 100644 --- a/include/ccf/crypto/verifier.h +++ b/include/ccf/crypto/verifier.h @@ -9,7 +9,7 @@ #include -namespace crypto +namespace ccf::crypto { class Verifier { @@ -252,13 +252,13 @@ namespace crypto */ VerifierPtr make_verifier(const Pem& pem); - crypto::Pem cert_der_to_pem(const std::vector& der); + ccf::crypto::Pem cert_der_to_pem(const std::vector& der); std::vector cert_pem_to_der(const Pem& pem); std::vector public_key_der_from_cert( const std::vector& der); - crypto::Pem public_key_pem_from_cert(const std::vector& der); + ccf::crypto::Pem public_key_pem_from_cert(const std::vector& der); std::string get_subject_name(const Pem& cert); } diff --git a/include/ccf/ds/json.h b/include/ccf/ds/json.h index c840c273860f..45c47638caf6 100644 --- a/include/ccf/ds/json.h +++ b/include/ccf/ds/json.h @@ -66,7 +66,7 @@ namespace std { if constexpr (std::is_same_v) { - j = crypto::b64_from_raw(t); + j = ccf::crypto::b64_from_raw(t); } else { @@ -87,7 +87,7 @@ namespace std { try { - t = crypto::raw_from_b64(j.get()); + t = ccf::crypto::raw_from_b64(j.get()); return; } catch (const std::exception& e) diff --git a/include/ccf/ds/json_schema.h b/include/ccf/ds/json_schema.h index c79b417ba37a..d8216726df90 100644 --- a/include/ccf/ds/json_schema.h +++ b/include/ccf/ds/json_schema.h @@ -97,11 +97,11 @@ namespace ccf::ds template inline std::string schema_name() { - if constexpr (nonstd::is_specialization::value) + if constexpr (ccf::nonstd::is_specialization::value) { return schema_name(); } - else if constexpr (nonstd::is_specialization::value) + else if constexpr (ccf::nonstd::is_specialization::value) { if constexpr (std::is_same>::value) { @@ -113,20 +113,20 @@ namespace ccf::ds return fmt::format("{}_array", schema_name()); } } - else if constexpr (nonstd::is_specialization::value) + else if constexpr (ccf::nonstd::is_specialization::value) { return fmt::format("{}_set", schema_name()); } else if constexpr ( - nonstd::is_specialization::value || - nonstd::is_specialization::value) + ccf::nonstd::is_specialization::value || + ccf::nonstd::is_specialization::value) { return fmt::format( "{}_to_{}", schema_name(), schema_name()); } - else if constexpr (nonstd::is_specialization::value) + else if constexpr (ccf::nonstd::is_specialization::value) { return fmt::format( "{}_and_{}", @@ -198,13 +198,13 @@ namespace ccf::ds template inline void fill_schema(nlohmann::json& schema) { - if constexpr (nonstd::is_specialization::value) + if constexpr (ccf::nonstd::is_specialization::value) { fill_schema(schema); } else if constexpr ( - nonstd::is_specialization::value || - nonstd::is_specialization::value) + ccf::nonstd::is_specialization::value || + ccf::nonstd::is_specialization::value) { if constexpr (std::is_same>::value) { @@ -219,8 +219,8 @@ namespace ccf::ds } } else if constexpr ( - nonstd::is_specialization::value || - nonstd::is_specialization::value) + ccf::nonstd::is_specialization::value || + ccf::nonstd::is_specialization::value) { // Nlohmann JSON serialises some maps as objects, if the keys can be // converted to strings. This should detect those cases. The others are @@ -247,7 +247,7 @@ namespace ccf::ds schema["items"] = items; } } - else if constexpr (nonstd::is_specialization::value) + else if constexpr (ccf::nonstd::is_specialization::value) { schema["type"] = "array"; auto items = nlohmann::json::array(); diff --git a/include/ccf/ds/logger.h b/include/ccf/ds/logger.h index 003bd4f9ef46..9d677820d960 100644 --- a/include/ccf/ds/logger.h +++ b/include/ccf/ds/logger.h @@ -15,7 +15,7 @@ #include #include -namespace logger +namespace ccf::logger { static constexpr LoggerLevel MOST_VERBOSE = #ifdef CCF_DISABLE_VERBOSE_LOGGING @@ -356,9 +356,9 @@ namespace logger // This allows: // CCF_LOG_OUT(DEBUG, "foo") << "this " << "msg"; #define CCF_LOG_OUT(LVL, TAG) \ - logger::config::ok(LoggerLevel::LVL) && \ - logger::Out() == \ - logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__) + ccf::logger::config::ok(LoggerLevel::LVL) && \ + ccf::logger::Out() == \ + ccf::logger::LogLine(LoggerLevel::LVL, TAG, __FILE__, __LINE__) // To avoid repeating the (s, ...) args for every macro, we cheat with a curried // macro here by ending the macro with another macro name, which then accepts @@ -376,7 +376,7 @@ namespace logger }; #ifndef CCF_LOGGER_NO_DEPRECATE -# define CCF_LOGGER_DEPRECATE(MACRO) logger::macro::MACRO; +# define CCF_LOGGER_DEPRECATE(MACRO) ccf::logger::macro::MACRO; #else # define CCF_LOGGER_DEPRECATE(MACRO) #endif diff --git a/include/ccf/ds/nonstd.h b/include/ccf/ds/nonstd.h index 55c7dfced6be..97ba29c13f25 100644 --- a/include/ccf/ds/nonstd.h +++ b/include/ccf/ds/nonstd.h @@ -17,7 +17,7 @@ * some are proposed, some are purely custom. They are defined here to avoid * repetition in other locations. */ -namespace nonstd +namespace ccf::nonstd { /** is_specialization detects type-specialized templates. This does not work * for value-dependent types (eg - std::array) @@ -96,7 +96,7 @@ namespace nonstd } /* split_1 wraps split and allows writing things like: - * auto [host, port] = nonstd::split_1("1.2.3.4:8000", ":") + * auto [host, port] = ccf::nonstd::split_1("1.2.3.4:8000", ":") */ static inline std::tuple split_1( const std::string_view& s, const std::string_view& separator) @@ -153,7 +153,7 @@ namespace nonstd /* rsplit_1 wraps rsplit _and reverses the result order_ and allows writing * things like: - * auto [host, port] = nonstd::rsplit_1("[1:2:3:4]:8000", ":") + * auto [host, port] = ccf::nonstd::rsplit_1("[1:2:3:4]:8000", ":") */ static inline std::tuple rsplit_1( const std::string_view& s, const std::string_view& separator) diff --git a/include/ccf/ds/openapi.h b/include/ccf/ds/openapi.h index 0874ea92568b..c53722fdbd92 100644 --- a/include/ccf/ds/openapi.h +++ b/include/ccf/ds/openapi.h @@ -94,7 +94,7 @@ namespace ccf::ds { // HTTP_GET becomes the string "get" std::string s = llhttp_method_name(verb); - nonstd::to_lower(s); + ccf::nonstd::to_lower(s); auto& po = access::get_object(path, s); if (default_responses) @@ -275,13 +275,13 @@ namespace ccf::ds nlohmann::json add_schema_component() { nlohmann::json schema; - if constexpr (nonstd::is_specialization::value) + if constexpr (ccf::nonstd::is_specialization::value) { return add_schema_component(); } else if constexpr ( - nonstd::is_specialization::value || - nonstd::is_specialization::value) + ccf::nonstd::is_specialization::value || + ccf::nonstd::is_specialization::value) { if constexpr (std::is_same>::value) { @@ -299,8 +299,8 @@ namespace ccf::ds document, ccf::ds::json::schema_name(), schema); } else if constexpr ( - nonstd::is_specialization::value || - nonstd::is_specialization::value) + ccf::nonstd::is_specialization::value || + ccf::nonstd::is_specialization::value) { if constexpr (nlohmann::detail:: is_compatible_object_type::value) @@ -330,7 +330,7 @@ namespace ccf::ds return add_schema_to_components( document, ccf::ds::json::schema_name(), schema); } - else if constexpr (nonstd::is_specialization::value) + else if constexpr (ccf::nonstd::is_specialization::value) { schema["type"] = "array"; auto items = nlohmann::json::array(); diff --git a/include/ccf/ds/unit_strings.h b/include/ccf/ds/unit_strings.h index d5a7fb729f94..e74b9a9f710c 100644 --- a/include/ccf/ds/unit_strings.h +++ b/include/ccf/ds/unit_strings.h @@ -34,7 +34,7 @@ namespace ccf::ds } auto unit = std::string(unit_begin, input.end()); - nonstd::to_lower(unit); + ccf::nonstd::to_lower(unit); auto value = std::string(input.begin(), unit_begin); size_t ret = 0; diff --git a/include/ccf/endpoint.h b/include/ccf/endpoint.h index 3ad91549217c..610f0a073902 100644 --- a/include/ccf/endpoint.h +++ b/include/ccf/endpoint.h @@ -30,7 +30,7 @@ namespace ccf::endpoints }; } -namespace kv::serialisers +namespace ccf::kv::serialisers { template <> struct BlitSerialiser @@ -101,7 +101,12 @@ namespace ccf::endpoints a primary, it should attempt to redirect to the primary, or else return an error. Any write operations must be executed on a primary, so should be marked as ToPrimary. */ - ToPrimary + ToPrimary, + + /** This operation should be executed on a backup. If the current node is + not a backup, it should attempt to redirect to a backup, or else return + an error. Only read operations can be marked as ToBackup. */ + ToBackup, }; enum class Mode @@ -126,7 +131,8 @@ namespace ccf::endpoints DECLARE_JSON_ENUM( RedirectionStrategy, {{RedirectionStrategy::None, "none"}, - {RedirectionStrategy::ToPrimary, "to_primary"}}); + {RedirectionStrategy::ToPrimary, "to_primary"}, + {RedirectionStrategy::ToBackup, "to_backup"}}); DECLARE_JSON_ENUM( Mode, diff --git a/include/ccf/endpoint_context.h b/include/ccf/endpoint_context.h index 0f55e53cef35..eca44c2299cb 100644 --- a/include/ccf/endpoint_context.h +++ b/include/ccf/endpoint_context.h @@ -53,12 +53,12 @@ namespace ccf::endpoints struct EndpointContext : public CommandEndpointContext { - EndpointContext(const std::shared_ptr& r, kv::Tx& t) : + EndpointContext(const std::shared_ptr& r, ccf::kv::Tx& t) : CommandEndpointContext(r), tx(t) {} - kv::Tx& tx; + ccf::kv::Tx& tx; }; using EndpointFunction = std::function; @@ -69,12 +69,12 @@ namespace ccf::endpoints struct ReadOnlyEndpointContext : public CommandEndpointContext { ReadOnlyEndpointContext( - const std::shared_ptr& r, kv::ReadOnlyTx& t) : + const std::shared_ptr& r, ccf::kv::ReadOnlyTx& t) : CommandEndpointContext(r), tx(t) {} - kv::ReadOnlyTx& tx; + ccf::kv::ReadOnlyTx& tx; }; using ReadOnlyEndpointFunction = std::function; diff --git a/include/ccf/endpoint_registry.h b/include/ccf/endpoint_registry.h index d9575399018e..ab991ee0785b 100644 --- a/include/ccf/endpoint_registry.h +++ b/include/ccf/endpoint_registry.h @@ -5,7 +5,6 @@ #include "ccf/ds/json_schema.h" #include "ccf/endpoint.h" #include "ccf/endpoint_context.h" -#include "ccf/pal/locking.h" #include "ccf/rpc_context.h" #include "ccf/tx.h" @@ -16,7 +15,7 @@ #include #include -namespace kv +namespace ccf::kv { class Consensus; class TxHistory; @@ -162,8 +161,8 @@ namespace ccf::endpoints std::map>> templated_endpoints; - kv::Consensus* consensus = nullptr; - kv::TxHistory* history = nullptr; + ccf::kv::Consensus* consensus = nullptr; + ccf::kv::TxHistory* history = nullptr; public: EndpointRegistry(const std::string& method_prefix_) : @@ -255,12 +254,12 @@ namespace ccf::endpoints * internally, so must be able to populate the document * with the supported endpoints however it defines them. */ - virtual void build_api(nlohmann::json& document, kv::ReadOnlyTx&); + virtual void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx&); virtual void init_handlers(); virtual EndpointDefinitionPtr find_endpoint( - kv::Tx&, ccf::RpcContext& rpc_ctx); + ccf::kv::Tx&, ccf::RpcContext& rpc_ctx); virtual void execute_endpoint( EndpointDefinitionPtr e, EndpointContext& args); @@ -269,7 +268,7 @@ namespace ccf::endpoints EndpointDefinitionPtr e, CommandEndpointContext& args, const TxID& tx_id); virtual std::set get_allowed_verbs( - kv::Tx&, const ccf::RpcContext& rpc_ctx); + ccf::kv::Tx&, const ccf::RpcContext& rpc_ctx); virtual bool request_needs_root(const ccf::RpcContext& rpc_ctx); @@ -279,9 +278,9 @@ namespace ccf::endpoints virtual void tick(std::chrono::milliseconds); - void set_consensus(kv::Consensus* c); + void set_consensus(ccf::kv::Consensus* c); - void set_history(kv::TxHistory* h); + void set_history(ccf::kv::TxHistory* h); // Override these methods to log or report request metrics. virtual void handle_event_request_completed( diff --git a/include/ccf/endpoints/authentication/all_of_auth.h b/include/ccf/endpoints/authentication/all_of_auth.h index f6bef98c8ede..f278bd882e40 100644 --- a/include/ccf/endpoints/authentication/all_of_auth.h +++ b/include/ccf/endpoints/authentication/all_of_auth.h @@ -37,7 +37,7 @@ namespace ccf const std::vector>& _policies); std::unique_ptr authenticate( - kv::ReadOnlyTx&, + ccf::kv::ReadOnlyTx&, const std::shared_ptr&, std::string&) override; diff --git a/include/ccf/endpoints/authentication/authentication_types.h b/include/ccf/endpoints/authentication/authentication_types.h index 81f63d6e0180..d22d2535344f 100644 --- a/include/ccf/endpoints/authentication/authentication_types.h +++ b/include/ccf/endpoints/authentication/authentication_types.h @@ -30,7 +30,7 @@ namespace ccf virtual ~AuthnPolicy() = default; virtual std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) = 0; diff --git a/include/ccf/endpoints/authentication/cert_auth.h b/include/ccf/endpoints/authentication/cert_auth.h index 3a84091820dc..6624cc01c059 100644 --- a/include/ccf/endpoints/authentication/cert_auth.h +++ b/include/ccf/endpoints/authentication/cert_auth.h @@ -38,7 +38,7 @@ namespace ccf virtual ~UserCertAuthnPolicy(); std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; @@ -72,7 +72,7 @@ namespace ccf virtual ~MemberCertAuthnPolicy(); std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; @@ -99,7 +99,7 @@ namespace ccf static constexpr auto SECURITY_SCHEME_NAME = "node_cert"; std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; diff --git a/include/ccf/endpoints/authentication/cose_auth.h b/include/ccf/endpoints/authentication/cose_auth.h index b714154c205e..03a59bb0fe6e 100644 --- a/include/ccf/endpoints/authentication/cose_auth.h +++ b/include/ccf/endpoints/authentication/cose_auth.h @@ -22,6 +22,12 @@ namespace ccf uint64_t gov_msg_created_at; }; + struct TimestampedProtectedHeader : ProtectedHeader + { + std::optional msg_type; + std::optional msg_created_at; + }; + struct COSESign1AuthnIdentity : public AuthnIdentity { /** COSE Content */ @@ -55,7 +61,7 @@ namespace ccf MemberId member_id; /** Member certificate, used to sign this request, described by keyId */ - crypto::Pem member_cert; + ccf::crypto::Pem member_cert; /** COSE Protected Header */ GovernanceProtectedHeader protected_header; @@ -65,7 +71,7 @@ namespace ccf const std::span& envelope_, const std::span& signature_, const MemberId& member_id_, - const crypto::Pem& member_cert_, + const ccf::crypto::Pem& member_cert_, const GovernanceProtectedHeader& protected_header_) : COSESign1AuthnIdentity(content_, envelope_, signature_), member_id(member_id_), @@ -80,18 +86,18 @@ namespace ccf UserId user_id; /** User certificate, used to sign this request, described by keyId */ - crypto::Pem user_cert; + ccf::crypto::Pem user_cert; /** COSE Protected Header */ - ProtectedHeader protected_header; + TimestampedProtectedHeader protected_header; UserCOSESign1AuthnIdentity( const std::span& content_, const std::span& envelope_, const std::span& signature_, const UserId& user_id_, - const crypto::Pem& user_cert_, - const ProtectedHeader& protected_header_) : + const ccf::crypto::Pem& user_cert_, + const TimestampedProtectedHeader& protected_header_) : COSESign1AuthnIdentity(content_, envelope_, signature_), user_id(user_id_), user_cert(user_cert_), @@ -119,7 +125,7 @@ namespace ccf ~MemberCOSESign1AuthnPolicy(); std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; @@ -152,7 +158,7 @@ namespace ccf using MemberCOSESign1AuthnPolicy::MemberCOSESign1AuthnPolicy; std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; @@ -163,20 +169,36 @@ namespace ccf }; /** User COSE Sign1 Authentication Policy + * + * Allows parametrising two optional protected header entries + * which are exposed to the endpoint if present. */ class UserCOSESign1AuthnPolicy : public AuthnPolicy { + std::string msg_type_name; + std::string msg_created_at_name; + protected: static const OpenAPISecuritySchema security_schema; + virtual std::unique_ptr _authenticate( + ccf::kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason); + public: static constexpr auto SECURITY_SCHEME_NAME = "user_cose_sign1"; - UserCOSESign1AuthnPolicy(); + UserCOSESign1AuthnPolicy( + const std::string& msg_type_name_ = "ccf.msg.type", + const std::string& msg_created_at_name_ = "ccf.msg.created_at") : + msg_type_name(msg_type_name_), + msg_created_at_name(msg_created_at_name_) + {} ~UserCOSESign1AuthnPolicy(); std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; @@ -195,4 +217,36 @@ namespace ccf return SECURITY_SCHEME_NAME; } }; -} + + /** Typed User COSE Sign1 Authentication Policy + * + * Extends UserCOSESign1AuthPolicy, to require that a specific message + * type is present in the corresponding protected header. + */ + class TypedUserCOSESign1AuthnPolicy : public UserCOSESign1AuthnPolicy + { + private: + std::string expected_msg_type; + + public: + static constexpr auto SECURITY_SCHEME_NAME = "typed_user_cose_sign1"; + + TypedUserCOSESign1AuthnPolicy( + const std::string& expected_msg_type_, + const std::string& msg_type_name_ = "ccf.msg.type", + const std::string& msg_created_at_name_ = "ccf.msg.created_at") : + UserCOSESign1AuthnPolicy(msg_type_name_, msg_created_at_name_), + expected_msg_type(expected_msg_type_) + {} + + std::unique_ptr authenticate( + ccf::kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) override; + + std::string get_security_scheme_name() override + { + return SECURITY_SCHEME_NAME; + } + }; +} \ No newline at end of file diff --git a/include/ccf/endpoints/authentication/empty_auth.h b/include/ccf/endpoints/authentication/empty_auth.h index 11d1e44d8562..f5ec3870e575 100644 --- a/include/ccf/endpoints/authentication/empty_auth.h +++ b/include/ccf/endpoints/authentication/empty_auth.h @@ -17,7 +17,7 @@ namespace ccf static constexpr auto SECURITY_SCHEME_NAME = "no_auth"; std::unique_ptr authenticate( - kv::ReadOnlyTx&, + ccf::kv::ReadOnlyTx&, const std::shared_ptr&, std::string&) override; diff --git a/include/ccf/endpoints/authentication/jwt_auth.h b/include/ccf/endpoints/authentication/jwt_auth.h index 3413dff4b8b8..3a44ee55a73a 100644 --- a/include/ccf/endpoints/authentication/jwt_auth.h +++ b/include/ccf/endpoints/authentication/jwt_auth.h @@ -37,7 +37,7 @@ namespace ccf virtual ~JwtAuthnPolicy(); std::unique_ptr authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) override; diff --git a/include/ccf/entity_id.h b/include/ccf/entity_id.h index 3b26668d7042..0c0f961df367 100644 --- a/include/ccf/entity_id.h +++ b/include/ccf/entity_id.h @@ -202,7 +202,7 @@ struct formatter> }; FMT_END_NAMESPACE -namespace kv::serialisers +namespace ccf::kv::serialisers { template struct BlitSerialiser> diff --git a/include/ccf/historical_queries_adapter.h b/include/ccf/historical_queries_adapter.h index 837c9d3eed1f..d51308cdd178 100644 --- a/include/ccf/historical_queries_adapter.h +++ b/include/ccf/historical_queries_adapter.h @@ -9,7 +9,7 @@ #include "ccf/tx_id.h" #include "ccf/tx_status.h" -namespace kv +namespace ccf::kv { class Consensus; } @@ -51,26 +51,26 @@ namespace ccf::historical ccf::View view, ccf::SeqNo seqno, std::string& error_reason)>; HistoricalTxStatus is_tx_committed_v2( - kv::Consensus* consensus, + ccf::kv::Consensus* consensus, ccf::View view, ccf::SeqNo seqno, std::string& error_reason); ccf::endpoints::EndpointFunction adapter_v3( const HandleHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const TxIDExtractor& extractor = txid_from_header); ccf::endpoints::ReadOnlyEndpointFunction read_only_adapter_v3( const HandleReadOnlyHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const ReadOnlyTxIDExtractor& extractor = txid_from_header); ccf::endpoints::EndpointFunction read_write_adapter_v3( const HandleReadWriteHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const TxIDExtractor& extractor = txid_from_header); } \ No newline at end of file diff --git a/include/ccf/historical_queries_interface.h b/include/ccf/historical_queries_interface.h index 32a2a4e8d91e..f4bc6ba9f4d4 100644 --- a/include/ccf/historical_queries_interface.h +++ b/include/ccf/historical_queries_interface.h @@ -16,14 +16,14 @@ namespace ccf::historical struct State { /// Read-only historical store at transaction_id - kv::ReadOnlyStorePtr store = nullptr; + ccf::kv::ReadOnlyStorePtr store = nullptr; /// Receipt for ledger entry at transaction_id TxReceiptImplPtr receipt = nullptr; /// View and Sequence Number for the State ccf::TxID transaction_id; State( - const kv::ReadOnlyStorePtr& store_, + const ccf::kv::ReadOnlyStorePtr& store_, const TxReceiptImplPtr& receipt_, const ccf::TxID& transaction_id_) : store(store_), @@ -49,6 +49,8 @@ namespace ccf::historical using ExpiryDuration = std::chrono::seconds; + using CacheSize = size_t; + /** Stores the progress of historical query requests. * * A request will generally need to be made multiple times (with the same @@ -79,6 +81,13 @@ namespace ccf::historical virtual void set_default_expiry_duration( ExpiryDuration seconds_until_expiry) = 0; + /** Set the cache limit (in bytes) to evict least recently used requests + * from the cache after its size grows beyond this limit. The limit is not + * strict. It is estimated based on serialized states' sizes approximation + * and is checked once per tick, and so it can overflow for a short time. + */ + virtual void set_soft_cache_limit(CacheSize cache_limit) = 0; + /** EXPERIMENTAL: Set the tracking of deletes on missing keys for historical * queries. * @@ -93,7 +102,7 @@ namespace ccf::historical * is equivalent to get_store_at(handle, seqno, seqno), but returns nullptr * if the state is currently unavailable. */ - virtual kv::ReadOnlyStorePtr get_store_at( + virtual ccf::kv::ReadOnlyStorePtr get_store_at( RequestHandle handle, ccf::SeqNo seqno, ExpiryDuration seconds_until_expiry) = 0; @@ -101,7 +110,7 @@ namespace ccf::historical /** Same as @c get_store_at but uses default expiry value. * @see get_store_at */ - virtual kv::ReadOnlyStorePtr get_store_at( + virtual ccf::kv::ReadOnlyStorePtr get_store_at( RequestHandle handle, ccf::SeqNo seqno) = 0; /** Retrieve a full state at a given seqno, including the Store, the TxID @@ -135,7 +144,7 @@ namespace ccf::historical * vector will be of length (end_seqno - start_seqno + 1) and will contain * no nullptrs. */ - virtual std::vector get_store_range( + virtual std::vector get_store_range( RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno, @@ -144,7 +153,7 @@ namespace ccf::historical /** Same as @c get_store_range but uses default expiry value. * @see get_store_range */ - virtual std::vector get_store_range( + virtual std::vector get_store_range( RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno) = 0; /** Retrieve a range of states at the given indices, including the Store, @@ -165,11 +174,11 @@ namespace ccf::historical /** Retrieve stores for a set of given indices. */ - virtual std::vector get_stores_for( + virtual std::vector get_stores_for( RequestHandle handle, const SeqNoCollection& seqnos, ExpiryDuration seconds_until_expiry) = 0; - virtual std::vector get_stores_for( + virtual std::vector get_stores_for( RequestHandle handle, const SeqNoCollection& seqnos) = 0; /** Retrieve states for a set of given indices. diff --git a/include/ccf/historical_queries_utils.h b/include/ccf/historical_queries_utils.h index 9d1f375ad4d5..133dcd10b996 100644 --- a/include/ccf/historical_queries_utils.h +++ b/include/ccf/historical_queries_utils.h @@ -19,7 +19,7 @@ namespace ccf::historical // additional entries have been requested, in which case the caller should // retry later. bool populate_service_endorsements( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, ccf::historical::StatePtr& state, AbstractStateCache& state_cache, std::shared_ptr diff --git a/include/ccf/http_configuration.h b/include/ccf/http_configuration.h index 2c1e068c27e0..ebe03a465100 100644 --- a/include/ccf/http_configuration.h +++ b/include/ccf/http_configuration.h @@ -7,7 +7,7 @@ #include -namespace http +namespace ccf::http { // Default parser limits, used as a DoS protection against // requests that are too large. diff --git a/include/ccf/http_consts.h b/include/ccf/http_consts.h index 8b021b4fff14..7f93b217b647 100644 --- a/include/ccf/http_consts.h +++ b/include/ccf/http_consts.h @@ -2,72 +2,74 @@ // Licensed under the Apache 2.0 License. #pragma once -namespace http +namespace ccf { - namespace headers + namespace http { - // All HTTP headers are expected to be lowercase - static constexpr auto ACCEPT = "accept"; - static constexpr auto ALLOW = "allow"; - static constexpr auto AUTHORIZATION = "authorization"; - static constexpr auto CACHE_CONTROL = "cache-control"; - static constexpr auto CONTENT_LENGTH = "content-length"; - static constexpr auto CONTENT_TYPE = "content-type"; - static constexpr auto DATE = "date"; - static constexpr auto DIGEST = "digest"; - static constexpr auto HOST = "host"; - static constexpr auto LOCATION = "location"; - static constexpr auto RETRY_AFTER = "retry-after"; - static constexpr auto TRAILER = "trailer"; - static constexpr auto WWW_AUTHENTICATE = "www-authenticate"; + namespace headers + { + // All HTTP headers are expected to be lowercase + static constexpr auto ACCEPT = "accept"; + static constexpr auto ALLOW = "allow"; + static constexpr auto AUTHORIZATION = "authorization"; + static constexpr auto CACHE_CONTROL = "cache-control"; + static constexpr auto CONTENT_LENGTH = "content-length"; + static constexpr auto CONTENT_TYPE = "content-type"; + static constexpr auto DATE = "date"; + static constexpr auto DIGEST = "digest"; + static constexpr auto HOST = "host"; + static constexpr auto LOCATION = "location"; + static constexpr auto RETRY_AFTER = "retry-after"; + static constexpr auto TRAILER = "trailer"; + static constexpr auto WWW_AUTHENTICATE = "www-authenticate"; - static constexpr auto CCF_TX_ID = "x-ms-ccf-transaction-id"; - } + static constexpr auto CCF_TX_ID = "x-ms-ccf-transaction-id"; + } - namespace headervalues - { - namespace contenttype + namespace headervalues { - static constexpr auto JSON = "application/json"; - static constexpr auto MSGPACK = "application/msgpack"; - static constexpr auto TEXT = "text/plain"; - static constexpr auto OCTET_STREAM = "application/octet-stream"; - static constexpr auto GRPC = "application/grpc"; - static constexpr auto COSE = "application/cose"; - static constexpr auto JAVASCRIPT = "text/javascript"; + namespace contenttype + { + static constexpr auto JSON = "application/json"; + static constexpr auto TEXT = "text/plain"; + static constexpr auto OCTET_STREAM = "application/octet-stream"; + static constexpr auto GRPC = "application/grpc"; + static constexpr auto COSE = "application/cose"; + static constexpr auto JAVASCRIPT = "text/javascript"; + } } - } - namespace auth - { - static constexpr auto DIGEST_SHA256 = "SHA-256"; + namespace auth + { + static constexpr auto DIGEST_SHA256 = "SHA-256"; - static constexpr auto SIGN_AUTH_SCHEME = "Signature"; - static constexpr auto SIGN_PARAMS_KEYID = "keyId"; - static constexpr auto SIGN_PARAMS_SIGNATURE = "signature"; - static constexpr auto SIGN_PARAMS_ALGORITHM = "algorithm"; - static constexpr auto SIGN_PARAMS_HEADERS = "headers"; - static constexpr auto SIGN_ALGORITHM_ECDSA_SHA256 = "ecdsa-sha256"; - static constexpr auto SIGN_ALGORITHM_HS_2019 = "hs2019"; + static constexpr auto SIGN_AUTH_SCHEME = "Signature"; + static constexpr auto SIGN_PARAMS_KEYID = "keyId"; + static constexpr auto SIGN_PARAMS_SIGNATURE = "signature"; + static constexpr auto SIGN_PARAMS_ALGORITHM = "algorithm"; + static constexpr auto SIGN_PARAMS_HEADERS = "headers"; + static constexpr auto SIGN_ALGORITHM_ECDSA_SHA256 = "ecdsa-sha256"; + static constexpr auto SIGN_ALGORITHM_HS_2019 = "hs2019"; - static constexpr auto SIGN_HEADER_REQUEST_TARGET = "(request-target)"; + static constexpr auto SIGN_HEADER_REQUEST_TARGET = "(request-target)"; - static constexpr auto SIGN_PARAMS_DELIMITER = ","; - static constexpr auto SIGN_PARAMS_HEADERS_DELIMITER = " "; + static constexpr auto SIGN_PARAMS_DELIMITER = ","; + static constexpr auto SIGN_PARAMS_HEADERS_DELIMITER = " "; - static constexpr auto BEARER_AUTH_SCHEME = "Bearer"; - } + static constexpr auto BEARER_AUTH_SCHEME = "Bearer"; + } - static constexpr char const* required_signature_headers[] = { - auth::SIGN_HEADER_REQUEST_TARGET, headers::DIGEST}; -} + static constexpr char const* required_signature_headers[] = { + auth::SIGN_HEADER_REQUEST_TARGET, ccf::http::headers::DIGEST}; + } -namespace http2 -{ - namespace headers + namespace http2 { - static constexpr auto PATH = ":path"; - static constexpr auto STATUS = ":status"; - static constexpr auto METHOD = ":method"; + namespace headers + { + static constexpr auto PATH = ":path"; + static constexpr auto STATUS = ":status"; + static constexpr auto METHOD = ":method"; + } } } \ No newline at end of file diff --git a/include/ccf/http_header_map.h b/include/ccf/http_header_map.h index c7d55d27d594..476075ca4591 100644 --- a/include/ccf/http_header_map.h +++ b/include/ccf/http_header_map.h @@ -5,7 +5,7 @@ #include #include -namespace http +namespace ccf::http { using HeaderMap = std::map>; using HeaderKeyValue = HeaderMap::value_type; diff --git a/include/ccf/http_query.h b/include/ccf/http_query.h index c9bc0baa44c9..42e731d9bdcb 100644 --- a/include/ccf/http_query.h +++ b/include/ccf/http_query.h @@ -11,7 +11,7 @@ #include #include -namespace http +namespace ccf::http { // Query is parsed into a multimap, so that duplicate keys are retained. // Handling of duplicates (or ignoring them entirely) is left to the caller. @@ -22,12 +22,12 @@ namespace http static ParsedQuery parse_query(const std::string_view& query) { ParsedQuery parsed; - const auto params = nonstd::split(query, "&"); + const auto params = ccf::nonstd::split(query, "&"); for (const auto& param : params) { // NB: This means both `foo=` and `foo` will be accepted and result in a // `{"foo": ""}` in the map - const auto& [key, value] = nonstd::split_1(param, "="); + const auto& [key, value] = ccf::nonstd::split_1(param, "="); parsed.emplace(key, value); } @@ -71,7 +71,7 @@ namespace http } else { - static_assert(nonstd::dependent_false::value, "Unsupported type"); + static_assert(ccf::nonstd::dependent_false::value, "Unsupported type"); return false; } } diff --git a/include/ccf/http_responder.h b/include/ccf/http_responder.h index 667dee9968dc..c3f530f27cd9 100644 --- a/include/ccf/http_responder.h +++ b/include/ccf/http_responder.h @@ -8,7 +8,7 @@ #include -namespace http +namespace ccf::http { using StreamOnCloseCallback = std::function; diff --git a/include/ccf/indexing/strategies/seqnos_by_key_bucketed.h b/include/ccf/indexing/strategies/seqnos_by_key_bucketed.h index 48df74644d20..96ec1d15ff46 100644 --- a/include/ccf/indexing/strategies/seqnos_by_key_bucketed.h +++ b/include/ccf/indexing/strategies/seqnos_by_key_bucketed.h @@ -30,7 +30,7 @@ namespace ccf::indexing::strategies public: SeqnosByKey_Bucketed_Untyped( const std::string& map_name_, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, size_t seqnos_per_bucket_ = 1000, size_t max_buckets_ = 10); @@ -45,7 +45,7 @@ namespace ccf::indexing::strategies SeqnosByKey_Bucketed( const M& map, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, size_t seqnos_per_bucket_ = 1000, size_t max_buckets_ = 10) : SeqnosByKey_Bucketed_Untyped( @@ -67,7 +67,7 @@ namespace ccf::indexing::strategies SeqnosForValue_Bucketed( const V& value, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, size_t seqnos_per_bucket_ = 1000, size_t max_buckets_ = 10) : SeqnosByKey_Bucketed_Untyped( diff --git a/include/ccf/indexing/strategies/visit_each_entry_in_map.h b/include/ccf/indexing/strategies/visit_each_entry_in_map.h index 7d0edabccc28..fb6b9613e058 100644 --- a/include/ccf/indexing/strategies/visit_each_entry_in_map.h +++ b/include/ccf/indexing/strategies/visit_each_entry_in_map.h @@ -34,7 +34,7 @@ namespace ccf::indexing::strategies virtual ~VisitEachEntryInMap() = default; void handle_committed_transaction( - const ccf::TxID& tx_id, const kv::ReadOnlyStorePtr& store) override; + const ccf::TxID& tx_id, const ccf::kv::ReadOnlyStorePtr& store) override; std::optional next_requested() override; nlohmann::json describe() override; diff --git a/include/ccf/indexing/strategy.h b/include/ccf/indexing/strategy.h index 06f3e7567110..1d29483adb64 100644 --- a/include/ccf/indexing/strategy.h +++ b/include/ccf/indexing/strategy.h @@ -38,7 +38,7 @@ namespace ccf::indexing * transaction. */ virtual void handle_committed_transaction( - const ccf::TxID& tx_id, const kv::ReadOnlyStorePtr& store) = 0; + const ccf::TxID& tx_id, const ccf::kv::ReadOnlyStorePtr& store) = 0; virtual void tick() {} diff --git a/include/ccf/js/audit_format.h b/include/ccf/js/audit_format.h new file mode 100644 index 000000000000..96a36932990f --- /dev/null +++ b/include/ccf/js/audit_format.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#pragma once +#include "ccf/ds/json.h" + +#include + +namespace ccf +{ + enum class ActionFormat + { + COSE = 0, + JSON = 1 + }; + DECLARE_JSON_ENUM( + ActionFormat, {{ActionFormat::COSE, "COSE"}, {ActionFormat::JSON, "JSON"}}); + + struct AuditInfo + { + ActionFormat format; + // Deliberately a string and not a ccf::UserId to allow extended usage, for + // example with OpenID + std::string user_id; + // Format left to the application, Verb + URL with some of kind of + // versioning is recommended + std::string action_name; + }; + + DECLARE_JSON_TYPE(AuditInfo) + DECLARE_JSON_REQUIRED_FIELDS(AuditInfo, format, user_id, action_name) +} \ No newline at end of file diff --git a/include/ccf/js/common_context.h b/include/ccf/js/common_context.h index 036aaa1ed253..613de5a029f2 100644 --- a/include/ccf/js/common_context.h +++ b/include/ccf/js/common_context.h @@ -48,7 +48,7 @@ namespace ccf::js class WithKVExtension : public Base { public: - WithKVExtension(TxAccess acc, kv::Tx* tx) : Base(acc) + WithKVExtension(TxAccess acc, ccf::kv::Tx* tx) : Base(acc) { // add ccf.kv.* Base::add_extension( diff --git a/include/ccf/js/extensions/ccf/gov_effects.h b/include/ccf/js/extensions/ccf/gov_effects.h index 63fe935e1663..bfde1a7984bd 100644 --- a/include/ccf/js/extensions/ccf/gov_effects.h +++ b/include/ccf/js/extensions/ccf/gov_effects.h @@ -21,9 +21,9 @@ namespace ccf::js::extensions class GovEffectsExtension : public ExtensionInterface { public: - kv::Tx* tx; + ccf::kv::Tx* tx; - GovEffectsExtension(kv::Tx* t) : tx(t) {} + GovEffectsExtension(ccf::kv::Tx* t) : tx(t) {} void install(js::core::Context& ctx) override; }; diff --git a/include/ccf/js/extensions/ccf/kv.h b/include/ccf/js/extensions/ccf/kv.h index 51b3430190cf..8dec00d412f2 100644 --- a/include/ccf/js/extensions/ccf/kv.h +++ b/include/ccf/js/extensions/ccf/kv.h @@ -7,7 +7,7 @@ #include -namespace kv +namespace ccf::kv { class Tx; } @@ -28,7 +28,7 @@ namespace ccf::js::extensions ccf::js::NamespaceRestriction namespace_restriction; - KvExtension(kv::Tx* t, const ccf::js::NamespaceRestriction& nr = {}); + KvExtension(ccf::kv::Tx* t, const ccf::js::NamespaceRestriction& nr = {}); ~KvExtension(); void install(js::core::Context& ctx); diff --git a/include/ccf/js/named_auth_policies.h b/include/ccf/js/named_auth_policies.h index 82f6b5a26f8b..a273a66a25d4 100644 --- a/include/ccf/js/named_auth_policies.h +++ b/include/ccf/js/named_auth_policies.h @@ -6,7 +6,7 @@ #include "ccf/common_auth_policies.h" #include "ccf/endpoint_registry.h" -namespace ccfapp +namespace ccf { using NamedAuthPolicies = std::unordered_map>; diff --git a/include/ccf/js/registry.h b/include/ccf/js/registry.h index 282504c8d4c0..edfdb02b83e4 100644 --- a/include/ccf/js/registry.h +++ b/include/ccf/js/registry.h @@ -4,6 +4,7 @@ // CCF #include "ccf/app_interface.h" #include "ccf/endpoint.h" +#include "ccf/js/audit_format.h" #include "ccf/js/bundle.h" #include "ccf/js/core/context.h" #include "ccf/js/interpreter_cache_interface.h" @@ -51,6 +52,9 @@ namespace ccf::js std::string modules_quickjs_version_map; std::string modules_quickjs_bytecode_map; std::string runtime_options_map; + std::string recent_actions_map; + std::string audit_input_map; + std::string audit_info_map; ccf::js::NamespaceRestriction namespace_restriction; @@ -72,7 +76,7 @@ namespace ccf::js public: DynamicJSEndpointRegistry( - ccfapp::AbstractNodeContext& context, + ccf::AbstractNodeContext& context, const std::string& kv_prefix = "public:custom_endpoints"); /** @@ -80,7 +84,7 @@ namespace ccf::js * later be dispatched to. */ ccf::ApiResult install_custom_endpoints_v1( - kv::Tx& tx, const ccf::js::Bundle& bundle); + ccf::kv::Tx& tx, const ccf::js::Bundle& bundle); /** * Retrieve all endpoint definitions currently in-use. This returns the same @@ -89,14 +93,14 @@ namespace ccf::js * due to internal normalisation. */ ccf::ApiResult get_custom_endpoints_v1( - ccf::js::Bundle& bundle, kv::ReadOnlyTx& tx); + ccf::js::Bundle& bundle, ccf::kv::ReadOnlyTx& tx); /** * Retrieve property definition for a single JS endpoint. */ ccf::ApiResult get_custom_endpoint_properties_v1( ccf::endpoints::EndpointProperties& properties, - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const ccf::RESTVerb& verb, const ccf::endpoints::URI& uri); @@ -104,7 +108,9 @@ namespace ccf::js * Retrieve content of a single JS module. */ ccf::ApiResult get_custom_endpoint_module_v1( - std::string& code, kv::ReadOnlyTx& tx, const std::string& module_name); + std::string& code, + ccf::kv::ReadOnlyTx& tx, + const std::string& module_name); /** * Pass a function to control which maps can be accessed by JS endpoints. @@ -117,7 +123,7 @@ namespace ccf::js * bound any values specified here. */ ccf::ApiResult set_js_runtime_options_v1( - kv::Tx& tx, const ccf::JSRuntimeOptions& options); + ccf::kv::Tx& tx, const ccf::JSRuntimeOptions& options); /** * Get the options which currently control JS execution. If no value has @@ -125,13 +131,35 @@ namespace ccf::js * which will be applied instead. */ ccf::ApiResult get_js_runtime_options_v1( - ccf::JSRuntimeOptions& options, kv::ReadOnlyTx& tx); + ccf::JSRuntimeOptions& options, ccf::kv::ReadOnlyTx& tx); + + /** + * Record action details by storing them in KV maps using a common format, + * for the purposes of offline audit using the ledger. + */ + ccf::ApiResult record_action_for_audit_v1( + ccf::kv::Tx& tx, + ccf::ActionFormat format, + const std::string& user_id, + const std::string& action_name, + const std::vector& action_body); + + /** + * Check an action is not being replayed, by looking it up + * in the history of recent actions. To place an upper bound on the history + * size, an authenticated timestamp (@p created_at) is required. + */ + ccf::ApiResult check_action_not_replayed_v1( + ccf::kv::Tx& tx, + uint64_t created_at, + const std::span action, + ccf::InvalidArgsReason& reason); /// \defgroup Overrides for base EndpointRegistry functions, looking up JS /// endpoints before delegating to base implementation. ///@{ ccf::endpoints::EndpointDefinitionPtr find_endpoint( - kv::Tx& tx, ccf::RpcContext& rpc_ctx) override; + ccf::kv::Tx& tx, ccf::RpcContext& rpc_ctx) override; void execute_endpoint( ccf::endpoints::EndpointDefinitionPtr e, @@ -142,7 +170,13 @@ namespace ccf::js ccf::endpoints::CommandEndpointContext& endpoint_ctx, const ccf::TxID& tx_id) override; - void build_api(nlohmann::json& document, kv::ReadOnlyTx& tx) override; + void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) override; ///@} + + virtual ccf::js::extensions::Extensions get_extensions( + const ccf::endpoints::EndpointContext& endpoint_ctx) + { + return {}; + }; }; } diff --git a/include/ccf/json_handler.h b/include/ccf/json_handler.h index e29ee50f4f2d..09576200516c 100644 --- a/include/ccf/json_handler.h +++ b/include/ccf/json_handler.h @@ -3,7 +3,6 @@ #pragma once #include "ccf/endpoint_registry.h" -#include "ccf/serdes.h" #include #include @@ -11,18 +10,16 @@ namespace ccf { /* - * For simple app methods which expect a JSON request, potentially msgpack'd, - * these functions do the common decoding of the input and setting of response - * fields, to reduce handler complexity and repetition. + * For simple app methods which expect a JSON request these functions do the + * common decoding of the input and setting of response fields, to reduce + * handler complexity and repetition. * * Rather than: * auto foo = [](auto& ctx) { * nlohmann::json params; - * serdes::Pack pack_type; * if () * { - * params = unpack(ctx.rpc_ctx->get_request_body()); - * pack_type = Text; + * params = nlohmann::json::parse(ctx.rpc_ctx->get_request_body()); * } * else * { @@ -35,15 +32,8 @@ namespace ccf * ctx.rpc_ctx->set_response_header(content_type, Text); * ctx.rpc_ctx->set_response_body(error_msg(result)); * } - * if (pack_type == Text) - * { - * ctx.rpc_ctx->set_response_header(content_type, JSON); - * ctx.rpc_ctx->set_response_body(pack(result, Text)); - * } - * else - * { - * ... - * } + * ctx.rpc_ctx->set_response_header(content_type, JSON); + * ctx.rpc_ctx->set_response_body(result.dump()); * }; * * it is possible to write the shorter, clearer, return-based lambda: @@ -65,24 +55,10 @@ namespace ccf using JsonAdapterResponse = std::variant; - char const* pack_to_content_type(serdes::Pack p); - - serdes::Pack detect_json_pack(const std::shared_ptr& ctx); - - serdes::Pack get_response_pack( - const std::shared_ptr& ctx, - serdes::Pack request_pack = serdes::Pack::Text); - - nlohmann::json get_params_from_body( - const std::shared_ptr& ctx, serdes::Pack pack); - - std::pair get_json_params( - const std::shared_ptr& ctx); + nlohmann::json get_json_params(const std::shared_ptr& ctx); void set_response( - JsonAdapterResponse&& res, - std::shared_ptr& ctx, - serdes::Pack request_packing); + JsonAdapterResponse&& res, std::shared_ptr& ctx); } jsonhandler::JsonAdapterResponse make_success(); diff --git a/include/ccf/kv/abstract_handle.h b/include/ccf/kv/abstract_handle.h index f37a9e7935c0..fa9d97190e35 100644 --- a/include/ccf/kv/abstract_handle.h +++ b/include/ccf/kv/abstract_handle.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -namespace kv +namespace ccf::kv { class AbstractHandle { diff --git a/include/ccf/kv/get_name.h b/include/ccf/kv/get_name.h index ad93f50357d9..7fb80d2ec309 100644 --- a/include/ccf/kv/get_name.h +++ b/include/ccf/kv/get_name.h @@ -4,7 +4,7 @@ #include -namespace kv +namespace ccf::kv { struct GetName { diff --git a/include/ccf/kv/hooks.h b/include/ccf/kv/hooks.h index 83a814fd1c7b..34e1d4972516 100644 --- a/include/ccf/kv/hooks.h +++ b/include/ccf/kv/hooks.h @@ -7,7 +7,7 @@ #include #include -namespace kv +namespace ccf::kv { class ConfigurableConsensus; diff --git a/include/ccf/kv/map.h b/include/ccf/kv/map.h index 7d19355060d5..d2cd51ce5170 100644 --- a/include/ccf/kv/map.h +++ b/include/ccf/kv/map.h @@ -10,7 +10,7 @@ #include "ccf/kv/serialisers/json_serialiser.h" #include "ccf/kv/untyped.h" -namespace kv +namespace ccf::kv { /** Defines the schema of a map accessed by a @c ccf::Tx, exposing associated * types. This map is an unordered associative container of key-value pairs. @@ -31,11 +31,11 @@ namespace kv public: // Expose correct public aliases of types using ReadOnlyHandle = - kv::ReadableMapHandle; + ccf::kv::ReadableMapHandle; using WriteOnlyHandle = - kv::WriteableMapHandle; - using Handle = kv::MapHandle; - using Diff = kv::MapDiff; + ccf::kv::WriteableMapHandle; + using Handle = ccf::kv::MapHandle; + using Diff = ccf::kv::MapDiff; using Write = std::map>; using CommitHook = CommitHook; @@ -49,7 +49,7 @@ namespace kv using GetName::GetName; private: - static Write deserialise_write(const kv::untyped::Write& w) + static Write deserialise_write(const ccf::kv::untyped::Write& w) { Write typed_writes; for (const auto& [uk, opt_uv] : w) @@ -70,16 +70,16 @@ namespace kv } public: - static kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) + static ccf::kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { hook(v, deserialise_write(w)); }; } - static kv::untyped::MapHook wrap_map_hook(const MapHook& hook) + static ccf::kv::untyped::MapHook wrap_map_hook(const MapHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { return hook(v, deserialise_write(w)); }; } @@ -95,14 +95,14 @@ namespace kv template using JsonSerialisedMap = - MapSerialisedWith; + MapSerialisedWith; template using RawCopySerialisedMap = TypedMap< K, V, - kv::serialisers::BlitSerialiser, - kv::serialisers::BlitSerialiser>; + ccf::kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::BlitSerialiser>; /** Short name for default-serialised maps, using JSON serialisers. Support * for custom types can be added through the DECLARE_JSON... macros. diff --git a/include/ccf/kv/map_diff.h b/include/ccf/kv/map_diff.h index 9161b4daf92d..a66ecda2995a 100644 --- a/include/ccf/kv/map_diff.h +++ b/include/ccf/kv/map_diff.h @@ -4,21 +4,21 @@ #include "ccf/kv/untyped_map_diff.h" -namespace kv +namespace ccf::kv { template class MapDiff : public AbstractHandle { protected: - kv::untyped::MapDiff map_diff; + ccf::kv::untyped::MapDiff map_diff; public: using KeyType = K; using ValueType = V; - MapDiff(kv::untyped::MapDiff map_diff_) : map_diff(map_diff_) {} + MapDiff(ccf::kv::untyped::MapDiff map_diff_) : map_diff(map_diff_) {} - MapDiff(kv::untyped::ChangeSet& changes, const std::string& map_name) : + MapDiff(ccf::kv::untyped::ChangeSet& changes, const std::string& map_name) : map_diff(changes, map_name) {} @@ -86,8 +86,8 @@ namespace kv { const auto& g = [&]( - const kv::serialisers::SerialisedEntry& k_rep, - const std::optional& v_rep) + const ccf::kv::serialisers::SerialisedEntry& k_rep, + const std::optional& v_rep) -> bool { const auto k = KSerialiser::from_serialised(k_rep); if (v_rep.has_value()) @@ -116,8 +116,8 @@ namespace kv void foreach_key(F&& f) { auto g = [&]( - const kv::serialisers::SerialisedEntry& k_rep, - const kv::serialisers::SerialisedEntry&) { + const ccf::kv::serialisers::SerialisedEntry& k_rep, + const ccf::kv::serialisers::SerialisedEntry&) { return f(KSerialiser::from_serialised(k_rep)); }; map_diff.foreach(g); @@ -133,18 +133,19 @@ namespace kv template void foreach_value(F&& f) { - auto g = [&]( - const kv::serialisers::SerialisedEntry&, - const std::optional& v_rep) { - if (v_rep.has_value()) - { - return f(VSerialiser::from_serialised(v_rep)); - } - else - { - return f(std::nullopt); - } - }; + auto g = + [&]( + const ccf::kv::serialisers::SerialisedEntry&, + const std::optional& v_rep) { + if (v_rep.has_value()) + { + return f(VSerialiser::from_serialised(v_rep)); + } + else + { + return f(std::nullopt); + } + }; map_diff.foreach(g); } diff --git a/include/ccf/kv/map_handle.h b/include/ccf/kv/map_handle.h index 6ac8e00a0c11..39c4347f2c6a 100644 --- a/include/ccf/kv/map_handle.h +++ b/include/ccf/kv/map_handle.h @@ -4,21 +4,21 @@ #include "ccf/kv/untyped_map_handle.h" -namespace kv +namespace ccf::kv { - /** Grants read access to a @c kv::Map, as part of a @c kv::Tx. + /** Grants read access to a @c ccf::kv::Map, as part of a @c ccf::kv::Tx. */ template class ReadableMapHandle { protected: - kv::untyped::MapHandle& read_handle; + ccf::kv::untyped::MapHandle& read_handle; public: using KeyType = K; using ValueType = V; - ReadableMapHandle(kv::untyped::MapHandle& uh) : read_handle(uh) {} + ReadableMapHandle(ccf::kv::untyped::MapHandle& uh) : read_handle(uh) {} /** Get name of this map. * @@ -146,8 +146,8 @@ namespace kv void foreach(F&& f) { auto g = [&]( - const kv::serialisers::SerialisedEntry& k_rep, - const kv::serialisers::SerialisedEntry& v_rep) { + const ccf::kv::serialisers::SerialisedEntry& k_rep, + const ccf::kv::serialisers::SerialisedEntry& v_rep) { return f( KSerialiser::from_serialised(k_rep), VSerialiser::from_serialised(v_rep)); @@ -169,8 +169,8 @@ namespace kv void foreach_key(F&& f) { auto g = [&]( - const kv::serialisers::SerialisedEntry& k_rep, - const kv::serialisers::SerialisedEntry&) { + const ccf::kv::serialisers::SerialisedEntry& k_rep, + const ccf::kv::serialisers::SerialisedEntry&) { return f(KSerialiser::from_serialised(k_rep)); }; read_handle.foreach(g); @@ -190,8 +190,8 @@ namespace kv void foreach_value(F&& f) { auto g = [&]( - const kv::serialisers::SerialisedEntry&, - const kv::serialisers::SerialisedEntry& v_rep) { + const ccf::kv::serialisers::SerialisedEntry&, + const ccf::kv::serialisers::SerialisedEntry& v_rep) { return f(VSerialiser::from_serialised(v_rep)); }; read_handle.foreach(g); @@ -211,16 +211,16 @@ namespace kv } }; - /** Grants write access to a @c kv::Map, as part of a @c kv::Tx. + /** Grants write access to a @c ccf::kv::Map, as part of a @c ccf::kv::Tx. */ template class WriteableMapHandle { protected: - kv::untyped::MapHandle& write_handle; + ccf::kv::untyped::MapHandle& write_handle; public: - WriteableMapHandle(kv::untyped::MapHandle& uh) : write_handle(uh) {} + WriteableMapHandle(ccf::kv::untyped::MapHandle& uh) : write_handle(uh) {} /** Write value at key. * @@ -255,10 +255,11 @@ namespace kv } }; - /** Grants read and write access to a @c kv::Map, as part of a @c kv::Tx. + /** Grants read and write access to a @c ccf::kv::Map, as part of a @c + * ccf::kv::Tx. * - * @see kv::ReadableMapHandle - * @see kv::WriteableMapHandle + * @see ccf::kv::ReadableMapHandle + * @see ccf::kv::WriteableMapHandle */ template class MapHandle : public AbstractHandle, @@ -266,13 +267,14 @@ namespace kv public WriteableMapHandle { protected: - kv::untyped::MapHandle untyped_handle; + ccf::kv::untyped::MapHandle untyped_handle; using ReadableBase = ReadableMapHandle; using WriteableBase = WriteableMapHandle; public: - MapHandle(kv::untyped::ChangeSet& changes, const std::string& map_name) : + MapHandle( + ccf::kv::untyped::ChangeSet& changes, const std::string& map_name) : ReadableBase(untyped_handle), WriteableBase(untyped_handle), untyped_handle(changes, map_name) diff --git a/include/ccf/kv/read_only_store.h b/include/ccf/kv/read_only_store.h index 9e273f49cf43..ae601906ec0f 100644 --- a/include/ccf/kv/read_only_store.h +++ b/include/ccf/kv/read_only_store.h @@ -7,7 +7,7 @@ #include -namespace kv +namespace ccf::kv { class ReadOnlyStore { @@ -15,9 +15,9 @@ namespace kv virtual ~ReadOnlyStore() = default; virtual ccf::TxID get_txid() = 0; - virtual kv::ReadOnlyTx create_read_only_tx() = 0; - virtual std::unique_ptr create_read_only_tx_ptr() = 0; - virtual kv::TxDiff create_tx_diff() = 0; + virtual ccf::kv::ReadOnlyTx create_read_only_tx() = 0; + virtual std::unique_ptr create_read_only_tx_ptr() = 0; + virtual ccf::kv::TxDiff create_tx_diff() = 0; }; using ReadOnlyStorePtr = std::shared_ptr; diff --git a/include/ccf/kv/serialisers/blit_serialiser.h b/include/ccf/kv/serialisers/blit_serialiser.h index 39793bb9ee84..6af7c1a4daeb 100644 --- a/include/ccf/kv/serialisers/blit_serialiser.h +++ b/include/ccf/kv/serialisers/blit_serialiser.h @@ -5,7 +5,7 @@ #include "ccf/ds/nonstd.h" #include "ccf/kv/serialisers/serialised_entry.h" -namespace kv::serialisers +namespace ccf::kv::serialisers { // Converts values to their raw, in-memory representation. To add support for // custom types, add a specialization of BlitSerialiser for them. @@ -18,7 +18,7 @@ namespace kv::serialisers { return SerialisedEntry(t.begin(), t.end()); } - else if constexpr (nonstd::is_std_array::value) + else if constexpr (ccf::nonstd::is_std_array::value) { return SerialisedEntry(t.begin(), t.end()); } @@ -35,7 +35,7 @@ namespace kv::serialisers else { static_assert( - nonstd::dependent_false::value, "Can't serialise this type"); + ccf::nonstd::dependent_false::value, "Can't serialise this type"); } } @@ -45,7 +45,7 @@ namespace kv::serialisers { return T(rep.begin(), rep.end()); } - else if constexpr (nonstd::is_std_array::value) + else if constexpr (ccf::nonstd::is_std_array::value) { T t; if (rep.size() != t.size()) @@ -77,7 +77,8 @@ namespace kv::serialisers else { static_assert( - nonstd::dependent_false::value, "Can't deserialise this type"); + ccf::nonstd::dependent_false::value, + "Can't deserialise this type"); } } }; diff --git a/include/ccf/kv/serialisers/json_serialiser.h b/include/ccf/kv/serialisers/json_serialiser.h index 2f866ee283a8..e1f22f40eb0d 100644 --- a/include/ccf/kv/serialisers/json_serialiser.h +++ b/include/ccf/kv/serialisers/json_serialiser.h @@ -6,7 +6,7 @@ #include -namespace kv::serialisers +namespace ccf::kv::serialisers { // Converts values to and from JSON, using nlohmann JSON. To add support for // custom types, make them convertible to nlohmann::json. You may do this diff --git a/include/ccf/kv/serialisers/serialised_entry.h b/include/ccf/kv/serialisers/serialised_entry.h index 34a600780ec2..aaadf096e5b6 100644 --- a/include/ccf/kv/serialisers/serialised_entry.h +++ b/include/ccf/kv/serialisers/serialised_entry.h @@ -3,7 +3,7 @@ #pragma once #include "ccf/byte_vector.h" -namespace kv::serialisers +namespace ccf::kv::serialisers { using SerialisedEntry = ccf::ByteVector; } diff --git a/include/ccf/kv/set.h b/include/ccf/kv/set.h index cc9341ad6238..62560adc6351 100644 --- a/include/ccf/kv/set.h +++ b/include/ccf/kv/set.h @@ -9,7 +9,7 @@ #include "ccf/kv/set_handle.h" #include "ccf/kv/untyped.h" -namespace kv +namespace ccf::kv { /** Defines the schema of a set type accessed by a @c ccf::Tx. This set is an * unordered container of unique keys. Each key is either present or missing @@ -21,24 +21,26 @@ namespace kv * evaluated on the serialised form; if unequal Ks produce the same * serialisation, they will coincide within this set. * - * This is implemented as a @c kv::Map from K to Unit, and the serialisation - * of the unit values is overridable with the Unit template parameter. + * This is implemented as a @c ccf::kv::Map from K to Unit, and the + * serialisation of the unit values is overridable with the Unit template + * parameter. */ template < typename K, typename KSerialiser, - typename Unit = kv::serialisers::ZeroBlitUnitCreator> + typename Unit = ccf::kv::serialisers::ZeroBlitUnitCreator> class TypedSet : public GetName { public: - using ReadOnlyHandle = kv::ReadableSetHandle; - using WriteOnlyHandle = kv::WriteableSetHandle; - using Handle = kv::SetHandle; + using ReadOnlyHandle = ccf::kv::ReadableSetHandle; + using WriteOnlyHandle = ccf::kv::WriteableSetHandle; + using Handle = ccf::kv::SetHandle; // Note: The type V of the value `std::optional` does not matter here. // The optional type is required to differentiate additions from deletions, // and to provide a consistent interface with the more generic `TypedMap`. - using Write = std::map>; + using Write = + std::map>; using MapHook = MapHook; using CommitHook = CommitHook; @@ -48,7 +50,7 @@ namespace kv using GetName::GetName; private: - static Write deserialise_write(const kv::untyped::Write& w) + static Write deserialise_write(const ccf::kv::untyped::Write& w) { Write typed_writes; for (const auto& [uk, opt_uv] : w) @@ -68,16 +70,16 @@ namespace kv } public: - static kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) + static ccf::kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { hook(v, deserialise_write(w)); }; } - static kv::untyped::MapHook wrap_map_hook(const MapHook& hook) + static ccf::kv::untyped::MapHook wrap_map_hook(const MapHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { return hook(v, deserialise_write(w)); }; } @@ -87,15 +89,16 @@ namespace kv typename K, template typename KSerialiser, - typename Unit = kv::serialisers::ZeroBlitUnitCreator> + typename Unit = ccf::kv::serialisers::ZeroBlitUnitCreator> using SetSerialisedWith = TypedSet, Unit>; template using JsonSerialisedSet = - SetSerialisedWith; + SetSerialisedWith; template - using RawCopySerialisedSet = TypedSet>; + using RawCopySerialisedSet = + TypedSet>; /** Short name for default-serialised sets, using JSON serialisers. Support * for custom types can be added through the DECLARE_JSON... macros. diff --git a/include/ccf/kv/set_handle.h b/include/ccf/kv/set_handle.h index 5c79bc423509..1d28a6c9e132 100644 --- a/include/ccf/kv/set_handle.h +++ b/include/ccf/kv/set_handle.h @@ -5,20 +5,20 @@ #include "ccf/kv/unit.h" #include "ccf/kv/untyped_map_handle.h" -namespace kv +namespace ccf::kv { - /** Grants read access to a @c kv::Set, as part of a @c kv::Tx. + /** Grants read access to a @c ccf::kv::Set, as part of a @c ccf::kv::Tx. */ template class ReadableSetHandle { protected: - kv::untyped::MapHandle& read_handle; + ccf::kv::untyped::MapHandle& read_handle; public: using KeyType = K; - ReadableSetHandle(kv::untyped::MapHandle& uh) : read_handle(uh) {} + ReadableSetHandle(ccf::kv::untyped::MapHandle& uh) : read_handle(uh) {} /** Test whether a key is present in the set. * @@ -37,7 +37,7 @@ namespace kv /** Test whether a key's presence is globally committed, meaning it has been * replciated and acknowledged by consensus protocol. * - * @see kv::ReadableMapHandle::get_globally_committed + * @see ccf::kv::ReadableMapHandle::get_globally_committed * * @param key Key to test * @@ -53,7 +53,7 @@ namespace kv * * Returns nullopt if the key is not present. * - * @see kv::ReadableMapHandle::get_version_of_previous_write + * @see ccf::kv::ReadableMapHandle::get_version_of_previous_write * * @param key Key to read * @@ -68,7 +68,7 @@ namespace kv /** Iterate over all entries in this set. * - * @see kv::ReadableMapHandle::foreach + * @see ccf::kv::ReadableMapHandle::foreach * * @tparam F Functor type. Should usually be derived implicitly from f * @param f Functor instance, taking (const K& k) and returning a @@ -79,8 +79,8 @@ namespace kv void foreach(F&& f) { auto g = [&]( - const kv::serialisers::SerialisedEntry& k_rep, - const kv::serialisers::SerialisedEntry&) { + const ccf::kv::serialisers::SerialisedEntry& k_rep, + const ccf::kv::serialisers::SerialisedEntry&) { return f(KSerialiser::from_serialised(k_rep)); }; read_handle.foreach(g); @@ -100,16 +100,16 @@ namespace kv } }; - /** Grants write access to a @c kv::Set, as part of a @c kv::Tx. + /** Grants write access to a @c ccf::kv::Set, as part of a @c ccf::kv::Tx. */ template class WriteableSetHandle { protected: - kv::untyped::MapHandle& write_handle; + ccf::kv::untyped::MapHandle& write_handle; public: - WriteableSetHandle(kv::untyped::MapHandle& uh) : write_handle(uh) {} + WriteableSetHandle(ccf::kv::untyped::MapHandle& uh) : write_handle(uh) {} /** Insert an element into this set. * @@ -143,24 +143,26 @@ namespace kv } }; - /** Grants read and write access to a @c kv::Set, as part of a @c kv::Tx. + /** Grants read and write access to a @c ccf::kv::Set, as part of a @c + * ccf::kv::Tx. * - * @see kv::ReadableSetHandle - * @see kv::WriteableSetHandle + * @see ccf::kv::ReadableSetHandle + * @see ccf::kv::WriteableSetHandle */ template - class SetHandle : public kv::AbstractHandle, + class SetHandle : public ccf::kv::AbstractHandle, public ReadableSetHandle, public WriteableSetHandle { protected: - kv::untyped::MapHandle untyped_handle; + ccf::kv::untyped::MapHandle untyped_handle; using ReadableBase = ReadableSetHandle; using WriteableBase = WriteableSetHandle; public: - SetHandle(kv::untyped::ChangeSet& changes, const std::string& map_name) : + SetHandle( + ccf::kv::untyped::ChangeSet& changes, const std::string& map_name) : ReadableBase(untyped_handle), WriteableBase(untyped_handle), untyped_handle(changes, map_name) diff --git a/include/ccf/kv/unit.h b/include/ccf/kv/unit.h index 0470c036a75f..08d00a47f7f7 100644 --- a/include/ccf/kv/unit.h +++ b/include/ccf/kv/unit.h @@ -4,23 +4,23 @@ #include "ccf/kv/serialisers/serialised_entry.h" -namespace kv::serialisers +namespace ccf::kv::serialisers { - // Unit serialisations are used as a utility type to convert kv::Maps to - // kv::Maps and kv::Sets. Specifically, these are implemented as wrappers so - // that kv::Value is essentially kv::Map, and kv::Set is - // kv::Map. - // This is used as a template parameter allowing the caller to specify what - // value is inserted into the ledger. + // Unit serialisations are used as a utility type to convert ccf::kv::Maps to + // ccf::kv::Maps and ccf::kv::Sets. Specifically, these are implemented as + // wrappers so that ccf::kv::Value is essentially ccf::kv::Map, + // and ccf::kv::Set is ccf::kv::Map. This is used as a template + // parameter allowing the caller to specify what value is inserted into the + // ledger. // This is the default UnitCreator, returning 8 null bytes for compatibility // with old ledgers (where Values were previously Maps with a single entry // at key 0, serialised as a uint64_t) struct ZeroBlitUnitCreator { - static kv::serialisers::SerialisedEntry get() + static ccf::kv::serialisers::SerialisedEntry get() { - kv::serialisers::SerialisedEntry e; + ccf::kv::serialisers::SerialisedEntry e; e.assign(sizeof(uint64_t), 0u); return e; } @@ -28,7 +28,7 @@ namespace kv::serialisers struct EmptyUnitCreator { - static kv::serialisers::SerialisedEntry get() + static ccf::kv::serialisers::SerialisedEntry get() { return {}; } diff --git a/include/ccf/kv/untyped.h b/include/ccf/kv/untyped.h index 2c3b5ed0eb0f..ddd7c601c97d 100644 --- a/include/ccf/kv/untyped.h +++ b/include/ccf/kv/untyped.h @@ -8,13 +8,13 @@ #include #include -namespace kv::untyped +namespace ccf::kv::untyped { // nullopt values represent deletions using Write = std::map< - kv::serialisers::SerialisedEntry, - std::optional>; + ccf::kv::serialisers::SerialisedEntry, + std::optional>; - using CommitHook = kv::CommitHook; - using MapHook = kv::MapHook; + using CommitHook = ccf::kv::CommitHook; + using MapHook = ccf::kv::MapHook; } \ No newline at end of file diff --git a/include/ccf/kv/untyped_map_diff.h b/include/ccf/kv/untyped_map_diff.h index ee31577467d6..cf9b68c10e7d 100644 --- a/include/ccf/kv/untyped_map_diff.h +++ b/include/ccf/kv/untyped_map_diff.h @@ -11,15 +11,15 @@ #include #include -namespace kv::untyped +namespace ccf::kv::untyped { struct ChangeSet; - class MapDiff : public kv::AbstractHandle + class MapDiff : public ccf::kv::AbstractHandle { public: - using KeyType = kv::serialisers::SerialisedEntry; - using ValueType = kv::serialisers::SerialisedEntry; + using KeyType = ccf::kv::serialisers::SerialisedEntry; + using ValueType = ccf::kv::serialisers::SerialisedEntry; using ElementVisitor = std::function& V)>; @@ -28,13 +28,13 @@ namespace kv::untyped std::function& V)>; protected: - kv::untyped::Write& writes; + ccf::kv::untyped::Write& writes; std::string map_name; void foreach_(const ElementVisitorWithEarlyOut& fn); public: - MapDiff(kv::untyped::ChangeSet& cs, const std::string& map_name); + MapDiff(ccf::kv::untyped::ChangeSet& cs, const std::string& map_name); std::optional> get(const KeyType& key); diff --git a/include/ccf/kv/untyped_map_handle.h b/include/ccf/kv/untyped_map_handle.h index 1e3e44caf032..77d0f96edfe7 100644 --- a/include/ccf/kv/untyped_map_handle.h +++ b/include/ccf/kv/untyped_map_handle.h @@ -10,17 +10,17 @@ #include #include -namespace kv::untyped +namespace ccf::kv::untyped { struct ChangeSet; - class MapHandle : public kv::AbstractHandle + class MapHandle : public ccf::kv::AbstractHandle { public: // Expose these types so that other code can use them as MyTx::KeyType or // MyMap::MapHandle::KeyType, templated on the MapHandle or Map type - using KeyType = kv::serialisers::SerialisedEntry; - using ValueType = kv::serialisers::SerialisedEntry; + using KeyType = ccf::kv::serialisers::SerialisedEntry; + using ValueType = ccf::kv::serialisers::SerialisedEntry; using ElementVisitor = std::function; @@ -29,7 +29,7 @@ namespace kv::untyped std::function; protected: - kv::untyped::ChangeSet& tx_changes; + ccf::kv::untyped::ChangeSet& tx_changes; std::string map_name; /** Get pointer to current value if this key exists, else nullptr if it does @@ -45,7 +45,7 @@ namespace kv::untyped const ElementVisitorWithEarlyOut& fn, bool always_consider_writes); public: - MapHandle(kv::untyped::ChangeSet& cs, const std::string& map_name); + MapHandle(ccf::kv::untyped::ChangeSet& cs, const std::string& map_name); std::string get_name_of_map() const; diff --git a/include/ccf/kv/value.h b/include/ccf/kv/value.h index cfc98ee92814..61aa8466149b 100644 --- a/include/ccf/kv/value.h +++ b/include/ccf/kv/value.h @@ -9,7 +9,7 @@ #include "ccf/kv/untyped.h" #include "ccf/kv/value_handle.h" -namespace kv +namespace ccf::kv { /** Defines the schema of a single-valued type accessed by a @c ccf::Tx. This * value type is a container for an optional single element of type V. This @@ -20,19 +20,20 @@ namespace kv * this V is serialised and deserialised, so it may be written to the ledger * and replicated by the consensus algorithm. * - * This is implemented as a @c kv::Map from Unit to V, and the serialisation - * of the unit key is overridable with the Unit template parameter. + * This is implemented as a @c ccf::kv::Map from Unit to V, and the + * serialisation of the unit key is overridable with the Unit template + * parameter. */ template < typename V, typename VSerialiser, - typename Unit = kv::serialisers::ZeroBlitUnitCreator> + typename Unit = ccf::kv::serialisers::ZeroBlitUnitCreator> class TypedValue : public GetName { public: - using ReadOnlyHandle = kv::ReadableValueHandle; - using WriteOnlyHandle = kv::WriteableValueHandle; - using Handle = kv::ValueHandle; + using ReadOnlyHandle = ccf::kv::ReadableValueHandle; + using WriteOnlyHandle = ccf::kv::WriteableValueHandle; + using Handle = ccf::kv::ValueHandle; using Write = std::optional; using MapHook = MapHook; @@ -43,13 +44,13 @@ namespace kv using GetName::GetName; - static kv::serialisers::SerialisedEntry create_unit() + static ccf::kv::serialisers::SerialisedEntry create_unit() { return Unit::get(); } private: - static Write deserialise_write(const kv::untyped::Write& w) + static Write deserialise_write(const ccf::kv::untyped::Write& w) { assert(w.size() == 1); // Value contains only one element const auto& value = w.begin()->second; @@ -61,16 +62,16 @@ namespace kv } public: - static kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) + static ccf::kv::untyped::CommitHook wrap_commit_hook(const CommitHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { hook(v, deserialise_write(w)); }; } - static kv::untyped::MapHook wrap_map_hook(const MapHook& hook) + static ccf::kv::untyped::MapHook wrap_map_hook(const MapHook& hook) { - return [hook](Version v, const kv::untyped::Write& w) { + return [hook](Version v, const ccf::kv::untyped::Write& w) { return hook(v, deserialise_write(w)); }; } @@ -80,16 +81,16 @@ namespace kv typename V, template typename VSerialiser, - typename Unit = kv::serialisers::ZeroBlitUnitCreator> + typename Unit = ccf::kv::serialisers::ZeroBlitUnitCreator> using ValueSerialisedWith = TypedValue, Unit>; template using JsonSerialisedValue = - ValueSerialisedWith; + ValueSerialisedWith; template using RawCopySerialisedValue = - TypedValue>; + TypedValue>; /** Short name for default-serialised values, using JSON serialisers. Support * for custom types can be added through the DECLARE_JSON... macros. diff --git a/include/ccf/kv/value_handle.h b/include/ccf/kv/value_handle.h index 4848594a46b6..a4f587fece1a 100644 --- a/include/ccf/kv/value_handle.h +++ b/include/ccf/kv/value_handle.h @@ -5,20 +5,20 @@ #include "ccf/kv/unit.h" #include "ccf/kv/untyped_map_handle.h" -namespace kv +namespace ccf::kv { - /** Grants read access to a @c kv::Value, as part of a @c kv::Tx. + /** Grants read access to a @c ccf::kv::Value, as part of a @c ccf::kv::Tx. */ template class ReadableValueHandle { protected: - kv::untyped::MapHandle& read_handle; + ccf::kv::untyped::MapHandle& read_handle; public: using ValueType = V; - ReadableValueHandle(kv::untyped::MapHandle& uh) : read_handle(uh) {} + ReadableValueHandle(ccf::kv::untyped::MapHandle& uh) : read_handle(uh) {} /** Get the stored value. * @@ -73,7 +73,7 @@ namespace kv /** Get version when this value was last written to, by a previous * transaction. * - * @see kv::ReadableMapHandle::get_version_of_previous_write + * @see ccf::kv::ReadableMapHandle::get_version_of_previous_write * * @return Optional containing version of applied transaction which last * wrote to this value, or nullopt if such a version does not exist @@ -84,16 +84,16 @@ namespace kv } }; - /** Grants write access to a @c kv::Value, as part of a @c kv::Tx. + /** Grants write access to a @c ccf::kv::Value, as part of a @c ccf::kv::Tx. */ template class WriteableValueHandle { protected: - kv::untyped::MapHandle& write_handle; + ccf::kv::untyped::MapHandle& write_handle; public: - WriteableValueHandle(kv::untyped::MapHandle& uh) : write_handle(uh) {} + WriteableValueHandle(ccf::kv::untyped::MapHandle& uh) : write_handle(uh) {} /** Modify this value. * @@ -116,10 +116,11 @@ namespace kv } }; - /** Grants read and write access to a @c kv::Value, as part of a @c kv::Tx. + /** Grants read and write access to a @c ccf::kv::Value, as part of a @c + * ccf::kv::Tx. * - * @see kv::ReadableValueHandle - * @see kv::WriteableValueHandle + * @see ccf::kv::ReadableValueHandle + * @see ccf::kv::WriteableValueHandle */ template class ValueHandle : public AbstractHandle, @@ -127,13 +128,14 @@ namespace kv public WriteableValueHandle { protected: - kv::untyped::MapHandle untyped_handle; + ccf::kv::untyped::MapHandle untyped_handle; using ReadableBase = ReadableValueHandle; using WriteableBase = WriteableValueHandle; public: - ValueHandle(kv::untyped::ChangeSet& changes, const std::string& map_name) : + ValueHandle( + ccf::kv::untyped::ChangeSet& changes, const std::string& map_name) : ReadableBase(untyped_handle), WriteableBase(untyped_handle), untyped_handle(changes, map_name) diff --git a/include/ccf/kv/version.h b/include/ccf/kv/version.h index 2a6c07fd519f..2fea6939539e 100644 --- a/include/ccf/kv/version.h +++ b/include/ccf/kv/version.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -namespace kv +namespace ccf::kv { // Version indexes modifications to the local kv store. using Version = uint64_t; diff --git a/include/ccf/node/acme_subsystem_interface.h b/include/ccf/node/acme_subsystem_interface.h index aef5e1d03852..26008848444d 100644 --- a/include/ccf/node/acme_subsystem_interface.h +++ b/include/ccf/node/acme_subsystem_interface.h @@ -28,7 +28,7 @@ namespace ccf virtual std::optional config( const NodeInfoNetwork::RpcInterfaceID& id) = 0; - virtual crypto::Pem network_cert() = 0; + virtual ccf::crypto::Pem network_cert() = 0; virtual void install_challenge_handler( const ccf::NodeInfoNetwork::RpcInterfaceID& interface_id, diff --git a/include/ccf/node/quote.h b/include/ccf/node/quote.h index 140f39d9bc19..f499857d7ead 100644 --- a/include/ccf/node/quote.h +++ b/include/ccf/node/quote.h @@ -36,7 +36,7 @@ namespace ccf static std::optional get_host_data(const QuoteInfo& quote_info); static QuoteVerificationResult verify_quote_against_store( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement); diff --git a/include/ccf/node/rpc_context_impl.h b/include/ccf/node/rpc_context_impl.h index f5343665e9f0..b5524953e2bb 100644 --- a/include/ccf/node/rpc_context_impl.h +++ b/include/ccf/node/rpc_context_impl.h @@ -79,7 +79,7 @@ namespace ccf const std::optional>& details = std::nullopt) override { - auto content_type = get_request_header(http::headers::CONTENT_TYPE); + auto content_type = get_request_header(ccf::http::headers::CONTENT_TYPE); if ( content_type.has_value() && content_type.value() == http::headervalues::contenttype::GRPC) @@ -111,7 +111,8 @@ namespace ccf set_response_status(status); set_response_body(std::vector(s.begin(), s.end())); set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); } void set_grpc_error(grpc_status grpc_status, std::string&& msg) @@ -123,7 +124,8 @@ namespace ccf set_response_status(HTTP_STATUS_OK); set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::GRPC); set_response_trailer(grpc::make_status_trailer(grpc_status)); set_response_trailer(grpc::make_message_trailer(msg)); } diff --git a/include/ccf/node/startup_config.h b/include/ccf/node/startup_config.h index 8dd762f37072..5d4fc0f893a2 100644 --- a/include/ccf/node/startup_config.h +++ b/include/ccf/node/startup_config.h @@ -29,7 +29,7 @@ struct CCFConfig { std::string subject_name = "CN=CCF Node"; std::vector subject_alt_names = {}; - crypto::CurveID curve_id = crypto::CurveID::SECP384R1; + ccf::crypto::CurveID curve_id = ccf::crypto::CurveID::SECP384R1; size_t initial_validity_days = 1; bool operator==(const NodeCertificateInfo&) const = default; diff --git a/include/ccf/node_context.h b/include/ccf/node_context.h index 29d26209572e..01d903e0c5d8 100644 --- a/include/ccf/node_context.h +++ b/include/ccf/node_context.h @@ -6,7 +6,7 @@ #include "ccf/indexing/indexer_interface.h" #include "ccf/node_subsystem_interface.h" -namespace ccfapp +namespace ccf { struct AbstractNodeContext { @@ -66,7 +66,7 @@ namespace ccfapp return {}; } - virtual crypto::Pem get_self_signed_certificate() const + virtual ccf::crypto::Pem get_self_signed_certificate() const { return {}; } diff --git a/include/ccf/odata_error.h b/include/ccf/odata_error.h index 19e9ea3ba1b3..4209b279a7e9 100644 --- a/include/ccf/odata_error.h +++ b/include/ccf/odata_error.h @@ -105,6 +105,7 @@ namespace ccf ERROR(TransactionPendingOrUnknown) ERROR(TransactionInvalid) ERROR(PrimaryNotFound) + ERROR(BackupNotFound) ERROR(RequestAlreadyForwarded) ERROR(NodeNotRetiredCommitted) ERROR(SessionConsistencyLost) diff --git a/include/ccf/pal/attestation.h b/include/ccf/pal/attestation.h index 2891d73f7033..c3a7ad859e00 100644 --- a/include/ccf/pal/attestation.h +++ b/include/ccf/pal/attestation.h @@ -78,7 +78,7 @@ namespace ccf::pal report_data = SnpAttestationReportData(quote.report_data); measurement = SnpAttestationMeasurement(quote.measurement); - auto certificates = crypto::split_x509_cert_bundle(std::string_view( + auto certificates = ccf::crypto::split_x509_cert_bundle(std::string_view( reinterpret_cast(quote_info.endorsements.data()), quote_info.endorsements.size())); if (certificates.size() != 3) @@ -90,7 +90,7 @@ namespace ccf::pal auto sev_version_certificate = certificates[1]; auto root_certificate = certificates[2]; - auto root_cert_verifier = crypto::make_verifier(root_certificate); + auto root_cert_verifier = ccf::crypto::make_verifier(root_certificate); if ( root_cert_verifier->public_key_pem().str() != @@ -109,7 +109,7 @@ namespace ccf::pal "self signed as expected"); } - auto chip_cert_verifier = crypto::make_verifier(chip_certificate); + auto chip_cert_verifier = ccf::crypto::make_verifier(chip_certificate); if (!chip_cert_verifier->verify_certificate( {&root_certificate, &sev_version_certificate})) { @@ -127,7 +127,7 @@ namespace ccf::pal } // Make ASN1 DER signature - auto quote_signature = crypto::ecdsa_sig_from_r_s( + auto quote_signature = ccf::crypto::ecdsa_sig_from_r_s( quote.signature.r, sizeof(quote.signature.r), quote.signature.s, diff --git a/include/ccf/pal/measurement.h b/include/ccf/pal/measurement.h index 3d4e91a00bb5..3c3a8c609c6f 100644 --- a/include/ccf/pal/measurement.h +++ b/include/ccf/pal/measurement.h @@ -143,7 +143,7 @@ namespace ccf::pal DECLARE_JSON_REQUIRED_FIELDS(PlatformAttestationMeasurement, data); } -namespace kv::serialisers +namespace ccf::kv::serialisers { template struct BlitSerialiser> diff --git a/include/ccf/pal/report_data.h b/include/ccf/pal/report_data.h index 8cc57b4e1cce..6fe8a7e6e087 100644 --- a/include/ccf/pal/report_data.h +++ b/include/ccf/pal/report_data.h @@ -53,7 +53,7 @@ namespace ccf::pal PlatformAttestationReportData() = default; - PlatformAttestationReportData(const crypto::Sha256Hash& hash) : + PlatformAttestationReportData(const ccf::crypto::Sha256Hash& hash) : data(hash.h.begin(), hash.h.end()) {} @@ -67,11 +67,11 @@ namespace ccf::pal return ds::to_hex(data); } - crypto::Sha256Hash to_sha256_hash() const + ccf::crypto::Sha256Hash to_sha256_hash() const { - std::span s( - data.data(), crypto::Sha256Hash::SIZE); - return crypto::Sha256Hash::from_span(s); + std::span s( + data.data(), ccf::crypto::Sha256Hash::SIZE); + return ccf::crypto::Sha256Hash::from_span(s); } }; } \ No newline at end of file diff --git a/include/ccf/receipt.h b/include/ccf/receipt.h index a9259cce88ac..31b13bfa53bf 100644 --- a/include/ccf/receipt.h +++ b/include/ccf/receipt.h @@ -22,12 +22,12 @@ namespace ccf // Signature over the root digest, signed by the identity described in cert std::vector signature = {}; - virtual crypto::Sha256Hash calculate_root() = 0; + virtual ccf::crypto::Sha256Hash calculate_root() = 0; ccf::NodeId node_id = {}; - crypto::Pem cert = {}; + ccf::crypto::Pem cert = {}; - std::vector service_endorsements = {}; + std::vector service_endorsements = {}; virtual bool is_signature_transaction() const = 0; }; @@ -42,7 +42,7 @@ namespace ccf public: struct Components { - crypto::Sha256Hash write_set_digest; + ccf::crypto::Sha256Hash write_set_digest; std::string commit_evidence; ccf::ClaimsDigest claims_digest; }; @@ -56,7 +56,7 @@ namespace ccf Right } direction; - crypto::Sha256Hash hash = {}; + ccf::crypto::Sha256Hash hash = {}; bool operator==(const ProofStep& other) const { @@ -68,7 +68,7 @@ namespace ccf // A merkle-tree path from the leaf digest to the signed root Proof proof = {}; - crypto::Sha256Hash calculate_root() override + ccf::crypto::Sha256Hash calculate_root() override { auto current = get_leaf_digest(); @@ -76,30 +76,31 @@ namespace ccf { if (element.direction == ProofStep::Left) { - current = crypto::Sha256Hash(element.hash, current); + current = ccf::crypto::Sha256Hash(element.hash, current); } else { - current = crypto::Sha256Hash(current, element.hash); + current = ccf::crypto::Sha256Hash(current, element.hash); } } return current; } - crypto::Sha256Hash get_leaf_digest() + ccf::crypto::Sha256Hash get_leaf_digest() { - crypto::Sha256Hash ce_dgst(leaf_components.commit_evidence); + ccf::crypto::Sha256Hash ce_dgst(leaf_components.commit_evidence); if (!leaf_components.claims_digest.empty()) { - return crypto::Sha256Hash( + return ccf::crypto::Sha256Hash( leaf_components.write_set_digest, ce_dgst, leaf_components.claims_digest.value()); } else { - return crypto::Sha256Hash(leaf_components.write_set_digest, ce_dgst); + return ccf::crypto::Sha256Hash( + leaf_components.write_set_digest, ce_dgst); } } @@ -114,9 +115,9 @@ namespace ccf class SignatureReceipt : public Receipt { public: - crypto::Sha256Hash signed_root = {}; + ccf::crypto::Sha256Hash signed_root = {}; - crypto::Sha256Hash calculate_root() override + ccf::crypto::Sha256Hash calculate_root() override { return signed_root; }; diff --git a/include/ccf/research/create_tx_claims_digest.h b/include/ccf/research/create_tx_claims_digest.h index f11b5092a6ce..bb0003717206 100644 --- a/include/ccf/research/create_tx_claims_digest.h +++ b/include/ccf/research/create_tx_claims_digest.h @@ -5,7 +5,7 @@ #include #include -namespace ccfapp +namespace ccf { /** Can be optionally implemented by the application to set the claims digest * for the initial network create transaction. @@ -13,5 +13,5 @@ namespace ccfapp * @return an optional claims digest */ std::optional get_create_tx_claims_digest( - kv::ReadOnlyTx& tx) + ccf::kv::ReadOnlyTx& tx) } diff --git a/include/ccf/research/custom_protocol_subsystem_interface.h b/include/ccf/research/custom_protocol_subsystem_interface.h index cff23ce8598a..6bcc136f32cc 100644 --- a/include/ccf/research/custom_protocol_subsystem_interface.h +++ b/include/ccf/research/custom_protocol_subsystem_interface.h @@ -46,7 +46,7 @@ namespace ccf struct Essentials { ringbuffer::WriterPtr writer; - std::shared_ptr tx; + std::shared_ptr tx; std::shared_ptr ctx; }; diff --git a/include/ccf/research/grpc_status.h b/include/ccf/research/grpc_status.h index bca1820845eb..ee75cd6d8437 100644 --- a/include/ccf/research/grpc_status.h +++ b/include/ccf/research/grpc_status.h @@ -97,7 +97,7 @@ static grpc_status http_status_to_grpc(http_status s) namespace ccf::grpc { static const http::HeaderMap default_response_headers = { - {http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC}}; + {ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::GRPC}}; static constexpr auto TRAILER_STATUS = "grpc-status"; static constexpr auto TRAILER_MESSAGE = "grpc-message"; diff --git a/include/ccf/rest_verb.h b/include/ccf/rest_verb.h index fc2bdebb4169..15e790f5e872 100644 --- a/include/ccf/rest_verb.h +++ b/include/ccf/rest_verb.h @@ -85,7 +85,7 @@ namespace ccf inline void to_json(nlohmann::json& j, const RESTVerb& verb) { std::string s(verb.c_str()); - nonstd::to_lower(s); + ccf::nonstd::to_lower(s); j = s; } @@ -98,7 +98,7 @@ namespace ccf } std::string s = j.get(); - nonstd::to_upper(s); + ccf::nonstd::to_upper(s); verb = RESTVerb(s.c_str()); } diff --git a/include/ccf/rpc_context.h b/include/ccf/rpc_context.h index 293c4b0a0fca..0292b731ae87 100644 --- a/include/ccf/rpc_context.h +++ b/include/ccf/rpc_context.h @@ -80,7 +80,7 @@ namespace ccf virtual const ccf::RESTVerb& get_request_verb() const = 0; virtual std::string get_request_path() const = 0; virtual std::string get_method() const = 0; - virtual std::shared_ptr get_responder() const = 0; + virtual std::shared_ptr get_responder() const = 0; /// Returns a map of all PathParams parsed out of the original query path. /// For instance if this endpoint was installed at `/foo/{name}/{age}`, and diff --git a/include/ccf/serdes.h b/include/ccf/serdes.h deleted file mode 100644 index 1fa9d34bd180..000000000000 --- a/include/ccf/serdes.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. -#pragma once - -#include "ccf/ds/json.h" - -#include -#include - -namespace serdes -{ - enum class Pack - { - Text, - MsgPack - }; - - inline std::vector pack(const nlohmann::json& j, Pack pack) - { - switch (pack) - { - case Pack::Text: - { - auto s = j.dump(); - return std::vector{s.begin(), s.end()}; - } - - case Pack::MsgPack: - return nlohmann::json::to_msgpack(j); - } - - throw std::logic_error("Invalid serdes::Pack"); - } - - inline nlohmann::json unpack(const std::vector& data, Pack pack) - { - switch (pack) - { - case Pack::Text: - return nlohmann::json::parse(data); - - case Pack::MsgPack: - return nlohmann::json::from_msgpack(data); - } - - throw std::logic_error("Invalid serdes::Pack"); - } - - inline std::optional detect_pack( - const std::vector& input) - { - if (input.size() == 0) - { - return std::nullopt; - } - - if (input[0] == '{') - { - return serdes::Pack::Text; - } - else - { - return serdes::Pack::MsgPack; - } - } -} diff --git a/include/ccf/service/blit_serialiser_pem.h b/include/ccf/service/blit_serialiser_pem.h index 65347b1fad42..3580b17af95b 100644 --- a/include/ccf/service/blit_serialiser_pem.h +++ b/include/ccf/service/blit_serialiser_pem.h @@ -5,20 +5,20 @@ #include "ccf/crypto/pem.h" #include "ccf/kv/serialisers/blit_serialiser.h" -namespace kv::serialisers +namespace ccf::kv::serialisers { template <> - struct BlitSerialiser + struct BlitSerialiser { - static SerialisedEntry to_serialised(const crypto::Pem& pem) + static SerialisedEntry to_serialised(const ccf::crypto::Pem& pem) { const auto& data = pem.raw(); return SerialisedEntry(data.begin(), data.end()); } - static crypto::Pem from_serialised(const SerialisedEntry& data) + static ccf::crypto::Pem from_serialised(const SerialisedEntry& data) { - return crypto::Pem(data.data(), data.size()); + return ccf::crypto::Pem(data.data(), data.size()); } }; } \ No newline at end of file diff --git a/include/ccf/service/map.h b/include/ccf/service/map.h index d797a8ae9215..1ce6d1d10e79 100644 --- a/include/ccf/service/map.h +++ b/include/ccf/service/map.h @@ -12,24 +12,24 @@ namespace ccf // - The raw key is trivially and cheaply deserialisable. // - The JSON value can conveniently be audited offline. // Note: Maps which include large values (e.g. certificate or serialised - // Merkle tree) can use the `kv::RawCopySerialisedMap` type to maximise + // Merkle tree) can use the `ccf::kv::RawCopySerialisedMap` type to maximise // performance. template - using ServiceMap = kv::MapSerialisedWith< + using ServiceMap = ccf::kv::MapSerialisedWith< K, V, - kv::serialisers::BlitSerialiser, - kv::serialisers::JsonSerialiser>; + ccf::kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::JsonSerialiser>; template - using ServiceValue = kv::ValueSerialisedWith< + using ServiceValue = ccf::kv::ValueSerialisedWith< V, - kv::serialisers::JsonSerialiser, - kv::serialisers::ZeroBlitUnitCreator>; + ccf::kv::serialisers::JsonSerialiser, + ccf::kv::serialisers::ZeroBlitUnitCreator>; template - using ServiceSet = kv::SetSerialisedWith< + using ServiceSet = ccf::kv::SetSerialisedWith< K, - kv::serialisers::BlitSerialiser, - kv::serialisers::ZeroBlitUnitCreator>; + ccf::kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::ZeroBlitUnitCreator>; } \ No newline at end of file diff --git a/include/ccf/service/node_info.h b/include/ccf/service/node_info.h index 68e3befecde5..5ef730475d45 100644 --- a/include/ccf/service/node_info.h +++ b/include/ccf/service/node_info.h @@ -31,13 +31,13 @@ namespace ccf /// Node enclave quote QuoteInfo quote_info; /// Node encryption public key, used to distribute ledger re-keys. - crypto::Pem encryption_pub_key; + ccf::crypto::Pem encryption_pub_key; /// Node status NodeStatus status = NodeStatus::PENDING; /** Set to the seqno of the latest ledger secret at the time the node is trusted */ - std::optional ledger_secret_seqno = std::nullopt; + std::optional ledger_secret_seqno = std::nullopt; /// Code identity for the node std::optional code_digest = std::nullopt; @@ -47,10 +47,10 @@ namespace ccf */ /// Node certificate signing request - std::optional certificate_signing_request = std::nullopt; + std::optional certificate_signing_request = std::nullopt; /// Public key - std::optional public_key = std::nullopt; + std::optional public_key = std::nullopt; /// Free-form user data, can be used to store operator correlation /// IDs/labels for the node for example @@ -64,7 +64,7 @@ namespace ccf * Node certificate. Only set for 1.x releases. Further releases record * node identity in `public_key` field. Service-endorsed certificate is * recorded in "public:ccf.nodes.endorsed_certificates" table */ - std::optional cert = std::nullopt; + std::optional cert = std::nullopt; /** Commit state for Retired state * diff --git a/include/ccf/service/node_info_network.h b/include/ccf/service/node_info_network.h index bd4d1f45be4e..56f54c0b82fc 100644 --- a/include/ccf/service/node_info_network.h +++ b/include/ccf/service/node_info_network.h @@ -118,6 +118,7 @@ namespace ccf struct Redirections { RedirectionResolverConfig to_primary; + RedirectionResolverConfig to_backup; bool operator==(const Redirections& other) const = default; }; @@ -165,7 +166,7 @@ namespace ccf NodeInfoNetwork_v2::NetInterface::Redirections); DECLARE_JSON_REQUIRED_FIELDS(NodeInfoNetwork_v2::NetInterface::Redirections); DECLARE_JSON_OPTIONAL_FIELDS( - NodeInfoNetwork_v2::NetInterface::Redirections, to_primary); + NodeInfoNetwork_v2::NetInterface::Redirections, to_primary, to_backup); DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(NodeInfoNetwork_v2::NetInterface); DECLARE_JSON_REQUIRED_FIELDS(NodeInfoNetwork_v2::NetInterface, bind_address); DECLARE_JSON_OPTIONAL_FIELDS( @@ -203,7 +204,7 @@ namespace ccf inline static std::pair split_net_address( const NodeInfoNetwork::NetAddress& addr) { - auto [host, port] = nonstd::rsplit_1(addr, ":"); + auto [host, port] = ccf::nonstd::rsplit_1(addr, ":"); return std::make_pair(std::string(host), std::string(port)); } diff --git a/include/ccf/service/signed_req.h b/include/ccf/service/signed_req.h index ea1039d3f86c..4ee3de849eac 100644 --- a/include/ccf/service/signed_req.h +++ b/include/ccf/service/signed_req.h @@ -21,7 +21,7 @@ namespace ccf std::vector request_body = {}; /// Hashing algorithm used to summarise content before signature - crypto::MDType md = crypto::MDType::NONE; + ccf::crypto::MDType md = ccf::crypto::MDType::NONE; /// Signer key id, if present in the request std::string key_id = {}; diff --git a/include/ccf/service/tables/acme_certificates.h b/include/ccf/service/tables/acme_certificates.h index 2388ac55b26f..b500a300218d 100644 --- a/include/ccf/service/tables/acme_certificates.h +++ b/include/ccf/service/tables/acme_certificates.h @@ -9,7 +9,7 @@ namespace ccf { // Maps each interface name to a certificate - using ACMECertificates = ServiceMap; + using ACMECertificates = ServiceMap; namespace Tables { diff --git a/include/ccf/service/tables/gov.h b/include/ccf/service/tables/gov.h index 04a7709a1ad0..0487c0d2188e 100644 --- a/include/ccf/service/tables/gov.h +++ b/include/ccf/service/tables/gov.h @@ -80,7 +80,7 @@ namespace ccf ProposalInfoSummary, votes, vote_failures, failure); using ProposalMap = - kv::RawCopySerialisedMap>; + ccf::kv::RawCopySerialisedMap>; using ProposalInfoMap = ServiceMap; namespace Tables diff --git a/include/ccf/service/tables/host_data.h b/include/ccf/service/tables/host_data.h index 44c7d8e92663..98c915417a2c 100644 --- a/include/ccf/service/tables/host_data.h +++ b/include/ccf/service/tables/host_data.h @@ -5,7 +5,7 @@ #include "ccf/crypto/sha256_hash.h" #include "ccf/service/map.h" -using HostData = crypto::Sha256Hash; +using HostData = ccf::crypto::Sha256Hash; using HostDataMetadata = std::string; // Optional raw (i.e. not base64-encoded) policy diff --git a/include/ccf/service/tables/jwt.h b/include/ccf/service/tables/jwt.h index 7bb1a3c9290a..f661b742971e 100644 --- a/include/ccf/service/tables/jwt.h +++ b/include/ccf/service/tables/jwt.h @@ -86,15 +86,16 @@ namespace ccf static constexpr auto JWT_PUBLIC_SIGNING_KEY_ISSUER = "public:ccf.gov.jwt.public_signing_key_issuer"; - using JwtPublicSigningKeys = kv::RawCopySerialisedMap; + using JwtPublicSigningKeys = + ccf::kv::RawCopySerialisedMap; using JwtPublicSigningKeyIssuer = - kv::RawCopySerialisedMap; + ccf::kv::RawCopySerialisedMap; } } struct JsonWebKeySet { - std::vector keys; + std::vector keys; bool operator!=(const JsonWebKeySet& rhs) const { diff --git a/include/ccf/service/tables/members.h b/include/ccf/service/tables/members.h index a4dbe7614ee8..74fb84475985 100644 --- a/include/ccf/service/tables/members.h +++ b/include/ccf/service/tables/members.h @@ -30,17 +30,17 @@ namespace ccf struct NewMember { - crypto::Pem cert; + ccf::crypto::Pem cert; // If encryption public key is set, the member is a recovery member - std::optional encryption_pub_key = std::nullopt; + std::optional encryption_pub_key = std::nullopt; nlohmann::json member_data = nullptr; NewMember() {} NewMember( - const crypto::Pem& cert_, - const std::optional& encryption_pub_key_ = std::nullopt, + const ccf::crypto::Pem& cert_, + const std::optional& encryption_pub_key_ = std::nullopt, const nlohmann::json& member_data_ = nullptr) : cert(cert_), encryption_pub_key(encryption_pub_key_), @@ -76,9 +76,9 @@ namespace ccf using MemberInfo = ServiceMap; - using MemberCerts = kv::RawCopySerialisedMap; + using MemberCerts = ccf::kv::RawCopySerialisedMap; using MemberPublicEncryptionKeys = - kv::RawCopySerialisedMap; + ccf::kv::RawCopySerialisedMap; namespace Tables { @@ -98,7 +98,8 @@ namespace ccf StateDigest() {} - StateDigest(const crypto::Sha256Hash& root) : state_digest(root.hex_str()) + StateDigest(const ccf::crypto::Sha256Hash& root) : + state_digest(root.hex_str()) {} }; DECLARE_JSON_TYPE(StateDigest) @@ -114,15 +115,16 @@ namespace ccf MemberAck() {} - MemberAck(const crypto::Sha256Hash& root) : StateDigest(root) {} + MemberAck(const ccf::crypto::Sha256Hash& root) : StateDigest(root) {} - MemberAck(const crypto::Sha256Hash& root, const SignedReq& signed_req_) : + MemberAck( + const ccf::crypto::Sha256Hash& root, const SignedReq& signed_req_) : StateDigest(root), signed_req(signed_req_) {} MemberAck( - const crypto::Sha256Hash& root, + const ccf::crypto::Sha256Hash& root, const std::vector& cose_sign1_req_) : StateDigest(root), cose_sign1_req(cose_sign1_req_) diff --git a/include/ccf/service/tables/modules.h b/include/ccf/service/tables/modules.h index 5ec0c3f19c2c..09bd07edd7d7 100644 --- a/include/ccf/service/tables/modules.h +++ b/include/ccf/service/tables/modules.h @@ -12,10 +12,10 @@ namespace ccf { using Module = std::string; - using Modules = kv::RawCopySerialisedMap; + using Modules = ccf::kv::RawCopySerialisedMap; using ModulesQuickJsBytecode = - kv::RawCopySerialisedMap>; - using ModulesQuickJsVersion = kv::RawCopySerialisedValue; + ccf::kv::RawCopySerialisedMap>; + using ModulesQuickJsVersion = ccf::kv::RawCopySerialisedValue; using InterpreterFlush = ServiceValue; namespace Tables diff --git a/include/ccf/service/tables/nodes.h b/include/ccf/service/tables/nodes.h index df4a9cf84f61..5f25c45de825 100644 --- a/include/ccf/service/tables/nodes.h +++ b/include/ccf/service/tables/nodes.h @@ -18,7 +18,7 @@ namespace ccf { using Nodes = ServiceMap; using NodeEndorsedCertificates = - kv::RawCopySerialisedMap; + ccf::kv::RawCopySerialisedMap; namespace Tables { @@ -30,17 +30,18 @@ namespace ccf inline NodeId compute_node_id_from_pubk_der( const std::vector& node_pubk_der) { - return crypto::Sha256Hash(node_pubk_der).hex_str(); + return ccf::crypto::Sha256Hash(node_pubk_der).hex_str(); } inline NodeId compute_node_id_from_cert_der( const std::vector& node_cert_der) { return compute_node_id_from_pubk_der( - crypto::public_key_der_from_cert(node_cert_der)); + ccf::crypto::public_key_der_from_cert(node_cert_der)); } - inline NodeId compute_node_id_from_kp(const crypto::KeyPairPtr& node_sign_kp) + inline NodeId compute_node_id_from_kp( + const ccf::crypto::KeyPairPtr& node_sign_kp) { return compute_node_id_from_pubk_der(node_sign_kp->public_key_der()); } diff --git a/include/ccf/service/tables/service.h b/include/ccf/service/tables/service.h index e31f7432c3d4..458e8275e8be 100644 --- a/include/ccf/service/tables/service.h +++ b/include/ccf/service/tables/service.h @@ -27,11 +27,12 @@ namespace ccf struct ServiceInfo { /// x.509 Service Certificate, as a PEM string - crypto::Pem cert; + ccf::crypto::Pem cert; /// Status of the service ServiceStatus status = ServiceStatus::OPENING; /// Version (seqno) of previous service identity (before the last recovery) - std::optional previous_service_identity_version = std::nullopt; + std::optional previous_service_identity_version = + std::nullopt; /// Number of disaster recoveries performed on this service std::optional recovery_count = std::nullopt; /// Free-form user data, can be used by members to store additional diff --git a/include/ccf/service/tables/users.h b/include/ccf/service/tables/users.h index a8cbaed4c837..4a74f54327e4 100644 --- a/include/ccf/service/tables/users.h +++ b/include/ccf/service/tables/users.h @@ -12,7 +12,7 @@ namespace ccf { struct NewUser { - crypto::Pem cert; + ccf::crypto::Pem cert; nlohmann::json user_data = nullptr; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(NewUser) @@ -28,7 +28,7 @@ namespace ccf DECLARE_JSON_TYPE(UserDetails) DECLARE_JSON_REQUIRED_FIELDS(UserDetails, user_data) - using UserCerts = kv::RawCopySerialisedMap; + using UserCerts = ccf::kv::RawCopySerialisedMap; using UserInfo = ServiceMap; namespace Tables diff --git a/include/ccf/tx.h b/include/ccf/tx.h index f5d873c6441a..288fd595669a 100644 --- a/include/ccf/tx.h +++ b/include/ccf/tx.h @@ -12,7 +12,7 @@ #include #include -namespace kv +namespace ccf::kv { class AbstractHandle; class AbstractMap; @@ -51,7 +51,7 @@ namespace kv OrderedChanges all_changes; - std::optional root_at_read_version = std::nullopt; + std::optional root_at_read_version = std::nullopt; void retain_change_set( const std::string& map_name, @@ -122,7 +122,7 @@ namespace kv virtual ~BaseTx(); - std::optional get_root_at_read_version() + std::optional get_root_at_read_version() { return root_at_read_version; } @@ -196,7 +196,7 @@ namespace kv * type-safety, prefer restricted handles returned by @c ro or @c wo where * possible, rather than the general @c rw. * - * @see kv::ReadOnlyTx + * @see ccf::kv::ReadOnlyTx */ class Tx : public ReadOnlyTx { diff --git a/js/ccf-app/package.json b/js/ccf-app/package.json index 6d0349f7d810..cc7ca38df1d9 100644 --- a/js/ccf-app/package.json +++ b/js/ccf-app/package.json @@ -29,7 +29,7 @@ "mocha": "^10.0.0", "node-forge": "^1.2.0", "ts-node": "^10.4.0", - "typedoc": "^0.25.0", + "typedoc": "^0.26.2", "typescript": "^5.0.2" } } diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp index 429b849e8a60..f5337627905f 100644 --- a/samples/apps/basic/basic.cpp +++ b/samples/apps/basic/basic.cpp @@ -18,13 +18,13 @@ using namespace nlohmann; namespace basicapp { - using RecordsMap = kv::Map>; + using RecordsMap = ccf::kv::Map>; static constexpr auto PRIVATE_RECORDS = "records"; class BasicHandlers : public ccf::UserEndpointRegistry { public: - BasicHandlers(ccfapp::AbstractNodeContext& context) : + BasicHandlers(ccf::AbstractNodeContext& context) : ccf::UserEndpointRegistry(context) { openapi_info.title = "CCF Basic App"; @@ -74,7 +74,8 @@ namespace basicapp { ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(record.value()); return; } @@ -109,10 +110,10 @@ namespace basicapp }; } -namespace ccfapp +namespace ccf { std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique(context); } diff --git a/samples/apps/logging/create_tx_claims_digest.cpp b/samples/apps/logging/create_tx_claims_digest.cpp index 7ed7a48bb358..b5b8d7a2aa74 100644 --- a/samples/apps/logging/create_tx_claims_digest.cpp +++ b/samples/apps/logging/create_tx_claims_digest.cpp @@ -5,10 +5,10 @@ #include #include -namespace ccfapp +namespace ccf { std::optional get_create_tx_claims_digest( - kv::ReadOnlyTx& tx) + ccf::kv::ReadOnlyTx& tx) { auto constitution = tx.ro(ccf::Tables::CONSTITUTION)->get(); diff --git a/samples/apps/logging/js/app.json b/samples/apps/logging/js/app.json index 09ab3f64c9b4..ecd7b61a6de9 100644 --- a/samples/apps/logging/js/app.json +++ b/samples/apps/logging/js/app.json @@ -65,6 +65,18 @@ "interpreter_reuse": { "key": "singleton_interpreter" } } }, + "/log/private/backup": { + "get": { + "js_module": "logging.js", + "js_function": "get_private", + "forwarding_required": "sometimes", + "redirection_strategy": "to_backup", + "authn_policies": ["jwt", "user_cert"], + "mode": "readonly", + "openapi": {}, + "interpreter_reuse": { "key": "singleton_interpreter" } + } + }, "/log/private/admin_only": { "post": { "js_module": "logging.js", @@ -193,6 +205,18 @@ "interpreter_reuse": { "key": "singleton_interpreter" } } }, + "/log/public/backup": { + "get": { + "js_module": "logging.js", + "js_function": "get_public", + "forwarding_required": "sometimes", + "redirection_strategy": "to_backup", + "authn_policies": ["jwt", "user_cert"], + "mode": "readonly", + "openapi": {}, + "interpreter_reuse": { "key": "singleton_interpreter" } + } + }, "/log/public/all": { "delete": { "js_module": "logging.js", diff --git a/samples/apps/logging/logging.cpp b/samples/apps/logging/logging.cpp index 7da07d138f85..78856d817765 100644 --- a/samples/apps/logging/logging.cpp +++ b/samples/apps/logging/logging.cpp @@ -28,7 +28,7 @@ using namespace nlohmann; namespace loggingapp { // SNIPPET: table_definition - using RecordsMap = kv::Map; + using RecordsMap = ccf::kv::Map; static constexpr auto PUBLIC_RECORDS = "public:records"; static constexpr auto PRIVATE_RECORDS = "records"; @@ -71,7 +71,7 @@ namespace loggingapp { public: std::unique_ptr authenticate( - kv::ReadOnlyTx&, + ccf::kv::ReadOnlyTx&, const std::shared_ptr& ctx, std::string& error_reason) override { @@ -169,7 +169,7 @@ namespace loggingapp {} void handle_committed_transaction( - const ccf::TxID& tx_id, const kv::ReadOnlyStorePtr& store) + const ccf::TxID& tx_id, const ccf::kv::ReadOnlyStorePtr& store) { std::lock_guard lock(txid_lock); auto tx_diff = store->create_tx_diff(); @@ -238,7 +238,7 @@ namespace loggingapp response += fmt::format( "\nThe caller is a user with ID: {}", user_cert_ident->user_id); - crypto::Pem user_cert; + ccf::crypto::Pem user_cert; if ( get_user_cert_v1(ctx.tx, user_cert_ident->user_id, user_cert) == ccf::ApiResult::OK) @@ -266,7 +266,7 @@ namespace loggingapp response += fmt::format( "\nThe caller is a member with ID: {}", member_cert_ident->member_id); - crypto::Pem member_cert; + ccf::crypto::Pem member_cert; if ( get_member_cert_v1( ctx.tx, member_cert_ident->member_id, member_cert) == @@ -367,9 +367,9 @@ namespace loggingapp static std::optional get_scope(auto& ctx) { const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_string; - return http::get_query_value_opt( + return ccf::http::get_query_value_opt( parsed_query, "scope", error_string); } @@ -441,7 +441,7 @@ namespace loggingapp } public: - LoggerHandlers(ccfapp::AbstractNodeContext& context) : + LoggerHandlers(ccf::AbstractNodeContext& context) : ccf::UserEndpointRegistry(context), record_public_params_schema(nlohmann::json::parse(j_record_public_in)), record_public_result_schema(nlohmann::json::parse(j_record_public_out)), @@ -460,6 +460,12 @@ namespace loggingapp PUBLIC_RECORDS, context, 10000, 20); context.get_indexing_strategies().install_strategy(index_per_public_key); + // According to manual obvervation it's enough to start evicting old + // requests on historical perf test, but not too small to get stuck + // because of a single request being larget than the cache. + constexpr size_t cache_limit = 1024 * 1024 * 10; // MB + context.get_historical_state().set_soft_cache_limit(cache_limit); + const ccf::AuthnPolicies auth_policies = { ccf::jwt_auth_policy, ccf::user_cert_auth_policy, @@ -528,11 +534,11 @@ namespace loggingapp records_handle->put(in.id, in.msg); const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; std::string fail; - http::get_query_value(parsed_query, "fail", fail, error_reason); + ccf::http::get_query_value(parsed_query, "fail", fail, error_reason); auto out = std::make_shared(); out->success = true; @@ -560,11 +566,11 @@ namespace loggingapp auto get = [this](auto& ctx, nlohmann::json&&) { // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { return ccf::make_error( HTTP_STATUS_BAD_REQUEST, @@ -599,6 +605,16 @@ namespace loggingapp .install(); // SNIPPET_END: install_get + make_read_only_endpoint( + "/log/private/backup", + HTTP_GET, + ccf::json_read_only_adapter(get), + auth_policies) + .set_redirection_strategy(ccf::endpoints::RedirectionStrategy::ToBackup) + .set_auto_schema() + .add_query_parameter("id") + .install(); + // install the committed index and tell the historical fetcher to keep // track of deleted keys too, so that the index can observe the deleted // keys. @@ -665,11 +681,11 @@ namespace loggingapp auto get_committed = [this](auto& ctx) { // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { auto response = nlohmann::json{{ "error", @@ -716,11 +732,11 @@ namespace loggingapp auto remove = [this](auto& ctx, nlohmann::json&&) { // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { return ccf::make_error( HTTP_STATUS_BAD_REQUEST, @@ -800,7 +816,7 @@ namespace loggingapp auto current_value = records_handle->get(id); if (current_value.has_value()) { - crypto::Sha256Hash value_digest(current_value.value()); + ccf::crypto::Sha256Hash value_digest(current_value.value()); auto etag = value_digest.hex_str(); // On a POST operation, If-Match failing or If-None-Match passing @@ -843,7 +859,7 @@ namespace loggingapp CCF_APP_INFO("Storing {} = {}", id, in.msg); // SNIPPET_START: public_table_post_etag - crypto::Sha256Hash value_digest(in.msg); + ccf::crypto::Sha256Hash value_digest(in.msg); // Succesful calls set an ETag ctx.rpc_ctx->set_response_header("ETag", value_digest.hex_str()); // SNIPPET_END: public_table_post_etag @@ -863,11 +879,11 @@ namespace loggingapp auto get_public = [this](auto& ctx, nlohmann::json&&) { // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { return ccf::make_error( HTTP_STATUS_BAD_REQUEST, @@ -895,7 +911,7 @@ namespace loggingapp // If a record is present, compute an Entity Tag, and apply // If-Match and If-None-Match. - crypto::Sha256Hash value_digest(record.value()); + ccf::crypto::Sha256Hash value_digest(record.value()); const auto etag = value_digest.hex_str(); if (match_headers.if_match.has_value()) @@ -943,14 +959,24 @@ namespace loggingapp .add_query_parameter("id") .install(); + make_read_only_endpoint( + "/log/public/backup", + HTTP_GET, + ccf::json_read_only_adapter(get_public), + auth_policies) + .set_redirection_strategy(ccf::endpoints::RedirectionStrategy::ToBackup) + .set_auto_schema() + .add_query_parameter("id") + .install(); + auto remove_public = [this](auto& ctx, nlohmann::json&&) { // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { return ccf::make_error( HTTP_STATUS_BAD_REQUEST, @@ -980,7 +1006,7 @@ namespace loggingapp { // If a Match header is present, we need to compute the ETag // to resolve the constraints - crypto::Sha256Hash value_digest(current_value.value()); + ccf::crypto::Sha256Hash value_digest(current_value.value()); const auto etag = value_digest.hex_str(); if (match_headers.if_match.has_value()) @@ -1073,7 +1099,8 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(true).dump()); }; make_endpoint( @@ -1172,9 +1199,9 @@ namespace loggingapp // SNIPPET_START: log_record_text auto log_record_text = [this](auto& ctx) { - const auto expected = http::headervalues::contenttype::TEXT; + const auto expected = ccf::http::headervalues::contenttype::TEXT; const auto actual = - ctx.rpc_ctx->get_request_header(http::headers::CONTENT_TYPE) + ctx.rpc_ctx->get_request_header(ccf::http::headers::CONTENT_TYPE) .value_or(""); if (expected != actual) { @@ -1217,15 +1244,13 @@ namespace loggingapp auto get_historical = [this]( ccf::endpoints::ReadOnlyEndpointContext& ctx, ccf::historical::StatePtr historical_state) { - const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx); - // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { ctx.rpc_ctx->set_error( HTTP_STATUS_BAD_REQUEST, @@ -1244,7 +1269,7 @@ namespace loggingapp LoggingGetHistorical::Out out; out.msg = v.value(); nlohmann::json j = out; - ccf::jsonhandler::set_response(std::move(j), ctx.rpc_ctx, pack); + ccf::jsonhandler::set_response(std::move(j), ctx.rpc_ctx); } else { @@ -1274,15 +1299,13 @@ namespace loggingapp [this]( ccf::endpoints::ReadOnlyEndpointContext& ctx, ccf::historical::StatePtr historical_state) { - const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx); - // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { ctx.rpc_ctx->set_error( HTTP_STATUS_BAD_REQUEST, @@ -1302,7 +1325,7 @@ namespace loggingapp out.msg = v.value(); assert(historical_state->receipt); out.receipt = ccf::describe_receipt_v1(*historical_state->receipt); - ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx, pack); + ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx); } else { @@ -1325,15 +1348,13 @@ namespace loggingapp [this]( ccf::endpoints::ReadOnlyEndpointContext& ctx, ccf::historical::StatePtr historical_state) { - const auto pack = ccf::jsonhandler::detect_json_pack(ctx.rpc_ctx); - // Parse id from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { ctx.rpc_ctx->set_error( HTTP_STATUS_BAD_REQUEST, @@ -1360,7 +1381,7 @@ namespace loggingapp out.receipt = full_receipt; out.receipt["leaf_components"].erase("claims_digest"); // SNIPPET_END: claims_digest_in_receipt - ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx, pack); + ccf::jsonhandler::set_response(std::move(out), ctx.rpc_ctx); } else { @@ -1385,12 +1406,12 @@ namespace loggingapp this](ccf::endpoints::EndpointContext& ctx) { // Parse arguments from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { ctx.rpc_ctx->set_error( HTTP_STATUS_BAD_REQUEST, @@ -1400,7 +1421,7 @@ namespace loggingapp } size_t from_seqno; - if (!http::get_query_value( + if (!ccf::http::get_query_value( parsed_query, "from_seqno", from_seqno, error_reason)) { // If no from_seqno is specified, defaults to very first transaction @@ -1409,7 +1430,7 @@ namespace loggingapp } size_t to_seqno; - if (!http::get_query_value( + if (!ccf::http::get_query_value( parsed_query, "to_seqno", to_seqno, error_reason)) { // If no end point is specified, use the last time this ID was @@ -1495,9 +1516,10 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); static constexpr size_t retry_after_seconds = 3; ctx.rpc_ctx->set_response_header( - http::headers::RETRY_AFTER, retry_after_seconds); + ccf::http::headers::RETRY_AFTER, retry_after_seconds); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(fmt::format( "Still constructing index for public records on key {} - indexed " "to {}/{}", @@ -1523,9 +1545,10 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); static constexpr size_t retry_after_seconds = 3; ctx.rpc_ctx->set_response_header( - http::headers::RETRY_AFTER, retry_after_seconds); + ccf::http::headers::RETRY_AFTER, retry_after_seconds); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(fmt::format( "Still constructing index for private records at {}", id)); return; @@ -1548,7 +1571,7 @@ namespace loggingapp // Fetch the requested range auto& historical_cache = context.get_historical_state(); - std::vector stores; + std::vector stores; if (!interesting_seqnos->empty()) { stores = @@ -1560,10 +1583,10 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); static constexpr size_t retry_after_seconds = 3; ctx.rpc_ctx->set_response_header( - http::headers::RETRY_AFTER, retry_after_seconds); + ccf::http::headers::RETRY_AFTER, retry_after_seconds); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, - http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(fmt::format( "Historical transactions from {} to {} are not yet " "available, fetching now", @@ -1644,7 +1667,8 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(j_response.dump()); // ALSO: Assume this response makes it all the way to the client, and @@ -1672,12 +1696,12 @@ namespace loggingapp ccf::endpoints::EndpointContext& ctx) { // Parse arguments from query const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error_reason; size_t id; - if (!http::get_query_value(parsed_query, "id", id, error_reason)) + if (!ccf::http::get_query_value(parsed_query, "id", id, error_reason)) { ctx.rpc_ctx->set_error( HTTP_STATUS_BAD_REQUEST, @@ -1689,7 +1713,7 @@ namespace loggingapp std::vector seqnos; { std::string seqnos_s; - if (!http::get_query_value( + if (!ccf::http::get_query_value( parsed_query, "seqnos", seqnos_s, error_reason)) { ctx.rpc_ctx->set_error( @@ -1699,7 +1723,7 @@ namespace loggingapp return; } - const auto terms = nonstd::split(seqnos_s, ","); + const auto terms = ccf::nonstd::split(seqnos_s, ","); for (const auto& term : terms) { size_t val; @@ -1778,9 +1802,10 @@ namespace loggingapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); static constexpr size_t retry_after_seconds = 3; ctx.rpc_ctx->set_response_header( - http::headers::RETRY_AFTER, retry_after_seconds); + ccf::http::headers::RETRY_AFTER, retry_after_seconds); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(fmt::format( "Historical transactions are not yet available, fetching now")); return; @@ -1813,7 +1838,8 @@ namespace loggingapp nlohmann::json j_response = response; ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(j_response.dump()); // ALSO: Assume this response makes it all the way to the client, and @@ -1910,7 +1936,8 @@ namespace loggingapp ctx.template get_caller(); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); std::vector response_body( caller_identity.content.begin(), caller_identity.content.end()); ctx.rpc_ctx->set_response_body(response_body); @@ -1928,11 +1955,11 @@ namespace loggingapp }; } -namespace ccfapp +namespace ccf { // SNIPPET_START: app_interface std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique(context); } diff --git a/samples/apps/nobuiltins/nobuiltins.cpp b/samples/apps/nobuiltins/nobuiltins.cpp index d265bdd8d6de..8d74457ac122 100644 --- a/samples/apps/nobuiltins/nobuiltins.cpp +++ b/samples/apps/nobuiltins/nobuiltins.cpp @@ -68,7 +68,7 @@ namespace nobuiltins class NoBuiltinsRegistry : public ccf::BaseEndpointRegistry { public: - NoBuiltinsRegistry(ccfapp::AbstractNodeContext& context) : + NoBuiltinsRegistry(ccf::AbstractNodeContext& context) : ccf::BaseEndpointRegistry("app", context) { auto node_summary = [this](auto& ctx) { @@ -224,11 +224,11 @@ namespace nobuiltins auto get_txid = [this](auto& ctx, nlohmann::json&&) { const auto query_string = ctx.rpc_ctx->get_request_query(); - const auto query_params = nonstd::split(query_string, "&"); + const auto query_params = ccf::nonstd::split(query_string, "&"); for (const auto& query_param : query_params) { const auto& [query_key, query_value] = - nonstd::split_1(query_param, "="); + ccf::nonstd::split_1(query_param, "="); if (query_key == "seqno") { ccf::SeqNo seqno; @@ -328,10 +328,10 @@ namespace nobuiltins }; } -namespace ccfapp +namespace ccf { std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique(context); } diff --git a/samples/apps/programmability/audit_info.h b/samples/apps/programmability/audit_info.h deleted file mode 100644 index e485a39c2224..000000000000 --- a/samples/apps/programmability/audit_info.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. - -#pragma once -#include "ccf/ds/json.h" -#include "ccf/entity_id.h" - -#include - -namespace programmabilityapp -{ - enum class AuditInputFormat - { - COSE = 0, - JSON = 1 - }; - DECLARE_JSON_ENUM( - AuditInputFormat, - {{AuditInputFormat::COSE, "COSE"}, {AuditInputFormat::JSON, "JSON"}}); - - enum class AuditInputContent - { - BUNDLE = 0, - OPTIONS = 1 - }; - DECLARE_JSON_ENUM( - AuditInputContent, - {{AuditInputContent::BUNDLE, "BUNDLE"}, - {AuditInputContent::OPTIONS, "OPTIONS"}}); - - struct AuditInfo - { - AuditInputFormat format; - AuditInputContent content; - ccf::UserId user_id; - }; - - DECLARE_JSON_TYPE(AuditInfo) - DECLARE_JSON_REQUIRED_FIELDS(AuditInfo, format, content, user_id) -} \ No newline at end of file diff --git a/samples/apps/programmability/programmability.cpp b/samples/apps/programmability/programmability.cpp index 5c00f887fb39..fc3d6710aad7 100644 --- a/samples/apps/programmability/programmability.cpp +++ b/samples/apps/programmability/programmability.cpp @@ -2,13 +2,13 @@ // Licensed under the Apache 2.0 License. // CCF -#include "audit_info.h" #include "ccf/app_interface.h" #include "ccf/common_auth_policies.h" #include "ccf/ds/hash.h" #include "ccf/http_query.h" #include "ccf/js/registry.h" #include "ccf/json_handler.h" +#include "ccf/service/tables/users.h" #include "ccf/version.h" #include @@ -19,12 +19,160 @@ using namespace nlohmann; namespace programmabilityapp { - using RecordsMap = kv::Map>; - using AuditInputValue = kv::Value>; - using AuditInfoValue = kv::Value; + using RecordsMap = ccf::kv::Map>; static constexpr auto PRIVATE_RECORDS = "programmability.records"; static constexpr auto CUSTOM_ENDPOINTS_NAMESPACE = "public:custom_endpoints"; + // The programmability sample demonstrates how signed payloads can be used to + // provide offline auditability without requiring trusting the hardware or the + // service owners/consortium. + // COSE Sign1 payloads must set these protected headers in order to guarantee + // the specificity of the payload for the endpoint, and avoid possible replay + // of payloads signed in the past. + static constexpr auto MSG_TYPE_NAME = "app.msg.type"; + static constexpr auto CREATED_AT_NAME = "app.msg.created_at"; + // Instances of ccf::TypedUserCOSESign1AuthnPolicy for the endpoints that + // support COSE Sign1 authentication. + static auto endpoints_user_cose_sign1_auth_policy = + std::make_shared( + "custom_endpoints", MSG_TYPE_NAME, CREATED_AT_NAME); + static auto options_user_cose_sign1_auth_policy = + std::make_shared( + "runtime_options", MSG_TYPE_NAME, CREATED_AT_NAME); + + // This is a pure helper function which can be called from either C++ or JS, + // to implement common functionality in a single place + static inline bool has_role_permitting_action( + ccf::kv::ReadOnlyTx& tx, + const std::string& user_id, + const std::string& action) + { + using RoleSet = ccf::kv::Set; + + auto users_handle = tx.ro(ccf::Tables::USER_INFO); + const auto user_info = users_handle->get(user_id); + if (user_info.has_value()) + { + const auto roles_it = user_info->user_data.find("roles"); + if (roles_it != user_info->user_data.end()) + { + const auto roles = roles_it->get>(); + for (const auto& role : roles) + { + auto role_handle = + tx.ro(fmt::format("public:ccf.gov.roles.{}", role)); + if (role_handle->contains(action)) + { + return true; + } + } + } + } + + return false; + } + + class MyExtension : public ccf::js::extensions::ExtensionInterface + { + public: + // Store any objects/state which the extension's functions might need on + // this extension object. + // In this case, since the extension adds a function that wants to read from + // the KV, it needs the current request's Tx. + ccf::kv::ReadOnlyTx* tx; + + MyExtension(ccf::kv::ReadOnlyTx* t) : tx(t) {} + + void install(ccf::js::core::Context& ctx) override; + }; + + // This is the signature for a function exposed to JS, interacting directly + // with JS interpreter state + JSValue js_has_role_permitting_action( + JSContext* ctx, JSValueConst this_val, int argc, JSValueConst* argv) + { + // Check correct number of args were passed, to avoid unsafe accesses to + // argv + if (argc != 2) + { + return JS_ThrowTypeError(ctx, "Passed %d arguments but expected 2", argc); + } + + // Retrieve the CCF context object from QuickJS's opaque pointer + ccf::js::core::Context& jsctx = + *(ccf::js::core::Context*)JS_GetContextOpaque(ctx); + + // Get the extension (by type), and the Tx* stashed on it + auto extension = jsctx.get_extension(); + if (extension == nullptr) + { + return JS_ThrowInternalError(ctx, "Failed to get extension object"); + } + + auto tx_ptr = extension->tx; + if (tx_ptr == nullptr) + { + return JS_ThrowInternalError(ctx, "No transaction available"); + } + + ccf::kv::ReadOnlyTx& tx = *tx_ptr; + + // Process the arguments passed to the JS function, confirming they're both + // strings + std::optional user_id = jsctx.to_str(argv[0]); + if (!user_id.has_value()) + { + return JS_ThrowTypeError(ctx, "user_id argument is not a string"); + } + + std::optional action = jsctx.to_str(argv[1]); + if (!action.has_value()) + { + return JS_ThrowTypeError(ctx, "action argument is not a string"); + } + + try + { + // Call function containing shared implementation + const bool permitted = + has_role_permitting_action(tx, user_id.value(), action.value()); + + // Return result (converting C++ type to QuickJS value) + return JS_NewBool(ctx, permitted); + } + catch (const std::exception& exc) + { + // Catch any exceptions from C++ function, report them to the JS layer as + // error + return JS_ThrowInternalError( + ctx, "Error checking for role permissions: %s", exc.what()); + } + } + + void MyExtension::install(ccf::js::core::Context& ctx) + { + // Nest all of this extension's functions in a single object, rather than + // inserting directly into the global namespace + auto my_global_object = + ctx.get_or_create_global_property("my_object", ctx.new_obj()); + + // Insert a constant string into the JS environment, accessible at + // my_object.my_constant + my_global_object.set("my_constant", ctx.new_string("Hello world")); + + // Insert a function into the JS environment, called at my_object.has_role + my_global_object.set( + // Name of field on object + "hasRole", + ctx.new_c_function( + // C/C++ function implementing this JS function + js_has_role_permitting_action, + // Repeated name of function, used in callstacks + "hasRole", + // Number of arguments to this function + 2)); + } + // This sample shows the features of DynamicJSEndpointRegistry. This sample // adds a PUT /app/custom_endpoints, which calls install_custom_endpoints(), // after first authenticating the caller (user_data["isAdmin"] is true), to @@ -53,23 +201,69 @@ namespace programmabilityapp return std::nullopt; } - std::pair> get_body( - ccf::endpoints::EndpointContext& ctx) + std::tuple< + ccf::ActionFormat, // JSON or COSE + std::span, // Content + std::optional // Created at timestamp, if passed + > + get_action_content(ccf::endpoints::EndpointContext& ctx) { if ( const auto* cose_ident = ctx.try_get_caller()) { - return {AuditInputFormat::COSE, cose_ident->content}; + return { + ccf::ActionFormat::COSE, + cose_ident->content, + cose_ident->protected_header.msg_created_at}; } else { - return {AuditInputFormat::JSON, ctx.rpc_ctx->get_request_body()}; + return { + ccf::ActionFormat::JSON, + ctx.rpc_ctx->get_request_body(), + std::nullopt}; + } + } + + bool set_error_details( + ccf::endpoints::EndpointContext& ctx, + ccf::ApiResult result, + ccf::InvalidArgsReason reason) + { + switch (result) + { + case ccf::ApiResult::OK: + { + return false; + } + case ccf::ApiResult::InvalidArgs: + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InvalidInput, + reason == ccf::InvalidArgsReason::ActionAlreadyApplied ? + "Action was already applied" : + "Action created_at timestamp is too old"); + return true; + } + case ccf::ApiResult::InternalError: + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Failed to check if action is original"); + return true; + } + default: + { + return true; + } } } public: - ProgrammabilityHandlers(ccfapp::AbstractNodeContext& context) : + ProgrammabilityHandlers(ccf::AbstractNodeContext& context) : ccf::js::DynamicJSEndpointRegistry( context, CUSTOM_ENDPOINTS_NAMESPACE // Internal KV space will be under @@ -125,7 +319,8 @@ namespace programmabilityapp { ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(record.value()); return; } @@ -228,18 +423,45 @@ namespace programmabilityapp } // End of Authorization Check - const auto [format, bundle] = get_body(ctx); - const auto j = nlohmann::json::parse(bundle.begin(), bundle.end()); - const auto parsed_bundle = j.get(); - - // Make operation auditable by writing user-supplied - // document to the ledger - auto audit_input = ctx.tx.template rw( - fmt::format("{}.audit.input", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_input->put(ctx.rpc_ctx->get_request_body()); - auto audit_info = ctx.tx.template rw( - fmt::format("{}.audit.info", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_info->put({format, AuditInputContent::BUNDLE, user_id.value()}); + const auto [format, content, created_at] = get_action_content(ctx); + const auto parsed_content = + nlohmann::json::parse(content.begin(), content.end()); + const auto parsed_bundle = parsed_content.get(); + + // Make operation auditable + record_action_for_audit_v1( + ctx.tx, + format, + user_id.value(), + fmt::format( + "{} {}", + ctx.rpc_ctx->get_method(), + ctx.rpc_ctx->get_request_path()), + ctx.rpc_ctx->get_request_body()); + + // Ensure signed actions are not replayed + if (format == ccf::ActionFormat::COSE) + { + if (!created_at.has_value()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::MissingRequiredHeader, + fmt::format("Missing {} protected header", CREATED_AT_NAME)); + return; + } + ccf::InvalidArgsReason reason; + result = check_action_not_replayed_v1( + ctx.tx, + created_at.value(), + ctx.rpc_ctx->get_request_body(), + reason); + + if (set_error_details(ctx, result, reason)) + { + return; + } + } result = install_custom_endpoints_v1(ctx.tx, parsed_bundle); if (result != ccf::ApiResult::OK) @@ -260,7 +482,7 @@ namespace programmabilityapp "/custom_endpoints", HTTP_PUT, put_custom_endpoints, - {ccf::user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) + {endpoints_user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) .set_auto_schema() .install(); @@ -280,7 +502,8 @@ namespace programmabilityapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(bundle).dump(2)); }; @@ -298,10 +521,10 @@ namespace programmabilityapp { const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); std::string error; - if (!http::get_query_value( + if (!ccf::http::get_query_value( parsed_query, "module_name", module_name, error)) { ctx.rpc_ctx->set_error( @@ -328,8 +551,8 @@ namespace programmabilityapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, - http::headervalues::contenttype::JAVASCRIPT); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JAVASCRIPT); ctx.rpc_ctx->set_response_body(std::move(code)); }; @@ -391,26 +614,53 @@ namespace programmabilityapp // - Convert current options to JSON auto j_options = nlohmann::json(options); - const auto [format, body] = get_body(ctx); - // - Parse argument as JSON body - const auto arg_body = nlohmann::json::parse(body.begin(), body.end()); + const auto [format, content, created_at] = get_action_content(ctx); + // - Parse content as JSON options + const auto arg_content = + nlohmann::json::parse(content.begin(), content.end()); // - Merge, to overwrite current options with anything from body. Note // that nulls mean deletions, which results in resetting to a default // value - j_options.merge_patch(arg_body); + j_options.merge_patch(arg_content); // - Parse patched options from JSON options = j_options.get(); - // Make operation auditable by writing user-supplied - // document to the ledger - auto audit = ctx.tx.template rw( - fmt::format("{}.audit.input", CUSTOM_ENDPOINTS_NAMESPACE)); - audit->put(ctx.rpc_ctx->get_request_body()); - auto audit_info = ctx.tx.template rw( - fmt::format("{}.audit.info", CUSTOM_ENDPOINTS_NAMESPACE)); - audit_info->put({format, AuditInputContent::BUNDLE, user_id.value()}); + // Make operation auditable + record_action_for_audit_v1( + ctx.tx, + format, + user_id.value(), + fmt::format( + "{} {}", + ctx.rpc_ctx->get_method(), + ctx.rpc_ctx->get_request_path()), + ctx.rpc_ctx->get_request_body()); + + // Ensure signed actions are not replayed + if (format == ccf::ActionFormat::COSE) + { + if (!created_at.has_value()) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::MissingRequiredHeader, + fmt::format("Missing {} protected header", CREATED_AT_NAME)); + return; + } + ccf::InvalidArgsReason reason; + result = check_action_not_replayed_v1( + ctx.tx, + created_at.value(), + ctx.rpc_ctx->get_request_body(), + reason); + + if (set_error_details(ctx, result, reason)) + { + return; + } + } result = set_js_runtime_options_v1(ctx.tx, options); if (result != ccf::ApiResult::OK) @@ -425,14 +675,15 @@ namespace programmabilityapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(options).dump(2)); }; make_endpoint( "/custom_endpoints/runtime_options", HTTP_PATCH, patch_runtime_options, - {ccf::user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) + {options_user_cose_sign1_auth_policy, ccf::user_cert_auth_policy}) .install(); auto get_runtime_options = [this](ccf::endpoints::EndpointContext& ctx) { @@ -452,7 +703,8 @@ namespace programmabilityapp ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(options).dump(2)); }; make_endpoint( @@ -463,13 +715,23 @@ namespace programmabilityapp .set_auto_schema() .install(); } + + ccf::js::extensions::Extensions get_extensions( + const ccf::endpoints::EndpointContext& endpoint_ctx) override + { + ccf::js::extensions::Extensions extensions; + + extensions.push_back(std::make_shared(&endpoint_ctx.tx)); + + return extensions; + } }; } -namespace ccfapp +namespace ccf { std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique( context); diff --git a/samples/constitutions/default/actions.js b/samples/constitutions/default/actions.js index 4e2904ccf897..fa380f3c9364 100644 --- a/samples/constitutions/default/actions.js +++ b/samples/constitutions/default/actions.js @@ -688,7 +688,7 @@ const actions = new Map([ if (redirection_strategy !== undefined) { checkEnum( info.redirection_strategy, - ["none", "to_primary"], + ["none", "to_primary", "to_backup"], `${prefix2}.redirection_strategy`, ); } diff --git a/samples/constitutions/roles/set_role_definition.js b/samples/constitutions/roles/set_role_definition.js index 2b178321d94c..6da73a4d2280 100644 --- a/samples/constitutions/roles/set_role_definition.js +++ b/samples/constitutions/roles/set_role_definition.js @@ -24,7 +24,7 @@ class KVSet { asSetOfStrings() { let set = new Set(); - this.#map.forEach((_, key) => set.add(ccf.bufToStr(key))); + this.#map.forEach((_, key) => set.add(ccf.bufToJsonCompatible(key))); return set; } } @@ -47,12 +47,12 @@ actions.set( let newValues = new Set(args.actions); for (const action of oldValues) { if (!newValues.has(action)) { - roleDefinition.delete(ccf.strToBuf(action)); + roleDefinition.delete(ccf.jsonCompatibleToBuf(action)); } } for (const action of newValues) { if (!oldValues.has(action)) { - roleDefinition.add(ccf.strToBuf(action)); + roleDefinition.add(ccf.jsonCompatibleToBuf(action)); } } }, diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py index 8c2b15556601..a4144e9f12b9 100644 --- a/scripts/azure_deployment/arm_aci.py +++ b/scripts/azure_deployment/arm_aci.py @@ -133,7 +133,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace: "--aci-image", help="The name of the image to deploy in the ACI", type=str, - default="ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp", + default="ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp", ) parser.add_argument( "--aci-type", diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 8d3b0b18a6ff..836a99352c82 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -3,11 +3,11 @@ #include "ccf/app_interface.h" #include "js_generic_base.h" -namespace ccfapp +namespace ccf { std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return make_user_endpoints_impl(context); } -} // namespace ccfapp +} // namespace ccf diff --git a/src/apps/js_generic/js_generic_base.cpp b/src/apps/js_generic/js_generic_base.cpp index 20864984a327..073ed433048e 100644 --- a/src/apps/js_generic/js_generic_base.cpp +++ b/src/apps/js_generic/js_generic_base.cpp @@ -33,16 +33,16 @@ #include #include -namespace ccfapp +namespace ccf { using namespace std; - using namespace kv; + using namespace ccf::kv; using namespace ccf; class JSHandlers : public UserEndpointRegistry { private: - ccfapp::AbstractNodeContext& context; + ccf::AbstractNodeContext& context; std::shared_ptr interpreter_cache = nullptr; @@ -302,7 +302,7 @@ namespace ccfapp if (array_buffer) { endpoint_ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, + ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::OCTET_STREAM); response_body = std::vector(array_buffer, array_buffer + buf_size); @@ -313,14 +313,14 @@ namespace ccfapp if (response_body_js.is_str()) { endpoint_ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, + ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); str = ctx.to_str(response_body_js); } else { endpoint_ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, + ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); auto rval = ctx.json_stringify(response_body_js); if (rval.is_exception()) @@ -592,7 +592,7 @@ namespace ccfapp } ccf::endpoints::EndpointDefinitionPtr find_endpoint( - kv::Tx& tx, ccf::RpcContext& rpc_ctx) override + ccf::kv::Tx& tx, ccf::RpcContext& rpc_ctx) override { const auto method = rpc_ctx.get_method(); const auto verb = rpc_ctx.get_request_verb(); @@ -685,7 +685,7 @@ namespace ccfapp } std::set get_allowed_verbs( - kv::Tx& tx, const ccf::RpcContext& rpc_ctx) override + ccf::kv::Tx& tx, const ccf::RpcContext& rpc_ctx) override { const auto method = rpc_ctx.get_method(); @@ -751,7 +751,7 @@ namespace ccfapp // Since we do our own dispatch within the default handler, report the // supported methods here - void build_api(nlohmann::json& document, kv::ReadOnlyTx& tx) override + void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) override { UserEndpointRegistry::build_api(document, tx); @@ -793,9 +793,9 @@ namespace ccfapp }; std::unique_ptr make_user_endpoints_impl( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique(context); } -} // namespace ccfapp +} // namespace ccf diff --git a/src/apps/js_generic/js_generic_base.h b/src/apps/js_generic/js_generic_base.h index ab1463dd2107..9693dfa03e2c 100644 --- a/src/apps/js_generic/js_generic_base.h +++ b/src/apps/js_generic/js_generic_base.h @@ -4,8 +4,8 @@ #include -namespace ccfapp +namespace ccf { std::unique_ptr make_user_endpoints_impl( - ccfapp::AbstractNodeContext& context); + ccf::AbstractNodeContext& context); } \ No newline at end of file diff --git a/src/apps/tpcc/app/tpcc.cpp b/src/apps/tpcc/app/tpcc.cpp index c11d69a3d158..5ac488f10fe7 100644 --- a/src/apps/tpcc/app/tpcc.cpp +++ b/src/apps/tpcc/app/tpcc.cpp @@ -14,7 +14,7 @@ using namespace std; using namespace nlohmann; using namespace ccf; -namespace ccfapp +namespace ccf { class TpccHandlers : public UserEndpointRegistry { @@ -24,7 +24,8 @@ namespace ccfapp { ctx.rpc_ctx->set_response_status(status); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::TEXT); ctx.rpc_ctx->set_response_body(std::move(message)); } @@ -32,7 +33,7 @@ namespace ccfapp { ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, + ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::OCTET_STREAM); } @@ -135,7 +136,7 @@ namespace ccfapp }; std::unique_ptr make_user_endpoints( - ccfapp::AbstractNodeContext& context) + ccf::AbstractNodeContext& context) { return std::make_unique(context); } diff --git a/src/apps/tpcc/app/tpcc_tables.h b/src/apps/tpcc/app/tpcc_tables.h index d937cfbd9142..81c603849dbd 100644 --- a/src/apps/tpcc/app/tpcc_tables.h +++ b/src/apps/tpcc/app/tpcc_tables.h @@ -19,15 +19,15 @@ namespace tpcc { template - kv::serialisers::SerialisedEntry tpcc_serialise(const T& t); + ccf::kv::serialisers::SerialisedEntry tpcc_serialise(const T& t); template - T tpcc_deserialise(const kv::serialisers::SerialisedEntry& rep); + T tpcc_deserialise(const ccf::kv::serialisers::SerialisedEntry& rep); template constexpr size_t serialised_size() { - if constexpr (nonstd::is_std_array::value) + if constexpr (ccf::nonstd::is_std_array::value) { return std::tuple_size_v * serialised_size(); } @@ -40,7 +40,7 @@ namespace tpcc template void write_value(const T& v, uint8_t*& data, size_t& size) { - if constexpr (nonstd::is_std_array::value) + if constexpr (ccf::nonstd::is_std_array::value) { if constexpr (std::is_integral_v) { @@ -64,7 +64,7 @@ namespace tpcc template void read_value(T& v, const uint8_t*& data, size_t& size) { - if constexpr (nonstd::is_std_array::value) + if constexpr (ccf::nonstd::is_std_array::value) { if constexpr (std::is_integral_v) { @@ -106,9 +106,9 @@ namespace tpcc _Pragma("clang diagnostic push"); \ _Pragma("clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\""); \ template <> \ - kv::serialisers::SerialisedEntry tpcc_serialise(const TYPE& t) \ + ccf::kv::serialisers::SerialisedEntry tpcc_serialise(const TYPE& t) \ { \ - kv::serialisers::SerialisedEntry rep; \ + ccf::kv::serialisers::SerialisedEntry rep; \ constexpr size_t required_size = 0 _FOR_JSON_COUNT_NN(__VA_ARGS__)(POP1)( \ ADD_SERIALIZED_SIZE, TYPE, ##__VA_ARGS__); \ rep.resize(required_size); \ @@ -118,7 +118,7 @@ namespace tpcc return rep; \ } \ template <> \ - TYPE tpcc_deserialise(const kv::serialisers::SerialisedEntry& rep) \ + TYPE tpcc_deserialise(const ccf::kv::serialisers::SerialisedEntry& rep) \ { \ auto data = rep.data(); \ auto size = rep.size(); \ @@ -132,12 +132,12 @@ namespace tpcc template struct TpccSerialiser { - static kv::serialisers::SerialisedEntry to_serialised(const T& t) + static ccf::kv::serialisers::SerialisedEntry to_serialised(const T& t) { return tpcc_serialise(t); } - static T from_serialised(const kv::serialisers::SerialisedEntry& rep) + static T from_serialised(const ccf::kv::serialisers::SerialisedEntry& rep) { return tpcc_deserialise(rep); } @@ -539,7 +539,7 @@ namespace tpcc History, c_id, c_d_id, c_w_id, d_id, w_id, amount, date, data); template - using TpccMap = kv::MapSerialisedWith; + using TpccMap = ccf::kv::MapSerialisedWith; struct TpccTables { diff --git a/src/apps/tpcc/clients/tpcc_client.cpp b/src/apps/tpcc/clients/tpcc_client.cpp index 07301c96d9c6..3bc17e16997c 100644 --- a/src/apps/tpcc/clients/tpcc_client.cpp +++ b/src/apps/tpcc/clients/tpcc_client.cpp @@ -155,9 +155,9 @@ class TpccClient : public Base int main(int argc, char** argv) { - logger::config::default_init(); - logger::config::level() = LoggerLevel::INFO; - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::logger::config::level() = LoggerLevel::INFO; + ccf::crypto::openssl_sha256_init(); CLI::App cli_app{"Tpcc Client"}; TpccClientOptions options(cli_app, argv[0]); @@ -165,7 +165,7 @@ int main(int argc, char** argv) TpccClient client(options); client.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return 0; } diff --git a/src/clients/perf/perf_client.h b/src/clients/perf/perf_client.h index 4b96b64f7e6f..e7cafce92e7a 100644 --- a/src/clients/perf/perf_client.h +++ b/src/clients/perf/perf_client.h @@ -203,7 +203,7 @@ namespace client }; private: - crypto::Pem key = {}; + ccf::crypto::Pem key = {}; std::string key_id = "Invalid"; std::shared_ptr<::tls::Cert> tls_cert = nullptr; @@ -301,11 +301,11 @@ namespace client const auto raw_key = files::slurp(options.key_file); const auto ca = files::slurp_string(options.ca_file); - key = crypto::Pem(raw_key); + key = ccf::crypto::Pem(raw_key); - const crypto::Pem cert_pem(raw_cert); - auto cert_der = crypto::cert_pem_to_der(cert_pem); - key_id = crypto::Sha256Hash(cert_der).hex_str(); + const ccf::crypto::Pem cert_pem(raw_cert); + auto cert_der = ccf::crypto::cert_pem_to_der(cert_pem); + key_id = ccf::crypto::Sha256Hash(cert_der).hex_str(); tls_cert = std::make_shared<::tls::Cert>( std::make_shared<::tls::CA>(ca), cert_pem, key); @@ -343,7 +343,7 @@ namespace client rpc_connection->gen_request( method, params, - http::headervalues::contenttype::JSON, + ccf::http::headervalues::contenttype::JSON, HTTP_POST, options.bearer_token.size() == 0 ? nullptr : options.bearer_token.c_str()), @@ -353,42 +353,6 @@ namespace client append_prepared_tx(tx, index); } - void add_prepared_tx( - const std::string& method, - const nlohmann::json& params, - bool expects_commit, - const std::optional& index, - const serdes::Pack& serdes) - { - auto body = serdes::pack(params, serdes); - - const PreparedTx tx{ - rpc_connection->gen_request( - method, - body, - serdes == serdes::Pack::Text ? - http::headervalues::contenttype::JSON : - http::headervalues::contenttype::MSGPACK, - HTTP_POST, - options.bearer_token.size() == 0 ? nullptr : - options.bearer_token.c_str()), - method, - expects_commit}; - - append_prepared_tx(tx, index); - } - - void add_prepared_tx( - const std::string& method, - const nlohmann::json& params, - bool expects_commit, - const std::optional& index) - { - const PreparedTx tx{ - rpc_connection->gen_request(method, params), method, expects_commit}; - append_prepared_tx(tx, index); - } - static size_t total_byte_size(const PreparedTxs& txs) { return std::accumulate( diff --git a/src/clients/perf/timing.h b/src/clients/perf/timing.h index 2f8f2cf446e6..d6ef6d51690f 100644 --- a/src/clients/perf/timing.h +++ b/src/clients/perf/timing.h @@ -154,7 +154,7 @@ namespace timing const client::RpcTlsClient::Response& response) { const auto& h = response.headers; - const auto it = h.find(http::headers::CCF_TX_ID); + const auto it = h.find(ccf::http::headers::CCF_TX_ID); if (it == h.end()) { return std::nullopt; diff --git a/src/clients/rpc_tls_client.h b/src/clients/rpc_tls_client.h index 2b107113528b..3ca1924fc865 100644 --- a/src/clients/rpc_tls_client.h +++ b/src/clients/rpc_tls_client.h @@ -4,7 +4,6 @@ #include "ccf/crypto/key_pair.h" #include "ccf/http_consts.h" -#include "ccf/serdes.h" #include "http/http_builder.h" #include "http/http_parser.h" #include "tls_client.h" @@ -17,7 +16,7 @@ namespace client { - class HttpRpcTlsClient : public TlsClient, public http::ResponseProcessor + class HttpRpcTlsClient : public TlsClient, public ::http::ResponseProcessor { public: struct PreparedRpc @@ -30,14 +29,14 @@ namespace client { size_t id; http_status status; - http::HeaderMap headers; + ccf::http::HeaderMap headers; std::vector body; }; protected: - http::ResponseParser parser; + ::http::ResponseParser parser; std::optional prefix; - crypto::KeyPairPtr key_pair = nullptr; + ccf::crypto::KeyPairPtr key_pair = nullptr; std::string key_id = "Invalid"; size_t next_send_id = 0; @@ -58,12 +57,13 @@ namespace client auto r = http::Request(path, verb); r.set_body(params.data(), params.size()); - r.set_header(http::headers::CONTENT_TYPE, content_type); + r.set_header(ccf::http::headers::CONTENT_TYPE, content_type); r.set_header("Host", host); if (auth_token != nullptr) { r.set_header( - http::headers::AUTHORIZATION, fmt::format("Bearer {}", auth_token)); + ccf::http::headers::AUTHORIZATION, + fmt::format("Bearer {}", auth_token)); } return r.build_request(); @@ -113,9 +113,9 @@ namespace client key_id(c.key_id) {} - void create_key_pair(const crypto::Pem priv_key) + void create_key_pair(const ccf::crypto::Pem priv_key) { - key_pair = crypto::make_key_pair(priv_key); + key_pair = ccf::crypto::make_key_pair(priv_key); } PreparedRpc gen_request( @@ -130,31 +130,24 @@ namespace client next_send_id++}; } - PreparedRpc gen_request( + Response call( const std::string& method, const nlohmann::json& params = nullptr, - llhttp_method verb = HTTP_POST, - const char* auth_token = nullptr) + llhttp_method verb = HTTP_POST) { std::vector body; if (!params.is_null()) { - body = serdes::pack(params, serdes::Pack::MsgPack); + auto body_s = params.dump(); + body = {body_s.begin(), body_s.end()}; } - return gen_request( + + return call_raw(gen_request( method, - {body.data(), body.size()}, - http::headervalues::contenttype::MSGPACK, + body, + ccf::http::headervalues::contenttype::JSON, verb, - auth_token); - } - - Response call( - const std::string& method, - const nlohmann::json& params = nullptr, - llhttp_method verb = HTTP_POST) - { - return call_raw(gen_request(method, params, verb, nullptr)); + nullptr)); } Response call( @@ -163,7 +156,7 @@ namespace client llhttp_method verb = HTTP_POST) { return call_raw(gen_request( - method, params, http::headervalues::contenttype::JSON, verb)); + method, params, ccf::http::headervalues::contenttype::JSON, verb)); } Response post(const std::string& method, const nlohmann::json& params) @@ -200,8 +193,8 @@ namespace client else if (http::status_success(resp.status)) { const auto& content_type = - resp.headers.find(http::headers::CONTENT_TYPE); - return serdes::unpack(resp.body, serdes::Pack::MsgPack); + resp.headers.find(ccf::http::headers::CONTENT_TYPE); + return nlohmann::json::parse(resp.body); } else { @@ -239,7 +232,7 @@ namespace client virtual void handle_response( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body) override { last_response = { diff --git a/src/clients/tls_client.h b/src/clients/tls_client.h index 02f75cb4cad7..4966af00a678 100644 --- a/src/clients/tls_client.h +++ b/src/clients/tls_client.h @@ -16,7 +16,7 @@ #include #include -using namespace crypto::OpenSSL; +using namespace ccf::crypto::OpenSSL; #ifdef _DEBUG static BIO* bio_err = NULL; diff --git a/src/consensus/aft/impl/state.h b/src/consensus/aft/impl/state.h index 61f848f42ee3..b41bc4cfe9aa 100644 --- a/src/consensus/aft/impl/state.h +++ b/src/consensus/aft/impl/state.h @@ -19,12 +19,12 @@ namespace aft class ViewHistory { // Entry i stores the first version in view i+1 - std::vector views; + std::vector views; public: static constexpr ccf::View InvalidView = ccf::VIEW_UNKNOWN; - void initialise(const std::vector& terms_) + void initialise(const std::vector& terms_) { views.clear(); for (size_t i = 0; i < terms_.size(); ++i) @@ -34,7 +34,7 @@ namespace aft LOG_DEBUG_FMT("Initialised views: {}", fmt::join(views, ", ")); } - void update(kv::Version idx, ccf::View view) + void update(ccf::kv::Version idx, ccf::View view) { LOG_DEBUG_FMT("Updating view to: {} at version: {}", view, idx); if (!views.empty()) @@ -56,7 +56,7 @@ namespace aft LOG_DEBUG_FMT("Resulting views: {}", fmt::join(views, ", ")); } - ccf::View view_at(kv::Version idx) + ccf::View view_at(ccf::kv::Version idx) { auto it = upper_bound(views.begin(), views.end(), idx); @@ -69,11 +69,11 @@ namespace aft return (it - views.begin()); } - kv::Version start_of_view(ccf::View view) + ccf::kv::Version start_of_view(ccf::View view) { if (view > views.size() || view == InvalidView) { - return kv::NoVersion; + return ccf::kv::NoVersion; } // NB: If views == {5, 10, 10}, then view 2 doesn't start at 10. View 2 @@ -81,23 +81,23 @@ namespace aft const auto tentative = views[view - 1]; if (view + 1 <= views.size() && views[view] == tentative) { - return kv::NoVersion; + return ccf::kv::NoVersion; } return tentative; } - kv::Version end_of_view(ccf::View view) + ccf::kv::Version end_of_view(ccf::View view) { // If this view has no start (potentially because it contains no // transactions), then it can't have an end - if (start_of_view(view) == kv::NoVersion) + if (start_of_view(view) == ccf::kv::NoVersion) { - return kv::NoVersion; + return ccf::kv::NoVersion; } if (view >= views.size() || view == InvalidView) { - return kv::NoVersion; + return ccf::kv::NoVersion; } // Otherwise the end of this view is the transaction before (- 1) the @@ -105,14 +105,14 @@ namespace aft return views[view] - 1; } - std::vector get_history_until( - kv::Version idx = std::numeric_limits::max()) + std::vector get_history_until( + ccf::kv::Version idx = std::numeric_limits::max()) { return {views.begin(), std::upper_bound(views.begin(), views.end(), idx)}; } // view should be non-zero as views start at one in here - std::vector get_history_since(uint64_t view) + std::vector get_history_since(uint64_t view) { if (view > views.size()) { @@ -121,7 +121,7 @@ namespace aft return {views.begin() + view - 1, views.end()}; } - void rollback(kv::Version idx) + void rollback(ccf::kv::Version idx) { auto it = upper_bound(views.begin(), views.end(), idx); views.erase(it, views.end()); @@ -135,7 +135,7 @@ namespace aft public: Replica(const ccf::NodeId& id_, const std::vector& cert_) : id(id_), - verifier(crypto::make_unique_verifier(cert_)) + verifier(ccf::crypto::make_unique_verifier(cert_)) {} ccf::NodeId get_id() const @@ -145,7 +145,7 @@ namespace aft private: ccf::NodeId id; - crypto::VerifierUniquePtr verifier; + ccf::crypto::VerifierUniquePtr verifier; }; struct State @@ -157,8 +157,8 @@ namespace aft ccf::NodeId node_id; ccf::View current_view = 0; - kv::Version last_idx = 0; - kv::Version commit_idx = 0; + ccf::kv::Version last_idx = 0; + ccf::kv::Version commit_idx = 0; ViewHistory view_history; // Indices that are eligible for global commit, from a Node's perspective @@ -176,10 +176,11 @@ namespace aft // the node // Leader -> Follower, when receiving entries for a newer term // Candidate -> Follower, when receiving entries for a newer term - kv::LeadershipState leadership_state = kv::LeadershipState::None; - kv::MembershipState membership_state = kv::MembershipState::Active; + ccf::kv::LeadershipState leadership_state = ccf::kv::LeadershipState::None; + ccf::kv::MembershipState membership_state = + ccf::kv::MembershipState::Active; - std::optional retirement_phase = std::nullopt; + std::optional retirement_phase = std::nullopt; // Index at which this node observes its retirement std::optional retirement_idx = std::nullopt; // Earliest index at which this node's retirement can be committed diff --git a/src/consensus/aft/raft.h b/src/consensus/aft/raft.h index 6e8094e10414..46d1844f51d5 100644 --- a/src/consensus/aft/raft.h +++ b/src/consensus/aft/raft.h @@ -89,10 +89,10 @@ static inline void add_committable_indices_start_and_end( namespace aft { - using Configuration = kv::Configuration; + using Configuration = ccf::kv::Configuration; template - class Aft : public kv::Consensus + class Aft : public ccf::kv::Consensus { private: struct NodeState @@ -216,8 +216,8 @@ namespace aft std::shared_ptr state_, std::shared_ptr rpc_request_context_, bool public_only_ = false, - kv::MembershipState initial_membership_state_ = - kv::MembershipState::Active, + ccf::kv::MembershipState initial_membership_state_ = + ccf::kv::MembershipState::Active, ReconfigurationType reconfiguration_type_ = ReconfigurationType::ONE_TRANSACTION) : store(std::move(store_)), @@ -258,12 +258,12 @@ namespace aft bool is_primary() override { - return state->leadership_state == kv::LeadershipState::Leader; + return state->leadership_state == ccf::kv::LeadershipState::Leader; } bool is_candidate() override { - return state->leadership_state == kv::LeadershipState::Candidate; + return state->leadership_state == ccf::kv::LeadershipState::Candidate; } bool can_replicate() override @@ -284,7 +284,7 @@ namespace aft return false; } std::unique_lock guard(state->lock); - return state->leadership_state == kv::LeadershipState::Leader && + return state->leadership_state == ccf::kv::LeadershipState::Leader && (state->last_idx - state->commit_idx >= max_uncommitted_tx_count); } @@ -310,46 +310,46 @@ namespace aft bool is_backup() override { - return state->leadership_state == kv::LeadershipState::Follower; + return state->leadership_state == ccf::kv::LeadershipState::Follower; } bool is_active() const { - return state->membership_state == kv::MembershipState::Active; + return state->membership_state == ccf::kv::MembershipState::Active; } bool is_retired() const { - return state->membership_state == kv::MembershipState::Retired; + return state->membership_state == ccf::kv::MembershipState::Retired; } bool is_retired_committed() const { - return state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::RetiredCommitted; + return state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::RetiredCommitted; } bool is_retired_completed() const { - return state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::Completed; + return state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::Completed; } void set_retired_committed( - ccf::SeqNo seqno, const std::vector& node_ids) override + ccf::SeqNo seqno, const std::vector& node_ids) override { for (auto& node_id : node_ids) { if (id() == node_id) { CCF_ASSERT( - state->membership_state == kv::MembershipState::Retired, + state->membership_state == ccf::kv::MembershipState::Retired, "Node is not retired, cannot become retired committed"); CCF_ASSERT( - state->retirement_phase == kv::RetirementPhase::Completed, + state->retirement_phase == ccf::kv::RetirementPhase::Completed, "Node is not retired, cannot become retired committed"); state->retired_committed_idx = seqno; - become_retired(seqno, kv::RetirementPhase::RetiredCommitted); + become_retired(seqno, ccf::kv::RetirementPhase::RetiredCommitted); } else { @@ -531,7 +531,7 @@ namespace aft public: void add_configuration( Index idx, - const kv::Configuration::Nodes& conf, + const ccf::kv::Configuration::Nodes& conf, const std::unordered_set& new_learner_nodes = {}, const std::unordered_set& new_retired_nodes = {}) override { @@ -561,7 +561,7 @@ namespace aft configurations.back().nodes.end() && conf.find(state->node_id) == conf.end()) { - become_retired(idx, kv::RetirementPhase::Ordered); + become_retired(idx, ccf::kv::RetirementPhase::Ordered); } if (conf != configurations.back().nodes) @@ -606,9 +606,9 @@ namespace aft return get_latest_configuration_unsafe(); } - kv::ConsensusDetails get_details() override + ccf::kv::ConsensusDetails get_details() override { - kv::ConsensusDetails details; + ccf::kv::ConsensusDetails details; std::lock_guard guard(state->lock); details.primary_id = leader_id; details.current_view = state->current_view; @@ -632,11 +632,11 @@ namespace aft return details; } - bool replicate(const kv::BatchVector& entries, Term term) override + bool replicate(const ccf::kv::BatchVector& entries, Term term) override { std::lock_guard guard(state->lock); - if (state->leadership_state != kv::LeadershipState::Leader) + if (state->leadership_state != ccf::kv::LeadershipState::Leader) { RAFT_DEBUG_FMT( "Failed to replicate {} items: not leader", entries.size()); @@ -702,10 +702,10 @@ namespace aft state->membership_state, state->leadership_state); if ( - state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::Ordered) + state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::Ordered) { - become_retired(index, kv::RetirementPhase::Signed); + become_retired(index, ccf::kv::RetirementPhase::Signed); } state->committable_indices.push_back(index); start_ticking_if_necessary(); @@ -825,7 +825,7 @@ namespace aft std::unique_lock guard(state->lock); timeout_elapsed += elapsed; - if (state->leadership_state == kv::LeadershipState::Leader) + if (state->leadership_state == ccf::kv::LeadershipState::Leader) { if (timeout_elapsed >= request_timeout) { @@ -962,13 +962,13 @@ namespace aft bool can_replicate_unsafe() { - return state->leadership_state == kv::LeadershipState::Leader && + return state->leadership_state == ccf::kv::LeadershipState::Leader && !is_retired_committed(); } bool can_sign_unsafe() { - return state->leadership_state == kv::LeadershipState::Leader && + return state->leadership_state == ccf::kv::LeadershipState::Leader && !is_retired_committed(); } @@ -998,7 +998,7 @@ namespace aft const auto term_of_ae = state->view_history.view_at(start); const auto index_at_end_of_term = state->view_history.end_of_view(term_of_ae); - if (index_at_end_of_term != kv::NoVersion) + if (index_at_end_of_term != ccf::kv::NoVersion) { max_idx = index_at_end_of_term; } @@ -1116,7 +1116,7 @@ namespace aft // follower if necessary if ( state->current_view == r.term && - state->leadership_state == kv::LeadershipState::Candidate) + state->leadership_state == ccf::kv::LeadershipState::Candidate) { become_aware_of_new_term(r.term); } @@ -1216,8 +1216,9 @@ namespace aft r.idx, r.prev_idx); - std::vector< - std::tuple, kv::Version>> + std::vector, + ccf::kv::Version>> append_entries; // Finally, deserialise each entry in the batch for (Index i = r.prev_idx + 1; i <= r.idx; i++) @@ -1299,7 +1300,7 @@ namespace aft return; } - kv::TxID expected{r.term_of_idx, i}; + ccf::kv::TxID expected{r.term_of_idx, i}; auto ds = store->deserialize(entry, public_only, expected); if (ds == nullptr) { @@ -1321,8 +1322,8 @@ namespace aft void execute_append_entries_sync( std::vector, - kv::Version>>&& append_entries, + std::unique_ptr, + ccf::kv::Version>>&& append_entries, const ccf::NodeId& from, AppendEntries&& r) { @@ -1341,9 +1342,9 @@ namespace aft #endif bool track_deletes_on_missing_keys = false; - kv::ApplyResult apply_success = + ccf::kv::ApplyResult apply_success = ds->apply(track_deletes_on_missing_keys); - if (apply_success == kv::ApplyResult::FAIL) + if (apply_success == ccf::kv::ApplyResult::FAIL) { ledger->truncate(i - 1); send_append_entries_response_nack(from); @@ -1357,7 +1358,7 @@ namespace aft } bool globally_committable = - (apply_success == kv::ApplyResult::PASS_SIGNATURE); + (apply_success == ccf::kv::ApplyResult::PASS_SIGNATURE); if (globally_committable) { start_ticking_if_necessary(); @@ -1370,7 +1371,7 @@ namespace aft switch (apply_success) { - case kv::ApplyResult::FAIL: + case ccf::kv::ApplyResult::FAIL: { RAFT_FAIL_FMT("Follower failed to apply log entry: {}", i); state->last_idx--; @@ -1379,14 +1380,14 @@ namespace aft break; } - case kv::ApplyResult::PASS_SIGNATURE: + case ccf::kv::ApplyResult::PASS_SIGNATURE: { RAFT_DEBUG_FMT("Deserialising signature at {}", i); if ( - state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::Ordered) + state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::Ordered) { - become_retired(i, kv::RetirementPhase::Signed); + become_retired(i, ccf::kv::RetirementPhase::Signed); } state->committable_indices.push_back(i); @@ -1416,12 +1417,12 @@ namespace aft break; } - case kv::ApplyResult::PASS: + case ccf::kv::ApplyResult::PASS: { break; } - case kv::ApplyResult::PASS_ENCRYPTED_PAST_LEDGER_SECRET: + case ccf::kv::ApplyResult::PASS_ENCRYPTED_PAST_LEDGER_SECRET: { break; } @@ -1560,7 +1561,7 @@ namespace aft #endif // Ignore if we're not the leader. - if (state->leadership_state != kv::LeadershipState::Leader) + if (state->leadership_state != ccf::kv::LeadershipState::Leader) { RAFT_FAIL_FMT( "Recv append entries response to {} from {}: no longer leader", @@ -1811,7 +1812,7 @@ namespace aft RAFT_TRACE_JSON_OUT(j); #endif - if (state->leadership_state != kv::LeadershipState::Candidate) + if (state->leadership_state != ccf::kv::LeadershipState::Candidate) { RAFT_INFO_FMT( "Recv request vote response to {} from: {}: we aren't a candidate", @@ -1929,7 +1930,7 @@ namespace aft return; } - state->leadership_state = kv::LeadershipState::Candidate; + state->leadership_state = ccf::kv::LeadershipState::Candidate; leader_id.reset(); voted_for = state->node_id; @@ -1986,7 +1987,7 @@ namespace aft store->initialise_term(state->current_view); } - state->leadership_state = kv::LeadershipState::Leader; + state->leadership_state = ccf::kv::LeadershipState::Leader; leader_id = state->node_id; should_sign = true; @@ -2045,7 +2046,7 @@ namespace aft // receiving a conflicting AppendEntries rollback(last_committable_index()); - state->leadership_state = kv::LeadershipState::Follower; + state->leadership_state = ccf::kv::LeadershipState::Follower; RAFT_INFO_FMT( "Becoming follower {}: {}.{}", state->node_id, @@ -2077,7 +2078,7 @@ namespace aft } private: - void become_retired(Index idx, kv::RetirementPhase phase) + void become_retired(Index idx, ccf::kv::RetirementPhase phase) { RAFT_INFO_FMT( "Becoming retired, phase {} (leadership {}): {}: {} at {}", @@ -2087,7 +2088,7 @@ namespace aft state->current_view, idx); - if (phase == kv::RetirementPhase::Ordered) + if (phase == ccf::kv::RetirementPhase::Ordered) { CCF_ASSERT_FMT( !state->retirement_idx.has_value(), @@ -2096,7 +2097,7 @@ namespace aft state->retirement_idx = idx; RAFT_INFO_FMT("Node retiring at {}", idx); } - else if (phase == kv::RetirementPhase::Signed) + else if (phase == ccf::kv::RetirementPhase::Signed) { assert(state->retirement_idx.has_value()); CCF_ASSERT_FMT( @@ -2107,15 +2108,15 @@ namespace aft state->retirement_committable_idx = idx; RAFT_INFO_FMT("Node retirement committable at {}", idx); } - else if (phase == kv::RetirementPhase::RetiredCommitted) + else if (phase == ccf::kv::RetirementPhase::RetiredCommitted) { - if (state->leadership_state == kv::LeadershipState::Leader) + if (state->leadership_state == ccf::kv::LeadershipState::Leader) { ProposeRequestVote prv{.term = state->current_view}; std::optional successor = std::nullopt; Index max_match_idx = 0; - kv::ReconfigurationId reconf_id_of_max_match = 0; + ccf::kv::ReconfigurationId reconf_id_of_max_match = 0; // Pick the node that has the highest match_idx, and break // ties by looking at the highest reconfiguration id they are @@ -2131,7 +2132,7 @@ namespace aft { if (node_state.match_idx >= max_match_idx) { - kv::ReconfigurationId latest_reconf_id = 0; + ccf::kv::ReconfigurationId latest_reconf_id = 0; auto conf = configurations.rbegin(); while (conf != configurations.rend()) { @@ -2160,10 +2161,10 @@ namespace aft } leader_id.reset(); - state->leadership_state = kv::LeadershipState::None; + state->leadership_state = ccf::kv::LeadershipState::None; } - state->membership_state = kv::MembershipState::Retired; + state->membership_state = ccf::kv::MembershipState::Retired; state->retirement_phase = phase; } @@ -2221,7 +2222,7 @@ namespace aft // idx. void update_commit() { - if (state->leadership_state != kv::LeadershipState::Leader) + if (state->leadership_state != ccf::kv::LeadershipState::Leader) { throw std::logic_error( "update_commit() must only be called while this node is leader"); @@ -2357,11 +2358,11 @@ namespace aft state->commit_idx = idx; if ( is_retired() && - state->retirement_phase == kv::RetirementPhase::Signed && + state->retirement_phase == ccf::kv::RetirementPhase::Signed && state->retirement_committable_idx.has_value() && idx >= state->retirement_committable_idx.value()) { - become_retired(idx, kv::RetirementPhase::Completed); + become_retired(idx, ccf::kv::RetirementPhase::Completed); } RAFT_DEBUG_FMT("Compacting..."); @@ -2457,27 +2458,27 @@ namespace aft } if ( - state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::Signed) + state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::Signed) { assert(state->retirement_committable_idx.has_value()); if (state->retirement_committable_idx.value() > idx) { state->retirement_committable_idx = std::nullopt; - state->retirement_phase = kv::RetirementPhase::Ordered; + state->retirement_phase = ccf::kv::RetirementPhase::Ordered; } } if ( - state->membership_state == kv::MembershipState::Retired && - state->retirement_phase == kv::RetirementPhase::Ordered) + state->membership_state == ccf::kv::MembershipState::Retired && + state->retirement_phase == ccf::kv::RetirementPhase::Ordered) { assert(state->retirement_idx.has_value()); if (state->retirement_idx.value() > idx) { state->retirement_idx = std::nullopt; state->retirement_phase = std::nullopt; - state->membership_state = kv::MembershipState::Active; + state->membership_state = ccf::kv::MembershipState::Active; RAFT_DEBUG_FMT("Becoming Active after rollback"); } } @@ -2558,7 +2559,7 @@ namespace aft all_other_nodes.try_emplace( node_info.first, node_info.second, index, 0); - if (state->leadership_state == kv::LeadershipState::Leader) + if (state->leadership_state == ccf::kv::LeadershipState::Leader) { send_append_entries(node_info.first, index); } diff --git a/src/consensus/aft/raft_types.h b/src/consensus/aft/raft_types.h index 5f3f160d2dde..b62fb8d833b0 100644 --- a/src/consensus/aft/raft_types.h +++ b/src/consensus/aft/raft_types.h @@ -19,11 +19,11 @@ namespace aft using Index = uint64_t; using Term = uint64_t; using Node2NodeMsg = uint64_t; - using Nonce = crypto::Sha256Hash; + using Nonce = ccf::crypto::Sha256Hash; using ReplyCallback = std::function&& data)>; @@ -34,12 +34,13 @@ namespace aft public: virtual ~Store() {} virtual void compact(Index v) = 0; - virtual void rollback(const kv::TxID& tx_id, Term term_of_next_version) = 0; + virtual void rollback( + const ccf::kv::TxID& tx_id, Term term_of_next_version) = 0; virtual void initialise_term(Term t) = 0; - virtual std::unique_ptr deserialize( + virtual std::unique_ptr deserialize( const std::vector data, bool public_only = false, - const std::optional& expected_txid = std::nullopt) = 0; + const std::optional& expected_txid = std::nullopt) = 0; }; template @@ -60,7 +61,8 @@ namespace aft } } - void rollback(const kv::TxID& tx_id, Term term_of_next_version) override + void rollback( + const ccf::kv::TxID& tx_id, Term term_of_next_version) override { auto p = x.lock(); if (p) @@ -78,10 +80,10 @@ namespace aft } } - std::unique_ptr deserialize( + std::unique_ptr deserialize( const std::vector data, bool public_only = false, - const std::optional& expected_txid = std::nullopt) override + const std::optional& expected_txid = std::nullopt) override { auto p = x.lock(); if (p) diff --git a/src/consensus/aft/test/committable_suffix.cpp b/src/consensus/aft/test/committable_suffix.cpp index c3b58a8db2bb..09eb276eb278 100644 --- a/src/consensus/aft/test/committable_suffix.cpp +++ b/src/consensus/aft/test/committable_suffix.cpp @@ -205,7 +205,7 @@ DOCTEST_TEST_CASE("Retention of dead leader's commit") DOCTEST_INFO("Entry at 1.1 is received by all nodes"); { auto entry = make_ledger_entry(1, 1); - rA.replicate(kv::BatchVector{{1, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{1, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 1); DOCTEST_REQUIRE(rA.get_committed_seqno() == 0); // Size limit was reached, so periodic is not needed @@ -235,21 +235,21 @@ DOCTEST_TEST_CASE("Retention of dead leader's commit") "committed"); { auto entry = make_ledger_entry(1, 2); - rA.replicate(kv::BatchVector{{2, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{2, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 2); DOCTEST_REQUIRE(rA.get_committed_seqno() == 1); // Size limit was reached, so periodic is not needed // rA.periodic(request_timeout); entry = make_ledger_entry(1, 3); - rA.replicate(kv::BatchVector{{3, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{3, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 3); DOCTEST_REQUIRE(rA.get_committed_seqno() == 1); // Size limit was reached, so periodic is not needed // rA.periodic(request_timeout); entry = make_ledger_entry(1, 4); - rA.replicate(kv::BatchVector{{4, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{4, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 4); DOCTEST_REQUIRE(rA.get_committed_seqno() == 1); // Size limit was reached, so periodic is not needed @@ -283,7 +283,7 @@ DOCTEST_TEST_CASE("Retention of dead leader's commit") "committed"); { auto entry = make_ledger_entry(1, 5); - rA.replicate(kv::BatchVector{{5, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{5, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 5); // Size limit was reached, so periodic is not needed // rB.periodic(request_timeout); @@ -350,11 +350,11 @@ DOCTEST_TEST_CASE("Retention of dead leader's commit") DOCTEST_INFO("Node B writes some entries, though they are lost"); { auto entry = make_ledger_entry(2, 6); - rB.replicate(kv::BatchVector{{6, entry, true, hooks}}, 2); + rB.replicate(ccf::kv::BatchVector{{6, entry, true, hooks}}, 2); DOCTEST_REQUIRE(rB.get_last_idx() == 6); entry = make_ledger_entry(2, 7); - rB.replicate(kv::BatchVector{{7, entry, true, hooks}}, 2); + rB.replicate(ccf::kv::BatchVector{{7, entry, true, hooks}}, 2); DOCTEST_REQUIRE(rB.get_last_idx() == 7); // Size limit was reached, so periodic is not needed @@ -401,15 +401,15 @@ DOCTEST_TEST_CASE("Retention of dead leader's commit") DOCTEST_REQUIRE("Node C produces 3.5, 3.6, and 3.7"); { auto entry = make_ledger_entry(3, 5); - rC.replicate(kv::BatchVector{{5, entry, true, hooks}}, 3); + rC.replicate(ccf::kv::BatchVector{{5, entry, true, hooks}}, 3); DOCTEST_REQUIRE(rC.get_last_idx() == 5); entry = make_ledger_entry(3, 6); - rC.replicate(kv::BatchVector{{6, entry, true, hooks}}, 3); + rC.replicate(ccf::kv::BatchVector{{6, entry, true, hooks}}, 3); DOCTEST_REQUIRE(rC.get_last_idx() == 6); entry = make_ledger_entry(3, 7); - rC.replicate(kv::BatchVector{{7, entry, true, hooks}}, 3); + rC.replicate(ccf::kv::BatchVector{{7, entry, true, hooks}}, 3); DOCTEST_REQUIRE(rC.get_last_idx() == 7); // The early AppendEntries that describe this are lost @@ -598,7 +598,7 @@ DOCTEST_TEST_CASE_TEMPLATE("Multi-term divergence", T, WorstCase, RandomCase) { auto entry = make_ledger_entry(primary.get_view(), idx); primary.replicate( - kv::BatchVector{{idx, entry, true, hooks}}, primary.get_view()); + ccf::kv::BatchVector{{idx, entry, true, hooks}}, primary.get_view()); } // All related AppendEntries are lost @@ -627,9 +627,9 @@ DOCTEST_TEST_CASE_TEMPLATE("Multi-term divergence", T, WorstCase, RandomCase) // be committed auto entry = make_ledger_entry(1, 1); - rA.replicate(kv::BatchVector{{1, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{1, entry, true, hooks}}, 1); entry = make_ledger_entry(1, 2); - rA.replicate(kv::BatchVector{{2, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{2, entry, true, hooks}}, 1); DOCTEST_REQUIRE(rA.get_last_idx() == 2); DOCTEST_REQUIRE(rA.get_committed_seqno() == 0); // Size limit was reached, so periodic is not needed @@ -662,18 +662,18 @@ DOCTEST_TEST_CASE_TEMPLATE("Multi-term divergence", T, WorstCase, RandomCase) // Node A produces 2 additional entries that A and B have, and 2 additional // entries that are only present on A entry = make_ledger_entry(1, 3); - rA.replicate(kv::BatchVector{{3, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{3, entry, true, hooks}}, 1); entry = make_ledger_entry(1, 4); - rA.replicate(kv::BatchVector{{4, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{4, entry, true, hooks}}, 1); keep_messages_for(node_idB, channelsA->messages); DOCTEST_REQUIRE(2 == dispatch_all(nodes, node_idA)); entry = make_ledger_entry(1, 5); - rA.replicate(kv::BatchVector{{5, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{5, entry, true, hooks}}, 1); entry = make_ledger_entry(1, 6); - rA.replicate(kv::BatchVector{{6, entry, true, hooks}}, 1); + rA.replicate(ccf::kv::BatchVector{{6, entry, true, hooks}}, 1); channelsA->messages.clear(); channelsB->messages.clear(); @@ -936,7 +936,7 @@ DOCTEST_TEST_CASE_TEMPLATE("Multi-term divergence", T, WorstCase, RandomCase) const auto seqno = rPrimary.get_last_idx() + 1; auto final_entry = make_ledger_entry(view, seqno); rPrimary.replicate( - kv::BatchVector{{seqno, final_entry, true, hooks}}, view); + ccf::kv::BatchVector{{seqno, final_entry, true, hooks}}, view); rPrimary.periodic(request_timeout); keep_earliest_append_entries_for_each_target(channelsPrimary->messages); diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 8064e412384c..c85802a43501 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -40,11 +40,11 @@ int main(int argc, char** argv) // Log all raft steps to stdout (python wrapper raft_scenario_runner.py // filters them). #ifdef CCF_RAFT_TRACING - logger::config::add_json_console_logger(); + ccf::logger::config::add_json_console_logger(); #else - logger::config::add_text_console_logger(); + ccf::logger::config::add_text_console_logger(); #endif - logger::config::level() = LoggerLevel::DEBUG; + ccf::logger::config::level() = LoggerLevel::DEBUG; threading::ThreadMessaging::init(1); diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 486d6f180427..3eacddc14e7d 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -17,7 +17,8 @@ #ifdef CCF_RAFT_TRACING # define RAFT_DRIVER_PRINT(...) \ std::cout << " " << fmt::format(__VA_ARGS__) \ - << fmt::format(" (ts={})", logger::logical_clock) << std::endl; + << fmt::format(" (ts={})", ccf::logger::logical_clock) \ + << std::endl; #else # define RAFT_DRIVER_PRINT(...) \ std::cout << " " << fmt::format(__VA_ARGS__) << std::endl; @@ -47,8 +48,8 @@ struct LedgerStubProxy_Mermaid : public aft::LedgerStubProxy void put_entry( const std::vector& data, bool globally_committable, - kv::Term term, - kv::Version index) override + ccf::kv::Term term, + ccf::kv::Version index) override { RAFT_DRIVER_PRINT( "{}->>{}: [ledger] appending: {}.{}={}", @@ -77,7 +78,7 @@ struct LoggingStubStore_Mermaid : public aft::LoggingStubStoreConfig aft::LoggingStubStoreConfig::compact(idx); } - void rollback(const kv::TxID& tx_id, aft::Term t) override + void rollback(const ccf::kv::TxID& tx_id, aft::Term t) override { RAFT_DRIVER_PRINT( "{}->>{}: [KV] rolling back to {}.{}, in term {}", @@ -123,8 +124,9 @@ class RaftDriver std::vector data, const size_t lineno, bool committable = false, - const std::optional& configuration = std::nullopt, - const std::optional& retired_committed = + const std::optional& configuration = + std::nullopt, + const std::optional& retired_committed = std::nullopt) { const auto opt = find_primary_in_term(term_s, lineno); @@ -149,7 +151,7 @@ class RaftDriver configuration.has_value() ? "reconfiguration" : "raw"); aft::ReplicatedDataType type = aft::ReplicatedDataType::raw; - auto hooks = std::make_shared(); + auto hooks = std::make_shared(); if (configuration.has_value() && retired_committed.has_value()) { throw std::logic_error( @@ -183,7 +185,7 @@ class RaftDriver auto s = nlohmann::json(aft::ReplicatedData{type, data}).dump(); auto d = std::make_shared>(s.begin(), s.end()); - raft->replicate(kv::BatchVector{{idx, d, committable, hooks}}, term); + raft->replicate(ccf::kv::BatchVector{{idx, d, committable, hooks}}, term); } void add_node(ccf::NodeId node_id) @@ -198,7 +200,7 @@ class RaftDriver std::make_shared(node_id), nullptr); kv->set_set_retired_committed_hook( - [raft](aft::Index idx, const std::vector& node_ids) { + [raft](aft::Index idx, const std::vector& node_ids) { raft->set_retired_committed(idx, node_ids); }); raft->start_ticking(); @@ -219,7 +221,7 @@ class RaftDriver { // Unrealistic way to create network. Initial configuration is automatically // added to all nodes. - kv::Configuration::Nodes configuration; + ccf::kv::Configuration::Nodes configuration; for (auto const& n : node_ids) { add_node(n); @@ -247,7 +249,7 @@ class RaftDriver { throw std::logic_error("Start node already exists"); } - kv::Configuration::Nodes configuration; + ccf::kv::Configuration::Nodes configuration; add_node(start_node_id); configuration.try_emplace(start_node_id); _nodes[start_node_id].raft->force_become_primary(); @@ -263,7 +265,7 @@ class RaftDriver const std::vector& node_ids, const size_t lineno) { - kv::Configuration::Nodes retired_committed; + ccf::kv::Configuration::Nodes retired_committed; for (const auto& id : node_ids) { if (_nodes.find(id) == _nodes.end()) @@ -287,7 +289,7 @@ class RaftDriver RAFT_DRIVER_PRINT( "Note over {}: Node {} trusted", ccf::NodeId(node_id), node_id); } - kv::Configuration::Nodes configuration; + ccf::kv::Configuration::Nodes configuration; for (const auto& [id, node] : _nodes) { configuration.try_emplace(id); @@ -324,7 +326,7 @@ class RaftDriver } std::set out(nodes_out.begin(), nodes_out.end()); - kv::Configuration::Nodes configuration; + ccf::kv::Configuration::Nodes configuration; for (const auto& [id, node] : _nodes) { if (out.find(id) == out.end()) @@ -351,7 +353,7 @@ class RaftDriver std::vector node_ids, const size_t lineno) { - kv::Configuration::Nodes configuration; + ccf::kv::Configuration::Nodes configuration; for (const auto& node_id_s : node_ids) { ccf::NodeId node_id(node_id_s); diff --git a/src/consensus/aft/test/enclave.cpp b/src/consensus/aft/test/enclave.cpp index b45c3f4e064c..e1cd1ee1edba 100644 --- a/src/consensus/aft/test/enclave.cpp +++ b/src/consensus/aft/test/enclave.cpp @@ -73,9 +73,9 @@ TEST_CASE("Enclave record") bool globally_committable = false; const std::vector entry = {'a', 'b', 'c'}; - kv::SerialisedEntryHeader entry_header; + ccf::kv::SerialisedEntryHeader entry_header; - std::vector tx(kv::serialised_entry_header_size + entry.size()); + std::vector tx(ccf::kv::serialised_entry_header_size + entry.size()); auto tx_ = tx.data(); auto size_ = tx.size(); serialized::write(tx_, size_, entry_header); diff --git a/src/consensus/aft/test/logging_stub.h b/src/consensus/aft/test/logging_stub.h index b7c42e7ac1cd..680a05acb217 100644 --- a/src/consensus/aft/test/logging_stub.h +++ b/src/consensus/aft/test/logging_stub.h @@ -30,8 +30,8 @@ namespace aft virtual void put_entry( const std::vector& original, bool globally_committable, - kv::Term term, - kv::Version index) + ccf::kv::Term term, + ccf::kv::Version index) { std::lock_guard lock(ledger_access); @@ -98,8 +98,8 @@ namespace aft // Remove the View and Index that were written during put_entry data->erase( data->begin(), - data->begin() + sizeof(size_t) + sizeof(kv::Term) + - sizeof(kv::Version)); + data->begin() + sizeof(size_t) + sizeof(ccf::kv::Term) + + sizeof(ccf::kv::Version)); } return data; @@ -196,7 +196,7 @@ namespace aft void close_channel(const ccf::NodeId& peer_id) override {} - void set_endorsed_node_cert(const crypto::Pem&) override {} + void set_endorsed_node_cert(const ccf::crypto::Pem&) override {} bool have_channel(const ccf::NodeId& nid) override { @@ -231,9 +231,9 @@ namespace aft void initialize( const ccf::NodeId& self_id, - const crypto::Pem& service_cert, - crypto::KeyPairPtr node_kp, - const std::optional& node_cert = std::nullopt) override + const ccf::crypto::Pem& service_cert, + ccf::crypto::KeyPairPtr node_kp, + const std::optional& node_cert = std::nullopt) override {} bool send_encrypted( @@ -286,20 +286,21 @@ namespace aft DECLARE_JSON_TYPE(ReplicatedData); DECLARE_JSON_REQUIRED_FIELDS(ReplicatedData, type, data); - class ConfigurationChangeHook : public kv::ConsensusHook + class ConfigurationChangeHook : public ccf::kv::ConsensusHook { - kv::Configuration::Nodes + ccf::kv::Configuration::Nodes new_configuration; // Absence of node means that node has been retired - kv::Version version; + ccf::kv::Version version; public: ConfigurationChangeHook( - kv::Configuration::Nodes new_configuration_, kv::Version version_) : + ccf::kv::Configuration::Nodes new_configuration_, + ccf::kv::Version version_) : new_configuration(new_configuration_), version(version_) {} - void call(kv::ConfigurableConsensus* consensus) override + void call(ccf::kv::ConfigurableConsensus* consensus) override { auto configuration = consensus->get_latest_configuration_unsafe(); std::unordered_set retired_nodes; @@ -329,7 +330,8 @@ namespace aft } }; - using RCHook = std::function&)>; + using RCHook = + std::function&)>; class LoggingStubStore { @@ -348,31 +350,32 @@ namespace aft virtual void compact(Index i) {} - virtual void rollback(const kv::TxID& tx_id, Term t) {} + virtual void rollback(const ccf::kv::TxID& tx_id, Term t) {} virtual void initialise_term(Term t) {} - kv::Version current_version() + ccf::kv::Version current_version() { - return kv::NoVersion; + return ccf::kv::NoVersion; } - class ExecutionWrapper : public kv::AbstractExecutionWrapper + class ExecutionWrapper : public ccf::kv::AbstractExecutionWrapper { private: - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; aft::Term term; - kv::Version index; + ccf::kv::Version index; std::vector entry; ccf::ClaimsDigest claims_digest; - std::optional commit_evidence_digest = std::nullopt; - kv::ApplyResult result; + std::optional commit_evidence_digest = + std::nullopt; + ccf::kv::ApplyResult result; public: ExecutionWrapper( const std::vector& data_, - const std::optional& expected_txid, - kv::ConsensusHookPtrs&& hooks_) : + const std::optional& expected_txid, + ccf::kv::ConsensusHookPtrs&& hooks_) : hooks(std::move(hooks_)) { const uint8_t* data = data_.data(); @@ -380,17 +383,17 @@ namespace aft const auto committable = serialized::read(data, size); term = serialized::read(data, size); - index = serialized::read(data, size); + index = serialized::read(data, size); entry = serialized::read(data, size, size); - result = - committable ? kv::ApplyResult::PASS_SIGNATURE : kv::ApplyResult::PASS; + result = committable ? ccf::kv::ApplyResult::PASS_SIGNATURE : + ccf::kv::ApplyResult::PASS; if (expected_txid.has_value()) { if (term != expected_txid->term || index != expected_txid->version) { - result = kv::ApplyResult::FAIL; + result = ccf::kv::ApplyResult::FAIL; } } } @@ -400,18 +403,18 @@ namespace aft return std::move(claims_digest); } - std::optional&& consume_commit_evidence_digest() + std::optional&& consume_commit_evidence_digest() override { return std::move(commit_evidence_digest); } - kv::ApplyResult apply(bool track_deletes_on_missing_keys) override + ccf::kv::ApplyResult apply(bool track_deletes_on_missing_keys) override { return result; } - kv::ConsensusHookPtrs& get_hooks() override + ccf::kv::ConsensusHookPtrs& get_hooks() override { return hooks; } @@ -426,7 +429,7 @@ namespace aft return term; } - kv::Version get_index() override + ccf::kv::Version get_index() override { return index; } @@ -447,22 +450,22 @@ namespace aft } }; - virtual std::unique_ptr deserialize( + virtual std::unique_ptr deserialize( const std::vector& data, bool public_only = false, - const std::optional& expected_txid = std::nullopt) + const std::optional& expected_txid = std::nullopt) { - kv::ConsensusHookPtrs hooks = {}; + ccf::kv::ConsensusHookPtrs hooks = {}; return std::make_unique( data, expected_txid, std::move(hooks)); } - bool flag_enabled(kv::AbstractStore::Flag) + bool flag_enabled(ccf::kv::AbstractStore::Flag) { return false; } - void unset_flag(kv::AbstractStore::Flag) {} + void unset_flag(ccf::kv::AbstractStore::Flag) {} }; class LoggingStubStoreConfig : public LoggingStubStore @@ -482,7 +485,7 @@ namespace aft { if (version <= i) { - std::vector retired_committed_node_ids; + std::vector retired_committed_node_ids; for (auto& [node_id, _] : configuration.items()) { retired_committed_node_ids.push_back(node_id); @@ -502,7 +505,7 @@ namespace aft retired_committed_entries.end()); } - virtual void rollback(const kv::TxID& tx_id, Term t) override + virtual void rollback(const ccf::kv::TxID& tx_id, Term t) override { retired_committed_entries.erase( std::remove_if( @@ -512,10 +515,10 @@ namespace aft retired_committed_entries.end()); } - virtual std::unique_ptr deserialize( + virtual std::unique_ptr deserialize( const std::vector& data, bool public_only = false, - const std::optional& expected_txid = std::nullopt) override + const std::optional& expected_txid = std::nullopt) override { // Set reconfiguration hook if there are any new nodes // Read wrapping term and version @@ -523,19 +526,21 @@ namespace aft auto size = data.size(); const auto committable = serialized::read(data_, size); serialized::read(data_, size); - auto version = serialized::read(data_, size); + auto version = serialized::read(data_, size); ReplicatedData r = nlohmann::json::parse(std::span{data_, size}); - kv::ConsensusHookPtrs hooks = {}; + ccf::kv::ConsensusHookPtrs hooks = {}; if (r.type == ReplicatedDataType::reconfiguration) { - kv::Configuration::Nodes configuration = nlohmann::json::parse(r.data); + ccf::kv::Configuration::Nodes configuration = + nlohmann::json::parse(r.data); auto hook = std::make_unique( configuration, version); hooks.push_back(std::move(hook)); } if (r.type == ReplicatedDataType::retired_committed) { - kv::Configuration::Nodes configuration = nlohmann::json::parse(r.data); + ccf::kv::Configuration::Nodes configuration = + nlohmann::json::parse(r.data); retired_committed_entries.emplace_back(version, configuration); } @@ -562,7 +567,7 @@ namespace aft Index, const std::vector&, const ccf::NodeId&, - const crypto::Pem&) + const ccf::crypto::Pem&) {} }; } \ No newline at end of file diff --git a/src/consensus/aft/test/main.cpp b/src/consensus/aft/test/main.cpp index c5e35233cc72..e3a9bac2c6d5 100644 --- a/src/consensus/aft/test/main.cpp +++ b/src/consensus/aft/test/main.cpp @@ -14,7 +14,7 @@ using ms = std::chrono::milliseconds; DOCTEST_TEST_CASE("Single node startup" * doctest::test_suite("single")) { - ccf::NodeId node_id = kv::test::PrimaryNodeId; + ccf::NodeId node_id = ccf::kv::test::PrimaryNodeId; auto kv_store = std::make_shared(node_id); TRaft r0( @@ -26,7 +26,7 @@ DOCTEST_TEST_CASE("Single node startup" * doctest::test_suite("single")) nullptr); r0.start_ticking(); - kv::Configuration::Nodes config; + ccf::kv::Configuration::Nodes config; config.try_emplace(node_id); r0.add_configuration(0, config); @@ -50,7 +50,7 @@ DOCTEST_TEST_CASE("Single node startup" * doctest::test_suite("single")) DOCTEST_TEST_CASE("Single node commit" * doctest::test_suite("single")) { - ccf::NodeId node_id = kv::test::PrimaryNodeId; + ccf::NodeId node_id = ccf::kv::test::PrimaryNodeId; auto kv_store = std::make_shared(node_id); TRaft r0( @@ -80,9 +80,9 @@ DOCTEST_TEST_CASE("Single node commit" * doctest::test_suite("single")) entry->push_back(2); entry->push_back(3); - auto hooks = std::make_shared(); + auto hooks = std::make_shared(); - r0.replicate(kv::BatchVector{{i, entry, true, hooks}}, 1); + r0.replicate(ccf::kv::BatchVector{{i, entry, true, hooks}}, 1); DOCTEST_REQUIRE(r0.get_last_idx() == i); DOCTEST_REQUIRE(r0.get_committed_seqno() == i); } @@ -91,10 +91,10 @@ DOCTEST_TEST_CASE("Single node commit" * doctest::test_suite("single")) DOCTEST_TEST_CASE( "Multiple nodes startup and election" * doctest::test_suite("multiple")) { - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; - ccf::NodeId node_id2 = kv::test::SecondBackupNodeId; - ccf::NodeId node_id3 = kv::test::ThirdBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; + ccf::NodeId node_id2 = ccf::kv::test::SecondBackupNodeId; + ccf::NodeId node_id3 = ccf::kv::test::ThirdBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); @@ -275,9 +275,9 @@ DOCTEST_TEST_CASE( DOCTEST_TEST_CASE( "Multiple nodes append entries" * doctest::test_suite("multiple")) { - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; - ccf::NodeId node_id2 = kv::test::SecondBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; + ccf::NodeId node_id2 = ccf::kv::test::SecondBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); @@ -359,12 +359,13 @@ DOCTEST_TEST_CASE( DOCTEST_INFO("Try to replicate on a follower, and fail"); std::vector entry = {1, 2, 3}; auto data = std::make_shared>(entry); - auto hooks = std::make_shared(); + auto hooks = std::make_shared(); DOCTEST_REQUIRE_FALSE( - r1.replicate(kv::BatchVector{{1, data, true, hooks}}, 1)); + r1.replicate(ccf::kv::BatchVector{{1, data, true, hooks}}, 1)); DOCTEST_INFO("Tell the leader to replicate a message"); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{1, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{1, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 1); // The test ledger adds its own header. Confirm that the expected data is @@ -411,9 +412,9 @@ DOCTEST_TEST_CASE( DOCTEST_TEST_CASE("Multiple nodes late join" * doctest::test_suite("multiple")) { - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; - ccf::NodeId node_id2 = kv::test::SecondBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; + ccf::NodeId node_id2 = ccf::kv::test::SecondBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); @@ -475,8 +476,9 @@ DOCTEST_TEST_CASE("Multiple nodes late join" * doctest::test_suite("multiple")) std::vector first_entry = {1, 2, 3}; auto data = std::make_shared>(first_entry); - auto hooks = std::make_shared(); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{1, data, true, hooks}}, 1)); + auto hooks = std::make_shared(); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{1, data, true, hooks}}, 1)); r0.periodic(request_timeout); DOCTEST_REQUIRE( @@ -528,8 +530,8 @@ DOCTEST_TEST_CASE("Multiple nodes late join" * doctest::test_suite("multiple")) DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); @@ -548,7 +550,7 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) std::make_shared(), std::make_shared(node_id1), nullptr); - auto hooks = std::make_shared(); + auto hooks = std::make_shared(); aft::Configuration::Nodes config0; config0[node_id0] = {}; @@ -586,8 +588,10 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) std::vector second_entry = {2, 2, 2}; auto data_2 = std::make_shared>(second_entry); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{1, data_1, true, hooks}}, 1)); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{2, data_2, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{1, data_1, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{2, data_2, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 2); r0.periodic(request_timeout); DOCTEST_REQUIRE(r0c->messages.size() == 1); @@ -608,7 +612,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector third_entry = {3, 3, 3}; auto data = std::make_shared>(third_entry); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{3, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{3, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 3); // Simulate that the append entries was not deserialised successfully @@ -640,7 +645,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector fourth_entry = {4, 4, 4}; auto data = std::make_shared>(fourth_entry); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{4, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{4, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 4); r0.periodic(request_timeout); DOCTEST_REQUIRE(r0c->messages.size() == 1); @@ -653,7 +659,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector fifth_entry = {5, 5, 5}; auto data = std::make_shared>(fifth_entry); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{5, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{5, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 5); r0.periodic(request_timeout); DOCTEST_REQUIRE(r0c->messages.size() == 1); @@ -682,7 +689,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector entry_6 = {6, 6, 6}; auto data = std::make_shared>(entry_6); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{6, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{6, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 6); } const auto last_correct_version = r0.ledger->ledger.size(); @@ -691,7 +699,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector entry_7 = {7, 7, 7}; auto data = std::make_shared>(entry_7); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{7, data, true, hooks}}, 1)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{7, data, true, hooks}}, 1)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 7); dead_branch = r0.ledger->ledger.back(); } @@ -712,7 +721,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector entry_7b = {7, 7, 'b'}; auto data = std::make_shared>(entry_7b); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{7, data, true, hooks}}, 4)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{7, data, true, hooks}}, 4)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 7); live_branch = r0.ledger->ledger.back(); } @@ -720,7 +730,8 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) { std::vector entry_8 = {8, 8, 8}; auto data = std::make_shared>(entry_8); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{8, data, true, hooks}}, 4)); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{8, data, true, hooks}}, 4)); DOCTEST_REQUIRE(r0.ledger->ledger.size() == 8); DOCTEST_REQUIRE(r0.ledger->ledger.size() > last_correct_version); } @@ -769,11 +780,11 @@ DOCTEST_TEST_CASE("Recv append entries logic" * doctest::test_suite("multiple")) DOCTEST_TEST_CASE("Exceed append entries limit") { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; - ccf::NodeId node_id2 = kv::test::SecondBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; + ccf::NodeId node_id2 = ccf::kv::test::SecondBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); @@ -847,8 +858,9 @@ DOCTEST_TEST_CASE("Exceed append entries limit") for (size_t i = 1; i <= num_big_entries; ++i) { - auto hooks = std::make_shared(); - DOCTEST_REQUIRE(r0.replicate(kv::BatchVector{{i, data, true, hooks}}, 1)); + auto hooks = std::make_shared(); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{i, data, true, hooks}}, 1)); const auto received_ae = dispatch_all_and_DOCTEST_CHECK( nodes, node_id0, r0c->messages, [&i](const auto& msg) { @@ -864,9 +876,9 @@ DOCTEST_TEST_CASE("Exceed append entries limit") for (size_t i = num_big_entries + 1; i <= individual_entries; ++i) { - auto hooks = std::make_shared(); + auto hooks = std::make_shared(); DOCTEST_REQUIRE( - r0.replicate(kv::BatchVector{{i, smaller_data, true, hooks}}, 1)); + r0.replicate(ccf::kv::BatchVector{{i, smaller_data, true, hooks}}, 1)); dispatch_all(nodes, node_id0, r0c->messages); } @@ -925,8 +937,8 @@ DOCTEST_TEST_CASE( "Nodes only run for election when they should" * doctest::test_suite("multiple")) { - ccf::NodeId node_id0 = kv::test::PrimaryNodeId; - ccf::NodeId node_id1 = kv::test::FirstBackupNodeId; + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; auto kv_store0 = std::make_shared(node_id0); auto kv_store1 = std::make_shared(node_id1); diff --git a/src/consensus/aft/test/test_common.h b/src/consensus/aft/test/test_common.h index dcf5011ecdce..cbbe6810766e 100644 --- a/src/consensus/aft/test/test_common.h +++ b/src/consensus/aft/test/test_common.h @@ -24,7 +24,7 @@ static const std::chrono::milliseconds election_timeout = election_timeout_; static const ccf::consensus::Configuration raft_settings{ request_timeout_, election_timeout_, max_uncommitted_tx_count_}; -static auto hooks = std::make_shared(); +static auto hooks = std::make_shared(); static aft::ChannelStubProxy* channel_stub_proxy(const TRaft& r) { diff --git a/src/consensus/aft/test/view_history.cpp b/src/consensus/aft/test/view_history.cpp index 493a0db13159..63a65554c875 100644 --- a/src/consensus/aft/test/view_history.cpp +++ b/src/consensus/aft/test/view_history.cpp @@ -19,8 +19,8 @@ TEST_CASE("Advancing view history" * doctest::test_suite("viewhistory")) CHECK(history.view_at(3) == ViewHistory::InvalidView); CHECK(history.view_at(4) == ViewHistory::InvalidView); - CHECK(history.start_of_view(1) == kv::NoVersion); - CHECK(history.end_of_view(1) == kv::NoVersion); + CHECK(history.start_of_view(1) == ccf::kv::NoVersion); + CHECK(history.end_of_view(1) == ccf::kv::NoVersion); } { @@ -64,9 +64,9 @@ TEST_CASE("Advancing view history" * doctest::test_suite("viewhistory")) CHECK(history.start_of_view(2) == 3); CHECK(history.end_of_view(2) == 3); CHECK(history.start_of_view(3) == 4); - CHECK(history.end_of_view(3) == kv::NoVersion); - CHECK(history.start_of_view(4) == kv::NoVersion); - CHECK(history.end_of_view(4) == kv::NoVersion); + CHECK(history.end_of_view(3) == ccf::kv::NoVersion); + CHECK(history.start_of_view(4) == ccf::kv::NoVersion); + CHECK(history.end_of_view(4) == ccf::kv::NoVersion); } } @@ -143,14 +143,14 @@ TEST_CASE("Edge case view histories" * doctest::test_suite("viewhistory")) CHECK(history.view_at(4) == 4); CHECK(history.start_of_view(4) == 2); - CHECK(history.end_of_view(4) == kv::NoVersion); - - CHECK(history.start_of_view(1) == kv::NoVersion); - CHECK(history.end_of_view(1) == kv::NoVersion); - CHECK(history.start_of_view(2) == kv::NoVersion); - CHECK(history.end_of_view(2) == kv::NoVersion); - CHECK(history.start_of_view(3) == kv::NoVersion); - CHECK(history.end_of_view(3) == kv::NoVersion); + CHECK(history.end_of_view(4) == ccf::kv::NoVersion); + + CHECK(history.start_of_view(1) == ccf::kv::NoVersion); + CHECK(history.end_of_view(1) == ccf::kv::NoVersion); + CHECK(history.start_of_view(2) == ccf::kv::NoVersion); + CHECK(history.end_of_view(2) == ccf::kv::NoVersion); + CHECK(history.start_of_view(3) == ccf::kv::NoVersion); + CHECK(history.end_of_view(3) == ccf::kv::NoVersion); } } @@ -168,12 +168,12 @@ TEST_CASE("Initialised view histories" * doctest::test_suite("viewhistory")) CHECK(history.start_of_view(1) == 2); CHECK(history.end_of_view(1) == 3); - CHECK(history.start_of_view(2) == kv::NoVersion); - CHECK(history.end_of_view(2) == kv::NoVersion); + CHECK(history.start_of_view(2) == ccf::kv::NoVersion); + CHECK(history.end_of_view(2) == ccf::kv::NoVersion); CHECK(history.start_of_view(3) == 4); CHECK(history.end_of_view(3) == 9); CHECK(history.start_of_view(4) == 10); - CHECK(history.end_of_view(4) == kv::NoVersion); + CHECK(history.end_of_view(4) == ccf::kv::NoVersion); CHECK_THROWS(history.initialise({2, 1})); CHECK_THROWS(history.initialise({1, 2, 1})); @@ -203,8 +203,8 @@ TEST_CASE("Initialised view histories" * doctest::test_suite("viewhistory")) CHECK(history.view_at(19) == 6); CHECK(history.view_at(20) == 6); - CHECK(history.start_of_view(1) == kv::NoVersion); - CHECK(history.end_of_view(1) == kv::NoVersion); + CHECK(history.start_of_view(1) == ccf::kv::NoVersion); + CHECK(history.end_of_view(1) == ccf::kv::NoVersion); CHECK(history.start_of_view(3) == 3); CHECK(history.end_of_view(3) == 4); CHECK(history.start_of_view(4) == 5); @@ -212,7 +212,7 @@ TEST_CASE("Initialised view histories" * doctest::test_suite("viewhistory")) CHECK(history.start_of_view(5) == 6); CHECK(history.end_of_view(5) == 11); CHECK(history.start_of_view(6) == 12); - CHECK(history.end_of_view(6) == kv::NoVersion); + CHECK(history.end_of_view(6) == ccf::kv::NoVersion); } } @@ -234,23 +234,28 @@ TEST_CASE( { INFO("Test that view history is correct"); - REQUIRE(history.get_history_until(kv::NoVersion).size() == 0); - REQUIRE(history.get_history_until(1) == std::vector({1})); - REQUIRE(history.get_history_until(2) == std::vector({1, 2})); - REQUIRE(history.get_history_until(3) == std::vector({1, 2})); - REQUIRE(history.get_history_until(4) == std::vector({1, 2})); + REQUIRE(history.get_history_until(ccf::kv::NoVersion).size() == 0); + REQUIRE(history.get_history_until(1) == std::vector({1})); REQUIRE( - history.get_history_until(5) == std::vector({1, 2, 5, 5})); + history.get_history_until(2) == std::vector({1, 2})); REQUIRE( - history.get_history_until(9) == std::vector({1, 2, 5, 5})); + history.get_history_until(3) == std::vector({1, 2})); + REQUIRE( + history.get_history_until(4) == std::vector({1, 2})); + REQUIRE( + history.get_history_until(5) == + std::vector({1, 2, 5, 5})); + REQUIRE( + history.get_history_until(9) == + std::vector({1, 2, 5, 5})); REQUIRE( history.get_history_until(10) == - std::vector({1, 2, 5, 5, 10})); + std::vector({1, 2, 5, 5, 10})); REQUIRE( history.get_history_until(11) == - std::vector({1, 2, 5, 5, 10})); + std::vector({1, 2, 5, 5, 10})); REQUIRE( history.get_history_until() == - std::vector({1, 2, 5, 5, 10})); + std::vector({1, 2, 5, 5, 10})); } } diff --git a/src/consensus/ledger_enclave.h b/src/consensus/ledger_enclave.h index 55fca34c16cf..d90c7d1c534e 100644 --- a/src/consensus/ledger_enclave.h +++ b/src/consensus/ledger_enclave.h @@ -25,8 +25,9 @@ namespace consensus */ static std::vector get_entry(const uint8_t*& data, size_t& size) { - auto header = serialized::peek(data, size); - size_t entry_size = kv::serialised_entry_header_size + header.size; + auto header = + serialized::peek(data, size); + size_t entry_size = ccf::kv::serialised_entry_header_size + header.size; std::vector entry(data, data + entry_size); serialized::skip(data, size, entry_size); return entry; @@ -51,8 +52,8 @@ namespace consensus void put_entry( const std::vector& entry, bool globally_committable, - kv::Term term, - kv::Version index) + ccf::kv::Term term, + ccf::kv::Version index) { put_entry(entry.data(), entry.size(), globally_committable, term, index); } @@ -72,8 +73,8 @@ namespace consensus const uint8_t* data, size_t size, bool globally_committable, - kv::Term term, - kv::Version index) + ccf::kv::Term term, + ccf::kv::Version index) { serializer::ByteRange byte_range = {data, size}; RINGBUFFER_WRITE_MESSAGE( @@ -90,7 +91,8 @@ namespace consensus */ static void skip_entry(const uint8_t*& data, size_t& size) { - auto header = serialized::read(data, size); + auto header = + serialized::read(data, size); serialized::skip(data, size, header.size); } diff --git a/src/crypto/base64.cpp b/src/crypto/base64.cpp index a1933f03b8db..99dc3b03b181 100644 --- a/src/crypto/base64.cpp +++ b/src/crypto/base64.cpp @@ -5,7 +5,7 @@ #include "ccf/ds/logger.h" -namespace crypto +namespace ccf::crypto { using Base64Impl = Base64_openssl; diff --git a/src/crypto/certs.h b/src/crypto/certs.h index 96c6428cecc6..1164bade373f 100644 --- a/src/crypto/certs.h +++ b/src/crypto/certs.h @@ -9,7 +9,7 @@ #include #include -namespace crypto +namespace ccf::crypto { static std::string compute_cert_valid_to_string( const std::string& valid_from, size_t validity_period_days) @@ -17,9 +17,9 @@ namespace crypto using namespace std::chrono_literals; // Note: As per RFC 5280, the validity period runs until "notAfter" // _inclusive_ so substract one second from the validity period. - auto valid_to = ds::time_point_from_string(valid_from) + + auto valid_to = ::ds::time_point_from_string(valid_from) + std::chrono::days(validity_period_days) - 1s; - return ds::to_x509_time_string(valid_to); + return ::ds::to_x509_time_string(valid_to); } static Pem create_self_signed_cert( diff --git a/src/crypto/csr.h b/src/crypto/csr.h index 90f955358ab8..258934ac9c2a 100644 --- a/src/crypto/csr.h +++ b/src/crypto/csr.h @@ -7,7 +7,7 @@ #include -namespace crypto +namespace ccf::crypto { /** Extracts the public key from a certificate signing request (CSR). * @param signing_request CSR to extract the public key from diff --git a/src/crypto/ecdsa.cpp b/src/crypto/ecdsa.cpp index 52a7b5ef12ad..7ad640631b45 100644 --- a/src/crypto/ecdsa.cpp +++ b/src/crypto/ecdsa.cpp @@ -9,7 +9,7 @@ #include #include -namespace crypto +namespace ccf::crypto { std::vector ecdsa_sig_from_r_s( const uint8_t* r, diff --git a/src/crypto/eddsa_key_pair.cpp b/src/crypto/eddsa_key_pair.cpp index 2da0417a368e..1b943ce19df6 100644 --- a/src/crypto/eddsa_key_pair.cpp +++ b/src/crypto/eddsa_key_pair.cpp @@ -12,7 +12,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using PublicKeyImpl = EdDSAPublicKey_OpenSSL; using KeyPairImpl = EdDSAKeyPair_OpenSSL; diff --git a/src/crypto/entropy.cpp b/src/crypto/entropy.cpp index 8a1dbcc48ab0..0ec986c6fd97 100644 --- a/src/crypto/entropy.cpp +++ b/src/crypto/entropy.cpp @@ -5,7 +5,7 @@ #include "openssl/entropy.h" -namespace crypto +namespace ccf::crypto { EntropyPtr get_entropy() { diff --git a/src/crypto/hash.cpp b/src/crypto/hash.cpp index e193a44e79a7..c787d9893917 100644 --- a/src/crypto/hash.cpp +++ b/src/crypto/hash.cpp @@ -6,7 +6,7 @@ #include "ccf/crypto/hkdf.h" #include "ccf/crypto/sha256.h" -namespace crypto +namespace ccf::crypto { void default_sha256(const std::span& data, uint8_t* h) { diff --git a/src/crypto/hmac.cpp b/src/crypto/hmac.cpp index 5c3c0fb43662..0678176a6c17 100644 --- a/src/crypto/hmac.cpp +++ b/src/crypto/hmac.cpp @@ -6,7 +6,7 @@ #include -namespace crypto +namespace ccf::crypto { namespace OpenSSL { diff --git a/src/crypto/key_exchange.h b/src/crypto/key_exchange.h index a6886374f4de..d1b90b05aca1 100644 --- a/src/crypto/key_exchange.h +++ b/src/crypto/key_exchange.h @@ -19,9 +19,9 @@ namespace tls class KeyExchangeContext { private: - crypto::KeyPairPtr own_key; - crypto::PublicKeyPtr peer_key; - crypto::CurveID curve; + ccf::crypto::KeyPairPtr own_key; + ccf::crypto::PublicKeyPtr peer_key; + ccf::crypto::CurveID curve; std::vector shared_secret; void compute_shared_secret() @@ -41,7 +41,7 @@ namespace tls } public: - KeyExchangeContext() : curve(crypto::CurveID::SECP384R1) {} + KeyExchangeContext() : curve(ccf::crypto::CurveID::SECP384R1) {} ~KeyExchangeContext() {} @@ -91,15 +91,15 @@ namespace tls std::vector tmp(ks.begin(), ks.end()); tmp.erase(tmp.begin()); - int nid = crypto::PublicKey_OpenSSL::get_openssl_group_id(curve); - auto pk = crypto::key_from_raw_ec_point(tmp, nid); + int nid = ccf::crypto::PublicKey_OpenSSL::get_openssl_group_id(curve); + auto pk = ccf::crypto::key_from_raw_ec_point(tmp, nid); if (!pk) { throw std::runtime_error("Failed to parse peer key share"); } - peer_key = std::make_shared(pk); + peer_key = std::make_shared(pk); shared_secret.clear(); } diff --git a/src/crypto/key_pair.cpp b/src/crypto/key_pair.cpp index 020ed6402bb6..f3d87dc1f186 100644 --- a/src/crypto/key_pair.cpp +++ b/src/crypto/key_pair.cpp @@ -12,7 +12,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using PublicKeyImpl = PublicKey_OpenSSL; using KeyPairImpl = KeyPair_OpenSSL; diff --git a/src/crypto/key_wrap.cpp b/src/crypto/key_wrap.cpp index 9868a96308ac..b1e3006ee211 100644 --- a/src/crypto/key_wrap.cpp +++ b/src/crypto/key_wrap.cpp @@ -13,7 +13,7 @@ #include #include -namespace crypto +namespace ccf::crypto { // With inspiration from // http://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/cos01/pkcs11-curr-v2.40-cos01.html diff --git a/src/crypto/openssl/base64.h b/src/crypto/openssl/base64.h index 9320d9536e3e..e268aca6a2e4 100644 --- a/src/crypto/openssl/base64.h +++ b/src/crypto/openssl/base64.h @@ -13,7 +13,7 @@ // Inspired by openssl/test/evp_test.c // Ref: https://www.openssl.org/docs/man1.1.1/man3/EVP_DecodeBlock.html -namespace crypto +namespace ccf::crypto { struct Base64_openssl { diff --git a/src/crypto/openssl/cose_verifier.cpp b/src/crypto/openssl/cose_verifier.cpp index f2fe746c6f87..bde1348863f5 100644 --- a/src/crypto/openssl/cose_verifier.cpp +++ b/src/crypto/openssl/cose_verifier.cpp @@ -15,7 +15,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/cose_verifier.h b/src/crypto/openssl/cose_verifier.h index a906f4860dec..6bd61dca037e 100644 --- a/src/crypto/openssl/cose_verifier.h +++ b/src/crypto/openssl/cose_verifier.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class COSEVerifier_OpenSSL : public COSEVerifier { diff --git a/src/crypto/openssl/eddsa_key_pair.cpp b/src/crypto/openssl/eddsa_key_pair.cpp index fee2f384dbdf..1a75d35fe54c 100644 --- a/src/crypto/openssl/eddsa_key_pair.cpp +++ b/src/crypto/openssl/eddsa_key_pair.cpp @@ -5,7 +5,7 @@ #include "openssl_wrappers.h" -namespace crypto +namespace ccf::crypto { EdDSAKeyPair_OpenSSL::EdDSAKeyPair_OpenSSL(CurveID curve_id) { diff --git a/src/crypto/openssl/eddsa_key_pair.h b/src/crypto/openssl/eddsa_key_pair.h index b19637bb47f1..491c282b37fd 100644 --- a/src/crypto/openssl/eddsa_key_pair.h +++ b/src/crypto/openssl/eddsa_key_pair.h @@ -6,7 +6,7 @@ #include "crypto/openssl/eddsa_public_key.h" #include "openssl_wrappers.h" -namespace crypto +namespace ccf::crypto { class EdDSAKeyPair_OpenSSL : public EdDSAPublicKey_OpenSSL, public EdDSAKeyPair diff --git a/src/crypto/openssl/eddsa_public_key.cpp b/src/crypto/openssl/eddsa_public_key.cpp index 1c98c0ac391d..19768c1534d3 100644 --- a/src/crypto/openssl/eddsa_public_key.cpp +++ b/src/crypto/openssl/eddsa_public_key.cpp @@ -5,7 +5,7 @@ #include "crypto/openssl/hash.h" #include "openssl_wrappers.h" -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/eddsa_public_key.h b/src/crypto/openssl/eddsa_public_key.h index 1b06cf47b24d..f4f376c8cb3f 100644 --- a/src/crypto/openssl/eddsa_public_key.h +++ b/src/crypto/openssl/eddsa_public_key.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class EdDSAPublicKey_OpenSSL : public EdDSAPublicKey { diff --git a/src/crypto/openssl/entropy.h b/src/crypto/openssl/entropy.h index fdaede7393da..f5bba949e71f 100644 --- a/src/crypto/openssl/entropy.h +++ b/src/crypto/openssl/entropy.h @@ -10,7 +10,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class Entropy_OpenSSL : public Entropy { diff --git a/src/crypto/openssl/hash.cpp b/src/crypto/openssl/hash.cpp index 290ef85c36fe..ab913267a266 100644 --- a/src/crypto/openssl/hash.cpp +++ b/src/crypto/openssl/hash.cpp @@ -7,7 +7,7 @@ #include #include -namespace crypto +namespace ccf::crypto { namespace OpenSSL { diff --git a/src/crypto/openssl/hash.h b/src/crypto/openssl/hash.h index d8117cbfb3a4..de356c9e5c0c 100644 --- a/src/crypto/openssl/hash.h +++ b/src/crypto/openssl/hash.h @@ -12,7 +12,7 @@ #define FMT_HEADER_ONLY #include -namespace crypto +namespace ccf::crypto { namespace OpenSSL { diff --git a/src/crypto/openssl/key_pair.cpp b/src/crypto/openssl/key_pair.cpp index bb3608c1124d..7bbf3e7a3c2e 100644 --- a/src/crypto/openssl/key_pair.cpp +++ b/src/crypto/openssl/key_pair.cpp @@ -26,17 +26,17 @@ # include #endif -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; static std::map parse_name(const std::string& name) { std::map result; - const auto ns = nonstd::split(name, ","); + const auto ns = ccf::nonstd::split(name, ","); for (const auto& n : ns) { - const auto& [key, value] = nonstd::split_1(n, "="); + const auto& [key, value] = ccf::nonstd::split_1(n, "="); result.emplace( std::string(key.data(), key.size()), std::string(value.data(), value.size())); @@ -457,8 +457,8 @@ namespace crypto std::vector KeyPair_OpenSSL::derive_shared_secret( const PublicKey& peer_key) { - crypto::CurveID cid = peer_key.get_curve_id(); - int nid = crypto::PublicKey_OpenSSL::get_openssl_group_id(cid); + ccf::crypto::CurveID cid = peer_key.get_curve_id(); + int nid = ccf::crypto::PublicKey_OpenSSL::get_openssl_group_id(cid); auto pk = key_from_raw_ec_point(peer_key.public_key_raw(), nid); std::vector shared_secret; diff --git a/src/crypto/openssl/key_pair.h b/src/crypto/openssl/key_pair.h index 201f88fe907b..07e4147bf638 100644 --- a/src/crypto/openssl/key_pair.h +++ b/src/crypto/openssl/key_pair.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class KeyPair_OpenSSL : public PublicKey_OpenSSL, public KeyPair { diff --git a/src/crypto/openssl/openssl_wrappers.h b/src/crypto/openssl/openssl_wrappers.h index 54b21e31d3f3..da54cbde91b8 100644 --- a/src/crypto/openssl/openssl_wrappers.h +++ b/src/crypto/openssl/openssl_wrappers.h @@ -26,7 +26,7 @@ # include #endif -namespace crypto +namespace ccf::crypto { namespace OpenSSL { @@ -341,7 +341,7 @@ namespace crypto Unique_X509_TIME(const std::string& s) : Unique_SSL_OBJECT(ASN1_TIME_new(), ASN1_TIME_free, /*check_null=*/false) { - auto t = ds::to_x509_time_string(s); + auto t = ::ds::to_x509_time_string(s); CHECK1(ASN1_TIME_set_string(*this, t.c_str())); CHECK1(ASN1_TIME_normalize(*this)); } @@ -349,7 +349,7 @@ namespace crypto Unique_SSL_OBJECT(t, ASN1_TIME_free, /*check_null=*/false) {} Unique_X509_TIME(const std::chrono::system_clock::time_point& t) : - Unique_X509_TIME(ds::to_x509_time_string(t)) + Unique_X509_TIME(::ds::to_x509_time_string(t)) {} }; diff --git a/src/crypto/openssl/public_key.cpp b/src/crypto/openssl/public_key.cpp index 20fb5d0db89e..6e1afaefef2a 100644 --- a/src/crypto/openssl/public_key.cpp +++ b/src/crypto/openssl/public_key.cpp @@ -23,7 +23,7 @@ # include #endif -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/public_key.h b/src/crypto/openssl/public_key.h index bc77b80f2f6f..599d34572700 100644 --- a/src/crypto/openssl/public_key.h +++ b/src/crypto/openssl/public_key.h @@ -10,7 +10,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class PublicKey_OpenSSL : public PublicKey { diff --git a/src/crypto/openssl/rsa_key_pair.cpp b/src/crypto/openssl/rsa_key_pair.cpp index affb5f1056de..4c31ddcbfd0f 100644 --- a/src/crypto/openssl/rsa_key_pair.cpp +++ b/src/crypto/openssl/rsa_key_pair.cpp @@ -10,7 +10,7 @@ # include #endif -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/rsa_key_pair.h b/src/crypto/openssl/rsa_key_pair.h index 9c3bb44bdcd9..e51b9bf45661 100644 --- a/src/crypto/openssl/rsa_key_pair.h +++ b/src/crypto/openssl/rsa_key_pair.h @@ -10,7 +10,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class RSAKeyPair_OpenSSL : public RSAPublicKey_OpenSSL, public RSAKeyPair { diff --git a/src/crypto/openssl/rsa_public_key.cpp b/src/crypto/openssl/rsa_public_key.cpp index ca5d1bc5ab72..e508b10e4454 100644 --- a/src/crypto/openssl/rsa_public_key.cpp +++ b/src/crypto/openssl/rsa_public_key.cpp @@ -10,7 +10,7 @@ # include #endif -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/rsa_public_key.h b/src/crypto/openssl/rsa_public_key.h index afe3164fa8aa..2ae58522bce1 100644 --- a/src/crypto/openssl/rsa_public_key.h +++ b/src/crypto/openssl/rsa_public_key.h @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class RSAPublicKey_OpenSSL : public PublicKey_OpenSSL, public RSAPublicKey { diff --git a/src/crypto/openssl/symmetric_key.cpp b/src/crypto/openssl/symmetric_key.cpp index 89edda87adfb..fc18e29eab01 100644 --- a/src/crypto/openssl/symmetric_key.cpp +++ b/src/crypto/openssl/symmetric_key.cpp @@ -11,7 +11,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; diff --git a/src/crypto/openssl/symmetric_key.h b/src/crypto/openssl/symmetric_key.h index 4f1fb7f13875..401d2844aee0 100644 --- a/src/crypto/openssl/symmetric_key.h +++ b/src/crypto/openssl/symmetric_key.h @@ -7,7 +7,7 @@ #include -namespace crypto +namespace ccf::crypto { class KeyAesGcm_OpenSSL : public KeyAesGcm { diff --git a/src/crypto/openssl/verifier.cpp b/src/crypto/openssl/verifier.cpp index b11a42d12455..5ebe4a34547c 100644 --- a/src/crypto/openssl/verifier.cpp +++ b/src/crypto/openssl/verifier.cpp @@ -14,7 +14,7 @@ #include #include -namespace crypto +namespace ccf::crypto { using namespace OpenSSL; @@ -197,7 +197,7 @@ namespace crypto const std::chrono::system_clock::time_point& now) const { auto [from, to] = validity_period(); - auto tp_to = ds::time_point_from_string(to); + auto tp_to = ::ds::time_point_from_string(to); return std::chrono::duration_cast(tp_to - now) .count() + 1; @@ -207,8 +207,8 @@ namespace crypto const std::chrono::system_clock::time_point& now) const { auto [from, to] = validity_period(); - auto tp_from = ds::time_point_from_string(from); - auto tp_to = ds::time_point_from_string(to); + auto tp_from = ::ds::time_point_from_string(from); + auto tp_to = ::ds::time_point_from_string(to); auto total_sec = std::chrono::duration_cast(tp_to - tp_from) .count() + diff --git a/src/crypto/openssl/verifier.h b/src/crypto/openssl/verifier.h index 76a97f8ed910..191e8af4e355 100644 --- a/src/crypto/openssl/verifier.h +++ b/src/crypto/openssl/verifier.h @@ -8,7 +8,7 @@ #include #include -namespace crypto +namespace ccf::crypto { class Verifier_OpenSSL : public Verifier { diff --git a/src/crypto/openssl/x509_time.h b/src/crypto/openssl/x509_time.h index d9d4d1212cc3..93bd2fd9ab36 100644 --- a/src/crypto/openssl/x509_time.h +++ b/src/crypto/openssl/x509_time.h @@ -7,7 +7,7 @@ #include -namespace crypto::OpenSSL +namespace ccf::crypto::OpenSSL { /** Set of utilities functions for working with x509 time, as defined in RFC 5280 (https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.5.1) */ @@ -30,6 +30,6 @@ namespace crypto::OpenSSL { std::tm t; CHECK1(ASN1_TIME_to_tm(time, &t)); - return ds::to_x509_time_string(t); + return ::ds::to_x509_time_string(t); } } diff --git a/src/crypto/rsa_key_pair.cpp b/src/crypto/rsa_key_pair.cpp index 0ca4cb36e80d..4df8644bb71b 100644 --- a/src/crypto/rsa_key_pair.cpp +++ b/src/crypto/rsa_key_pair.cpp @@ -5,7 +5,7 @@ #include "crypto/openssl/rsa_key_pair.h" -namespace crypto +namespace ccf::crypto { using RSAPublicKeyImpl = RSAPublicKey_OpenSSL; using RSAKeyPairImpl = RSAKeyPair_OpenSSL; diff --git a/src/crypto/sha256_hash.cpp b/src/crypto/sha256_hash.cpp index 04f3489d1612..9923aadd05bc 100644 --- a/src/crypto/sha256_hash.cpp +++ b/src/crypto/sha256_hash.cpp @@ -5,7 +5,7 @@ #include "ccf/ds/hex.h" -namespace crypto +namespace ccf::crypto { extern void default_sha256(const std::span& data, uint8_t* h); @@ -48,9 +48,9 @@ namespace crypto default_sha256(data, h.data()); } - std::ostream& operator<<(std::ostream& os, const crypto::Sha256Hash& h) + std::ostream& operator<<(std::ostream& os, const ccf::crypto::Sha256Hash& h) { - for (unsigned i = 0; i < crypto::Sha256Hash::SIZE; i++) + for (unsigned i = 0; i < ccf::crypto::Sha256Hash::SIZE; i++) { os << std::hex << static_cast(h.h[i]); } @@ -123,7 +123,7 @@ namespace crypto bool operator==(const Sha256Hash& lhs, const Sha256Hash& rhs) { - for (unsigned i = 0; i < crypto::Sha256Hash::SIZE; i++) + for (unsigned i = 0; i < ccf::crypto::Sha256Hash::SIZE; i++) { if (lhs.h[i] != rhs.h[i]) { diff --git a/src/crypto/sharing.cpp b/src/crypto/sharing.cpp index 32bd3ecd95b3..ff1cce731f57 100644 --- a/src/crypto/sharing.cpp +++ b/src/crypto/sharing.cpp @@ -6,7 +6,7 @@ #include -namespace crypto +namespace ccf::crypto { namespace sharing { @@ -94,7 +94,7 @@ namespace crypto // We assume the lower 31 bits are uniformly distributed, // and retry if they are all set to get uniformity in F[prime]. - static element sample(const crypto::EntropyPtr& entropy) + static element sample(const ccf::crypto::EntropyPtr& entropy) { uint64_t res = prime; while (res == prime) @@ -107,7 +107,7 @@ namespace crypto /* POLYNOMIAL SHARING AND INTERPOLATION */ static void sample_polynomial( - element p[], size_t degree, const crypto::EntropyPtr& entropy) + element p[], size_t degree, const ccf::crypto::EntropyPtr& entropy) { for (size_t i = 0; i <= degree; i++) { @@ -148,7 +148,7 @@ namespace crypto shares[s].x = s + 1; } - auto entropy = crypto::get_entropy(); + auto entropy = ccf::crypto::get_entropy(); for (size_t limb = 0; limb < LIMBS; limb++) { diff --git a/src/crypto/sharing.h b/src/crypto/sharing.h index 4a3c225d2991..75af6754bda7 100644 --- a/src/crypto/sharing.h +++ b/src/crypto/sharing.h @@ -14,7 +14,7 @@ #include -namespace crypto +namespace ccf::crypto { namespace sharing { @@ -51,7 +51,8 @@ namespace crypto reinterpret_cast(y), sizeof(y)); const std::span label( reinterpret_cast(y), sizeof(y)); - auto k = crypto::hkdf(crypto::MDType::SHA256, key_size, ikm, {}, label); + auto k = ccf::crypto::hkdf( + ccf::crypto::MDType::SHA256, key_size, ikm, {}, label); return k; } diff --git a/src/crypto/symmetric_key.cpp b/src/crypto/symmetric_key.cpp index f2e36e8e447a..70b517c69cbc 100644 --- a/src/crypto/symmetric_key.cpp +++ b/src/crypto/symmetric_key.cpp @@ -10,7 +10,7 @@ #define FMT_HEADER_ONLY #include -namespace crypto +namespace ccf::crypto { /// GcmHeader implementation GcmHeader::GcmHeader(size_t iv_size) diff --git a/src/crypto/test/base64.cpp b/src/crypto/test/base64.cpp index b9058b419952..77638ce93afc 100644 --- a/src/crypto/test/base64.cpp +++ b/src/crypto/test/base64.cpp @@ -8,7 +8,7 @@ #include using namespace std; -using namespace crypto; +using namespace ccf::crypto; static constexpr auto MAX_LEN = 2000; diff --git a/src/crypto/test/bench.cpp b/src/crypto/test/bench.cpp index 089069f2fba0..7297fbf62d3b 100644 --- a/src/crypto/test/bench.cpp +++ b/src/crypto/test/bench.cpp @@ -18,7 +18,7 @@ #include using namespace std; -using namespace crypto; +using namespace ccf::crypto; static const string lorem_ipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing " @@ -97,13 +97,14 @@ template static void benchmark_hmac(picobench::state& s) { const auto contents = make_contents(); - const auto key = crypto::get_entropy()->random(crypto::GCM_DEFAULT_KEY_SIZE); + const auto key = + ccf::crypto::get_entropy()->random(ccf::crypto::GCM_DEFAULT_KEY_SIZE); s.start_timer(); for (auto _ : s) { (void)_; - HashBytes hash = crypto::hmac(M, key, contents); + HashBytes hash = ccf::crypto::hmac(M, key, contents); do_not_optimize(hash); clobber_memory(); } @@ -381,7 +382,7 @@ namespace Hashes template static void sha256_bench(picobench::state& s) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); std::vector v(size); for (size_t i = 0; i < size; ++i) @@ -389,15 +390,15 @@ static void sha256_bench(picobench::state& s) v[i] = rand(); } - crypto::Sha256Hash h; + ccf::crypto::Sha256Hash h; s.start_timer(); for (size_t i = 0; i < 10; ++i) { - crypto::openssl_sha256(v, h.h.data()); + ccf::crypto::openssl_sha256(v, h.h.data()); } s.stop_timer(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } PICOBENCH_SUITE("digest sha256"); @@ -436,8 +437,8 @@ namespace Base64_bench { // We don't check the outputs as this is done elsewhere std::string encoded = - crypto::Base64_openssl::b64_from_raw(v.data(), v.size()); - crypto::Base64_openssl::raw_from_b64(encoded); + ccf::crypto::Base64_openssl::b64_from_raw(v.data(), v.size()); + ccf::crypto::Base64_openssl::raw_from_b64(encoded); } s.stop_timer(); } @@ -473,7 +474,7 @@ namespace HMAC_bench PICOBENCH(openssl_hmac_sha256_64).PICO_HASH_SUFFIX(); } -std::vector shares; +std::vector shares; PICOBENCH_SUITE("share"); namespace SHARE_bench @@ -487,8 +488,8 @@ namespace SHARE_bench for (auto _ : s) { (void)_; - crypto::sharing::Share secret; - crypto::sharing::sample_secret_and_shares(secret, shares, threshold); + ccf::crypto::sharing::Share secret; + ccf::crypto::sharing::sample_secret_and_shares(secret, shares, threshold); do_not_optimize(secret); clobber_memory(); } @@ -520,9 +521,9 @@ namespace SHARE_bench for (auto _ : s) { (void)_; - crypto::sharing::Share secret; - crypto::sharing::sample_secret_and_shares(secret, shares, threshold); - crypto::sharing::recover_unauthenticated_secret( + ccf::crypto::sharing::Share secret; + ccf::crypto::sharing::sample_secret_and_shares(secret, shares, threshold); + ccf::crypto::sharing::recover_unauthenticated_secret( secret, shares, threshold); do_not_optimize(secret); clobber_memory(); diff --git a/src/crypto/test/crypto.cpp b/src/crypto/test/crypto.cpp index cf01aba31df8..4746f69430bd 100644 --- a/src/crypto/test/crypto.cpp +++ b/src/crypto/test/crypto.cpp @@ -29,7 +29,7 @@ #include using namespace std; -using namespace crypto; +using namespace ccf::crypto; static const string contents_ = "Lorem ipsum dolor sit amet, consectetur adipiscing " @@ -57,15 +57,15 @@ static constexpr CurveID supported_curves[] = { static constexpr char const* labels[] = {"secp384r1", "secp256r1", "secp256k1"}; -crypto::Pem generate_self_signed_cert( +ccf::crypto::Pem generate_self_signed_cert( const KeyPairPtr& kp, const std::string& name) { constexpr size_t certificate_validity_period_days = 365; using namespace std::literals; auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); - return crypto::create_self_signed_cert( + return ccf::crypto::create_self_signed_cert( kp, name, {}, valid_from, certificate_validity_period_days); } @@ -219,12 +219,12 @@ TEST_CASE("Sign, verify. Fail to verify with bad contents") } } -crypto::HashBytes bad_manual_hash(const std::vector& data) +ccf::crypto::HashBytes bad_manual_hash(const std::vector& data) { // secp256r1 requires 32-byte hashes, other curves don't care. So use 32 for // general hasher constexpr auto n = 32; - crypto::HashBytes hash(n); + ccf::crypto::HashBytes hash(n); for (size_t i = 0; i < data.size(); ++i) { @@ -241,7 +241,7 @@ TEST_CASE("Manually hash, sign, verify, with PublicKey") INFO("With curve: " << labels[static_cast(curve) - 1]); auto kp = make_key_pair(curve); vector contents(contents_.begin(), contents_.end()); - crypto::HashBytes hash = bad_manual_hash(contents); + ccf::crypto::HashBytes hash = bad_manual_hash(contents); const vector signature = kp->sign_hash(hash.data(), hash.size()); const auto public_key = kp->public_key_pem(); @@ -259,7 +259,7 @@ TEST_CASE("Manually hash, sign, verify, with certificate") INFO("With curve: " << labels[static_cast(curve) - 1]); auto kp = make_key_pair(curve); vector contents(contents_.begin(), contents_.end()); - crypto::HashBytes hash = bad_manual_hash(contents); + ccf::crypto::HashBytes hash = bad_manual_hash(contents); const vector signature = kp->sign_hash(hash.data(), hash.size()); auto cert = generate_self_signed_cert(kp, "CN=name"); @@ -495,71 +495,71 @@ void run_csr(bool corrupt_csr = false) TEST_CASE("2-digit years") { auto time_str = "220405175422Z"; - auto tp = ds::time_point_from_string(time_str); - auto conv = ds::to_x509_time_string(tp); + auto tp = ::ds::time_point_from_string(time_str); + auto conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == std::string("20") + time_str); } TEST_CASE("Non-ASN.1 timepoint formats") { auto time_str = "2022-04-05 18:53:27"; - auto tp = ds::time_point_from_string(time_str); - auto conv = ds::to_x509_time_string(tp); + auto tp = ::ds::time_point_from_string(time_str); + auto conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405185327Z"); time_str = "2022-04-05 18:53:27.190380"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405185327Z"); time_str = "2022-04-05 18:53:27 +03:00"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405155327Z"); time_str = "2022-04-05 18:53:27 +0300"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405155327Z"); time_str = "2022-04-05 18:53:27.190380+03:00"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405155327Z"); time_str = "2022-04-05 18:53:27 -03:00"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220405215327Z"); time_str = "2022-04-07T10:37:49.567612"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220407103749Z"); time_str = "2022-04-07T10:37:49.567612+03:00"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220407073749Z"); time_str = "2022-04-07T10:37:49.567612Z"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220407103749Z"); time_str = "220425165619+0000"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220425165619Z"); time_str = "220425165619+0200"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220425145619Z"); time_str = "20220425165619-0300"; - tp = ds::time_point_from_string(time_str); - conv = ds::to_x509_time_string(tp); + tp = ::ds::time_point_from_string(time_str); + conv = ::ds::to_x509_time_string(tp); REQUIRE(conv == "20220425195619Z"); } @@ -581,7 +581,7 @@ static const vector& get_raw_key() TEST_CASE("ExtendedIv0") { - auto k = crypto::make_key_aes_gcm(get_raw_key()); + auto k = ccf::crypto::make_key_aes_gcm(get_raw_key()); // setup plain text std::vector plain(100); @@ -601,7 +601,7 @@ TEST_CASE("ExtendedIv0") std::vector cipher; k->encrypt(h.get_iv(), plain, {}, cipher, h.tag); - auto k2 = crypto::make_key_aes_gcm(get_raw_key()); + auto k2 = ccf::crypto::make_key_aes_gcm(get_raw_key()); std::vector decrypted_plain; REQUIRE(k2->decrypt(h.get_iv(), h.tag, cipher, {}, decrypted_plain)); REQUIRE(plain == decrypted_plain); @@ -630,14 +630,14 @@ TEST_CASE("CKM_RSA_PKCS_OAEP") auto rsa_kp = make_rsa_key_pair(); auto rsa_pk = make_rsa_public_key(rsa_kp->public_key_pem()); - auto wrapped = crypto::ckm_rsa_pkcs_oaep_wrap(rsa_pk, key); - auto wrapped_ = crypto::ckm_rsa_pkcs_oaep_wrap(rsa_pk, key); + auto wrapped = ccf::crypto::ckm_rsa_pkcs_oaep_wrap(rsa_pk, key); + auto wrapped_ = ccf::crypto::ckm_rsa_pkcs_oaep_wrap(rsa_pk, key); // CKM_RSA_PKCS_OAEP wrap is non deterministic REQUIRE(wrapped != wrapped_); - auto unwrapped = crypto::ckm_rsa_pkcs_oaep_unwrap(rsa_kp, wrapped); - auto unwrapped_ = crypto::ckm_rsa_pkcs_oaep_unwrap(rsa_kp, wrapped_); + auto unwrapped = ccf::crypto::ckm_rsa_pkcs_oaep_unwrap(rsa_kp, wrapped); + auto unwrapped_ = ccf::crypto::ckm_rsa_pkcs_oaep_unwrap(rsa_kp, wrapped_); REQUIRE(unwrapped == unwrapped_); } @@ -705,9 +705,9 @@ TEST_CASE("x509 time") const auto& from = data.input.from; const auto& to = data.input.to; REQUIRE( - crypto::OpenSSL::validate_chronological_times( - crypto::OpenSSL::Unique_X509_TIME(from), - crypto::OpenSSL::Unique_X509_TIME(to), + ccf::crypto::OpenSSL::validate_chronological_times( + ccf::crypto::OpenSSL::Unique_X509_TIME(from), + ccf::crypto::OpenSSL::Unique_X509_TIME(to), data.input.maximum_validity_period_days) == data.expected_verification_result); } @@ -723,14 +723,14 @@ TEST_CASE("x509 time") { auto adjusted_time = t + std::chrono::days(days_offset); - auto from = crypto::OpenSSL::Unique_X509_TIME(t); - auto to = crypto::OpenSSL::Unique_X509_TIME(adjusted_time); + auto from = ccf::crypto::OpenSSL::Unique_X509_TIME(t); + auto to = ccf::crypto::OpenSSL::Unique_X509_TIME(adjusted_time); // Convert to string and back to time_points - auto from_conv = - ds::time_point_from_string(crypto::OpenSSL::to_x509_time_string(from)); - auto to_conv = - ds::time_point_from_string(crypto::OpenSSL::to_x509_time_string(to)); + auto from_conv = ::ds::time_point_from_string( + ccf::crypto::OpenSSL::to_x509_time_string(from)); + auto to_conv = ::ds::time_point_from_string( + ccf::crypto::OpenSSL::to_x509_time_string(to)); // Diff is still the same amount of days auto days_diff = @@ -747,9 +747,9 @@ TEST_CASE("x509 time") for (auto const& days_offset : days_offsets) { auto adjusted_time = time + std::chrono::days(days_offset); - auto adjusted_str = ds::to_x509_time_string(adjusted_time); - auto asn1_time = crypto::OpenSSL::Unique_X509_TIME(adjusted_str); - auto converted_str = crypto::OpenSSL::to_x509_time_string(asn1_time); + auto adjusted_str = ::ds::to_x509_time_string(adjusted_time); + auto asn1_time = ccf::crypto::OpenSSL::Unique_X509_TIME(adjusted_str); + auto converted_str = ccf::crypto::OpenSSL::to_x509_time_string(asn1_time); REQUIRE(converted_str == adjusted_str); } } @@ -764,15 +764,15 @@ TEST_CASE("hmac") INFO("Same inputs, same hmac"); { - auto r0 = crypto::hmac(MDType::SHA256, key, zeros); - auto r1 = crypto::hmac(MDType::SHA256, key, zeros); + auto r0 = ccf::crypto::hmac(MDType::SHA256, key, zeros); + auto r1 = ccf::crypto::hmac(MDType::SHA256, key, zeros); REQUIRE(r0 == r1); } INFO("Different inputs, different hmacs"); { - auto r0 = crypto::hmac(MDType::SHA256, key, zeros); - auto r1 = crypto::hmac(MDType::SHA256, key, mostly_zeros); + auto r0 = ccf::crypto::hmac(MDType::SHA256, key, zeros); + auto r1 = ccf::crypto::hmac(MDType::SHA256, key, mostly_zeros); REQUIRE(r0 != r1); } } @@ -889,8 +889,8 @@ TEST_CASE("PEM to JWK and back") TEST_CASE("Incremental hash") { - crypto::openssl_sha256_init(); - auto simple_hash = crypto::Sha256Hash(contents); + ccf::crypto::openssl_sha256_init(); + auto simple_hash = ccf::crypto::Sha256Hash(contents); INFO("Incremental hash"); { @@ -930,5 +930,5 @@ TEST_CASE("Incremental hash") REQUIRE_THROWS_AS(ihash->finalise(), std::logic_error); } } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } \ No newline at end of file diff --git a/src/crypto/test/kp_cert.cpp b/src/crypto/test/kp_cert.cpp index 9b02434cdb4c..f7140359a1fb 100644 --- a/src/crypto/test/kp_cert.cpp +++ b/src/crypto/test/kp_cert.cpp @@ -7,8 +7,8 @@ #include constexpr size_t certificate_validity_period_days = 365; -auto valid_from = ds::to_x509_time_string(std::chrono::system_clock::now()); -auto valid_to = crypto::compute_cert_valid_to_string( +auto valid_from = ::ds::to_x509_time_string(std::chrono::system_clock::now()); +auto valid_to = ccf::crypto::compute_cert_valid_to_string( valid_from, certificate_validity_period_days); int main(int argc, char** argv) @@ -20,7 +20,7 @@ int main(int argc, char** argv) "--sn", name, "Subject Name in node certificate, eg. CN=CCF Node") ->capture_default_str(); - std::vector sans = {}; + std::vector sans = {}; cli::add_subject_alternative_name_option( app, sans, @@ -29,7 +29,7 @@ int main(int argc, char** argv) "iPAddress:xxx.xxx.xxx.xxx, or dNSName:sub.domain.tld"); CLI11_PARSE(app, argc, argv); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto icrt = kp->self_sign("CN=issuer", valid_from, valid_to); auto csr = kp->create_csr(name, sans); auto cert = kp->sign_csr(icrt, csr, valid_from, valid_to); diff --git a/src/crypto/test/secret_sharing.cpp b/src/crypto/test/secret_sharing.cpp index 775798f83d0b..a0f72d754d17 100644 --- a/src/crypto/test/secret_sharing.cpp +++ b/src/crypto/test/secret_sharing.cpp @@ -11,7 +11,7 @@ #include #include -using namespace crypto::sharing; +using namespace ccf::crypto::sharing; void check_share_is_not_trivially_wrong(const Share& share) { diff --git a/src/crypto/verifier.cpp b/src/crypto/verifier.cpp index 6e81ebfe543b..2ae3976fd901 100644 --- a/src/crypto/verifier.cpp +++ b/src/crypto/verifier.cpp @@ -5,7 +5,7 @@ #include "crypto/openssl/verifier.h" -namespace crypto +namespace ccf::crypto { using VerifierPtr = std::shared_ptr; using VerifierUniquePtr = std::unique_ptr; @@ -30,12 +30,12 @@ namespace crypto return make_verifier(pem.raw()); } - crypto::Pem cert_der_to_pem(const std::vector& der) + ccf::crypto::Pem cert_der_to_pem(const std::vector& der) { return make_verifier(der)->cert_pem(); } - std::vector cert_pem_to_der(const crypto::Pem& pem) + std::vector cert_pem_to_der(const ccf::crypto::Pem& pem) { return make_verifier(pem)->cert_der(); } @@ -45,7 +45,7 @@ namespace crypto return make_unique_verifier(der)->public_key_der(); } - crypto::Pem public_key_pem_from_cert(const std::vector& der) + ccf::crypto::Pem public_key_pem_from_cert(const std::vector& der) { return make_unique_verifier(der)->public_key_pem(); } diff --git a/src/ds/cli_helper.h b/src/ds/cli_helper.h index 8d9b2bd50fcc..6f5d373b38eb 100644 --- a/src/ds/cli_helper.h +++ b/src/ds/cli_helper.h @@ -96,7 +96,7 @@ namespace cli static CLI::Option* add_subject_alternative_name_option( CLI::App& app, - std::vector& parsed, + std::vector& parsed, const std::string& option_name, const std::string& option_desc) { diff --git a/src/ds/ring_buffer_types.h b/src/ds/ring_buffer_types.h index a9d0010587ee..c0b493c8ce38 100644 --- a/src/ds/ring_buffer_types.h +++ b/src/ds/ring_buffer_types.h @@ -137,7 +137,7 @@ namespace ringbuffer return false; auto next = initial_marker; - nonstd::tuple_for_each(sections, [&](const auto& s) { + ccf::nonstd::tuple_for_each(sections, [&](const auto& s) { next = write_bytes(next, s->data(), s->size()); }); @@ -166,7 +166,7 @@ namespace ringbuffer struct MessageSerializers { static_assert( - nonstd::value_dependent_false::value, + ccf::nonstd::value_dependent_false::value, "No payload specialization for this Message"); }; diff --git a/src/ds/test/hex.cpp b/src/ds/test/hex.cpp index 3ada1b80e484..0d6fdf9b6990 100644 --- a/src/ds/test/hex.cpp +++ b/src/ds/test/hex.cpp @@ -35,7 +35,7 @@ TEST_CASE("Hex string to and from conversion") std::string uppercase_hex_str("0123456789ABCDEF"); std::string lowercase_hex_str = uppercase_hex_str; - nonstd::to_lower(lowercase_hex_str); + ccf::nonstd::to_lower(lowercase_hex_str); auto data = ccf::ds::from_hex(uppercase_hex_str); REQUIRE(data.size() == uppercase_hex_str.size() / 2); diff --git a/src/ds/test/logger.cpp b/src/ds/test/logger.cpp index bf6d79a44dd4..35eec285a981 100644 --- a/src/ds/test/logger.cpp +++ b/src/ds/test/logger.cpp @@ -20,14 +20,14 @@ class TestLogger : public Base } }; -using TestTextLogger = TestLogger; -using TestJsonLogger = TestLogger; +using TestTextLogger = TestLogger; +using TestJsonLogger = TestLogger; TEST_CASE("Framework logging macros") { std::vector logs; - logger::config::loggers().emplace_back( + ccf::logger::config::loggers().emplace_back( std::make_unique(logs)); { @@ -69,14 +69,14 @@ TEST_CASE("Framework logging macros") logs.clear(); } - logger::config::loggers().clear(); + ccf::logger::config::loggers().clear(); } TEST_CASE("Application logging macros") { std::vector logs; - logger::config::loggers().emplace_back( + ccf::logger::config::loggers().emplace_back( std::make_unique(logs)); { @@ -121,7 +121,7 @@ TEST_CASE("Application logging macros") logs.clear(); } - logger::config::loggers().clear(); + ccf::logger::config::loggers().clear(); } constexpr auto custom_tag = "my tag"; @@ -135,7 +135,7 @@ TEST_CASE("Custom logging macros") { std::vector logs; - logger::config::loggers().emplace_back( + ccf::logger::config::loggers().emplace_back( std::make_unique(logs)); { @@ -177,5 +177,5 @@ TEST_CASE("Custom logging macros") logs.clear(); } - logger::config::loggers().clear(); + ccf::logger::config::loggers().clear(); } \ No newline at end of file diff --git a/src/ds/test/logger_bench.cpp b/src/ds/test/logger_bench.cpp index 6e54b15bdde5..8eff56c00306 100644 --- a/src/ds/test/logger_bench.cpp +++ b/src/ds/test/logger_bench.cpp @@ -18,18 +18,18 @@ enum LoggerKind template static void prepare_loggers() { - logger::config::loggers().clear(); + ccf::logger::config::loggers().clear(); if constexpr ((LK & LoggerKind::Console) != 0) { - logger::config::loggers().emplace_back( - std::make_unique()); + ccf::logger::config::loggers().emplace_back( + std::make_unique()); } if constexpr ((LK & LoggerKind::JSON) != 0) { - logger::config::loggers().emplace_back( - std::make_unique()); + ccf::logger::config::loggers().emplace_back( + std::make_unique()); } if constexpr (Absorb) @@ -41,7 +41,7 @@ static void prepare_loggers() static void reset_loggers() { - logger::config::loggers().clear(); + ccf::logger::config::loggers().clear(); std::cout.clear(); } @@ -51,7 +51,7 @@ static void log_accepted(picobench::state& s) { prepare_loggers(); - logger::config::level() = LoggerLevel::DEBUG; + ccf::logger::config::level() = LoggerLevel::DEBUG; { picobench::scope scope(s); @@ -69,7 +69,7 @@ static void log_accepted_fmt(picobench::state& s) { prepare_loggers(); - logger::config::level() = LoggerLevel::DEBUG; + ccf::logger::config::level() = LoggerLevel::DEBUG; { picobench::scope scope(s); @@ -87,7 +87,7 @@ static void log_rejected(picobench::state& s) { prepare_loggers(); - logger::config::level() = LoggerLevel::FAIL; + ccf::logger::config::level() = LoggerLevel::FAIL; { picobench::scope scope(s); @@ -105,7 +105,7 @@ static void log_rejected_fmt(picobench::state& s) { prepare_loggers(); - logger::config::level() = LoggerLevel::FAIL; + ccf::logger::config::level() = LoggerLevel::FAIL; { picobench::scope scope(s); diff --git a/src/ds/test/logger_json_test.cpp b/src/ds/test/logger_json_test.cpp index 3d6e2b8daef4..4f75114fa35f 100644 --- a/src/ds/test/logger_json_test.cpp +++ b/src/ds/test/logger_json_test.cpp @@ -13,8 +13,8 @@ TEST_CASE("Test custom log format") { std::string test_log_file = "./test_json_logger.txt"; remove(test_log_file.c_str()); - logger::config::add_json_console_logger(); - logger::config::level() = LoggerLevel::DEBUG; + ccf::logger::config::add_json_console_logger(); + ccf::logger::config::level() = LoggerLevel::DEBUG; std::string log_msg_dbg = "log_msg_dbg"; std::string log_msg_trace = "log_msg_trace"; diff --git a/src/ds/test/map_test.cpp b/src/ds/test/map_test.cpp index c6cb70bede4d..ad14e3553ec8 100644 --- a/src/ds/test/map_test.cpp +++ b/src/ds/test/map_test.cpp @@ -22,8 +22,8 @@ struct CollisionHash } }; -using K = kv::serialisers::SerialisedEntry; -using V = kv::serialisers::SerialisedEntry; +using K = ccf::kv::serialisers::SerialisedEntry; +using V = ccf::kv::serialisers::SerialisedEntry; constexpr static size_t max_key_value_size = 128; namespace map diff --git a/src/ds/test/nonstd.cpp b/src/ds/test/nonstd.cpp index aacc2689f574..ccd352b73719 100644 --- a/src/ds/test/nonstd.cpp +++ b/src/ds/test/nonstd.cpp @@ -19,7 +19,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("Split by spaces"); - auto v = nonstd::split(s, " "); + auto v = ccf::nonstd::split(s, " "); REQUIRE(v.size() == 7); REQUIRE(v[0] == "Good"); REQUIRE(v[1] == "afternoon,"); @@ -32,7 +32,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("Split by commas"); - auto v = nonstd::split(s, ","); + auto v = ccf::nonstd::split(s, ","); REQUIRE(v.size() == 3); REQUIRE(v[0] == "Good afternoon"); REQUIRE(v[1] == " good evening"); @@ -41,7 +41,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("Split by comma-with-space"); - auto v = nonstd::split(s, ", "); + auto v = ccf::nonstd::split(s, ", "); REQUIRE(v.size() == 3); REQUIRE(v[0] == "Good afternoon"); REQUIRE(v[1] == "good evening"); @@ -50,7 +50,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("split(max_splits=3)"); { - auto v = nonstd::split(s, " ", 3); + auto v = ccf::nonstd::split(s, " ", 3); // NB: max_splits=3 => 4 returned segments REQUIRE(v.size() == 4); REQUIRE(v[0] == "Good"); @@ -60,7 +60,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) } { - auto v = nonstd::split(s, "afternoon", 3); + auto v = ccf::nonstd::split(s, "afternoon", 3); // NB: max_splits=3, but only 1 split possible => 2 returned segments REQUIRE(v.size() == 2); REQUIRE(v[0] == "Good "); @@ -70,7 +70,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("split_1"); - auto t = nonstd::split_1(s, ", "); + auto t = ccf::nonstd::split_1(s, ", "); REQUIRE(std::get<0>(t) == "Good afternoon"); REQUIRE(std::get<1>(t) == "good evening, and good night!"); } @@ -78,7 +78,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("Split by commas"); - auto v = nonstd::split(s, ","); + auto v = ccf::nonstd::split(s, ","); REQUIRE(v.size() == 3); REQUIRE(v[0] == "Good afternoon"); REQUIRE(v[1] == " good evening"); @@ -87,7 +87,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { INFO("Split by 'oo'"); - auto v = nonstd::split(s, "oo"); + auto v = ccf::nonstd::split(s, "oo"); REQUIRE(v.size() == 5); REQUIRE(v[0] == "G"); REQUIRE(v[1] == "d aftern"); @@ -102,7 +102,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { const auto s = " bob "; - auto v = nonstd::split(s, " "); + auto v = ccf::nonstd::split(s, " "); REQUIRE(v.size() == 4); REQUIRE(v[0].empty()); REQUIRE(v[1].empty()); @@ -113,13 +113,13 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { const auto s = "bobbob"; { - auto v = nonstd::split(s, " "); + auto v = ccf::nonstd::split(s, " "); REQUIRE(v.size() == 1); REQUIRE(v[0] == "bobbob"); } { - auto v = nonstd::split(s, "bob"); + auto v = ccf::nonstd::split(s, "bob"); REQUIRE(v.size() == 3); REQUIRE(v[0].empty()); REQUIRE(v[1].empty()); @@ -127,7 +127,7 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) } { - auto t = nonstd::split_1(s, "bob"); + auto t = ccf::nonstd::split_1(s, "bob"); REQUIRE(std::get<0>(t).empty()); REQUIRE(std::get<1>(t) == "bob"); } @@ -136,19 +136,19 @@ TEST_CASE("split" * doctest::test_suite("nonstd")) { const auto s = ""; { - auto v = nonstd::split(s, " "); + auto v = ccf::nonstd::split(s, " "); REQUIRE(v.size() == 1); REQUIRE(v[0].empty()); } { - auto v = nonstd::split(s, "bob"); + auto v = ccf::nonstd::split(s, "bob"); REQUIRE(v.size() == 1); REQUIRE(v[0].empty()); } { - auto t = nonstd::split_1(s, " "); + auto t = ccf::nonstd::split_1(s, " "); REQUIRE(std::get<0>(t).empty()); REQUIRE(std::get<1>(t).empty()); } @@ -165,7 +165,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { INFO("rsplit by spaces"); - auto v = nonstd::rsplit(s, " "); + auto v = ccf::nonstd::rsplit(s, " "); REQUIRE(v.size() == 7); REQUIRE(v[0] == "night!"); REQUIRE(v[1] == "good"); @@ -179,7 +179,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { INFO("rsplit(max_splits=3)"); { - auto v = nonstd::rsplit(s, " ", 3); + auto v = ccf::nonstd::rsplit(s, " ", 3); // NB: max_splits=3 => 4 returned segments REQUIRE(v.size() == 4); REQUIRE(v[0] == "night!"); @@ -189,7 +189,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) } { - auto v = nonstd::rsplit(s, "afternoon", 3); + auto v = ccf::nonstd::rsplit(s, "afternoon", 3); // NB: max_splits=3, but only 1 split possible => 2 returned segments REQUIRE(v.size() == 2); REQUIRE(v[0] == ", good evening, and good night!"); @@ -199,7 +199,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { INFO("rsplit_1"); - auto t = nonstd::rsplit_1(s, ", "); + auto t = ccf::nonstd::rsplit_1(s, ", "); REQUIRE(std::get<0>(t) == "Good afternoon, good evening"); REQUIRE(std::get<1>(t) == "and good night!"); } @@ -210,7 +210,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { const auto s = " bob "; - auto v = nonstd::rsplit(s, " "); + auto v = ccf::nonstd::rsplit(s, " "); REQUIRE(v.size() == 4); REQUIRE(v[0].empty()); REQUIRE(v[1] == "bob"); @@ -221,13 +221,13 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { const auto s = "bobbob"; { - auto v = nonstd::rsplit(s, " "); + auto v = ccf::nonstd::rsplit(s, " "); REQUIRE(v.size() == 1); REQUIRE(v[0] == "bobbob"); } { - auto v = nonstd::rsplit(s, "bob"); + auto v = ccf::nonstd::rsplit(s, "bob"); REQUIRE(v.size() == 3); REQUIRE(v[0].empty()); REQUIRE(v[1].empty()); @@ -235,7 +235,7 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) } { - auto t = nonstd::rsplit_1(s, "bob"); + auto t = ccf::nonstd::rsplit_1(s, "bob"); REQUIRE(std::get<0>(t) == "bob"); REQUIRE(std::get<1>(t).empty()); } @@ -244,19 +244,19 @@ TEST_CASE("rsplit" * doctest::test_suite("nonstd")) { const auto s = ""; { - auto v = nonstd::rsplit(s, " "); + auto v = ccf::nonstd::rsplit(s, " "); REQUIRE(v.size() == 1); REQUIRE(v[0].empty()); } { - auto v = nonstd::rsplit(s, "bob"); + auto v = ccf::nonstd::rsplit(s, "bob"); REQUIRE(v.size() == 1); REQUIRE(v[0].empty()); } { - auto t = nonstd::rsplit_1(s, " "); + auto t = ccf::nonstd::rsplit_1(s, " "); REQUIRE(std::get<0>(t).empty()); REQUIRE(std::get<1>(t).empty()); } @@ -272,14 +272,15 @@ TEST_CASE("envvars" * doctest::test_suite("nonstd")) std::string test_value("test_value"); ::setenv("TEST_ENV_VAR", test_value.c_str(), 1); - REQUIRE("" == nonstd::expand_envvar("")); - REQUIRE("not an env var" == nonstd::expand_envvar("not an env var")); - REQUIRE("$ENV_VAR_NOT_SET" == nonstd::expand_envvar("$ENV_VAR_NOT_SET")); - REQUIRE(test_value == nonstd::expand_envvar("$TEST_ENV_VAR")); + REQUIRE("" == ccf::nonstd::expand_envvar("")); + REQUIRE("not an env var" == ccf::nonstd::expand_envvar("not an env var")); + REQUIRE( + "$ENV_VAR_NOT_SET" == ccf::nonstd::expand_envvar("$ENV_VAR_NOT_SET")); + REQUIRE(test_value == ccf::nonstd::expand_envvar("$TEST_ENV_VAR")); // ${} syntax is not supported REQUIRE( - "${ENV_VAR_NOT_SET}" == nonstd::expand_envvar("${ENV_VAR_NOT_SET}")); + "${ENV_VAR_NOT_SET}" == ccf::nonstd::expand_envvar("${ENV_VAR_NOT_SET}")); } { INFO("Expand path"); @@ -289,33 +290,33 @@ TEST_CASE("envvars" * doctest::test_suite("nonstd")) std::string test_value2("test_value2"); ::setenv("TEST_ENV_VAR2", test_value2.c_str(), 1); - REQUIRE("" == nonstd::expand_envvars_in_path("")); - REQUIRE("foo" == nonstd::expand_envvars_in_path("foo")); - REQUIRE("foo/" == nonstd::expand_envvars_in_path("foo/")); - REQUIRE("foo/bar" == nonstd::expand_envvars_in_path("foo/bar")); - REQUIRE("/" == nonstd::expand_envvars_in_path("/")); - REQUIRE("/foo" == nonstd::expand_envvars_in_path("/foo")); - REQUIRE("/foo/" == nonstd::expand_envvars_in_path("/foo/")); - REQUIRE("/foo/bar" == nonstd::expand_envvars_in_path("/foo/bar")); + REQUIRE("" == ccf::nonstd::expand_envvars_in_path("")); + REQUIRE("foo" == ccf::nonstd::expand_envvars_in_path("foo")); + REQUIRE("foo/" == ccf::nonstd::expand_envvars_in_path("foo/")); + REQUIRE("foo/bar" == ccf::nonstd::expand_envvars_in_path("foo/bar")); + REQUIRE("/" == ccf::nonstd::expand_envvars_in_path("/")); + REQUIRE("/foo" == ccf::nonstd::expand_envvars_in_path("/foo")); + REQUIRE("/foo/" == ccf::nonstd::expand_envvars_in_path("/foo/")); + REQUIRE("/foo/bar" == ccf::nonstd::expand_envvars_in_path("/foo/bar")); REQUIRE( fmt::format("{}", test_value1) == - nonstd::expand_envvars_in_path("$TEST_ENV_VAR1")); + ccf::nonstd::expand_envvars_in_path("$TEST_ENV_VAR1")); REQUIRE( fmt::format("{}/", test_value1) == - nonstd::expand_envvars_in_path("$TEST_ENV_VAR1/")); + ccf::nonstd::expand_envvars_in_path("$TEST_ENV_VAR1/")); REQUIRE( fmt::format("{}/{}", test_value1, test_value2) == - nonstd::expand_envvars_in_path("$TEST_ENV_VAR1/$TEST_ENV_VAR2")); + ccf::nonstd::expand_envvars_in_path("$TEST_ENV_VAR1/$TEST_ENV_VAR2")); REQUIRE( fmt::format("/{}", test_value1) == - nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1")); + ccf::nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1")); REQUIRE( fmt::format("/{}/", test_value1) == - nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1/")); + ccf::nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1/")); REQUIRE( fmt::format("/{}/{}", test_value1, test_value2) == - nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1/$TEST_ENV_VAR2")); + ccf::nonstd::expand_envvars_in_path("/$TEST_ENV_VAR1/$TEST_ENV_VAR2")); } } \ No newline at end of file diff --git a/src/ds/test/openapi.cpp b/src/ds/test/openapi.cpp index 5fb6ab0452b7..b4559badbb9a 100644 --- a/src/ds/test/openapi.cpp +++ b/src/ds/test/openapi.cpp @@ -43,14 +43,14 @@ TEST_CASE("Manual construction") auto& foo_post = openapi::path_operation(foo, HTTP_POST); auto& foo_post_request = openapi::request_body(foo_post); auto& foo_post_request_json = openapi::media_type( - foo_post_request, http::headervalues::contenttype::JSON); + foo_post_request, ccf::http::headervalues::contenttype::JSON); auto& foo_post_request_json_schema = openapi::schema(foo_post_request_json); foo_post_request_json_schema = string_schema; auto& foo_post_response_ok = openapi::response( foo_post, HTTP_STATUS_OK, "Indicates that everything went ok"); auto& foo_post_response_ok_json = openapi::media_type( - foo_post_response_ok, http::headervalues::contenttype::JSON); + foo_post_response_ok, ccf::http::headervalues::contenttype::JSON); auto& foo_post_response_ok_json_schema = openapi::schema(foo_post_response_ok_json); foo_post_response_ok_json_schema = string_schema; diff --git a/src/enclave/client_session.h b/src/enclave/client_session.h index e7de86e08e61..e938e8cd10bc 100644 --- a/src/enclave/client_session.h +++ b/src/enclave/client_session.h @@ -34,7 +34,7 @@ namespace ccf to_host(writer_factory.create_writer_to_outside()) {} - virtual void send_request(http::Request&& request) = 0; + virtual void send_request(::http::Request&& request) = 0; virtual void connect( const std::string& hostname, diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index 249abb3a2829..a51d0fbf0f68 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -59,7 +59,7 @@ namespace ccf StartType start_type; - struct NodeContext : public ccfapp::AbstractNodeContext + struct NodeContext : public ccf::AbstractNodeContext { const ccf::NodeId this_node; @@ -88,7 +88,7 @@ namespace ccf size_t sig_tx_interval, size_t sig_ms_interval, const ccf::consensus::Configuration& consensus_config, - const crypto::CurveID& curve_id) : + const ccf::crypto::CurveID& curve_id) : circuit(std::move(circuit_)), basic_writer_factory(std::move(basic_writer_factory_)), writer_factory(std::move(writer_factory_)), @@ -99,7 +99,7 @@ namespace ccf { ccf::pal::initialize_enclave(); ccf::initialize_verifiers(); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); // https://github.com/microsoft/CCF/issues/5569 // Open Enclave with OpenSSL 3.x (default for SGX) is built with RDCPU @@ -178,7 +178,7 @@ namespace ccf rpc_map->register_frontend( std::make_unique( - network, ccfapp::make_user_endpoints(*context), *context)); + network, ccf::make_user_endpoints(*context), *context)); rpc_map->register_frontend( std::make_unique(network, *context)); @@ -189,7 +189,7 @@ namespace ccf rpc_map->register_frontend( std::make_unique(network, *context)); - ccf::js::register_ffi_plugins(ccfapp::get_js_plugins()); + ccf::js::register_ffi_plugins(ccf::get_js_plugins()); LOG_TRACE_FMT("Initialize node"); node->initialize( @@ -214,7 +214,7 @@ namespace ccf LOG_TRACE_FMT("Shutting down enclave"); ccf::shutdown_verifiers(); ccf::pal::shutdown_enclave(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } CreateNodeStatus create_new_node( @@ -295,7 +295,7 @@ namespace ccf bool run_main() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); LOG_DEBUG_FMT("Running main thread"); #ifndef VIRTUAL_ENCLAVE try @@ -502,7 +502,7 @@ namespace ccf LOG_INFO_FMT("Enclave stopped successfully. Stopping host..."); RINGBUFFER_WRITE_MESSAGE(AdminMessage::stopped, to_host); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return true; } @@ -514,7 +514,7 @@ namespace ccf // exceptions bubble up to here and cause the node to shutdown. RINGBUFFER_WRITE_MESSAGE( AdminMessage::fatal_error_msg, to_host, std::string(e.what())); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return false; } #endif @@ -532,7 +532,7 @@ namespace ccf bool run_worker() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); LOG_DEBUG_FMT("Running worker thread"); #ifndef VIRTUAL_ENCLAVE try @@ -544,14 +544,14 @@ namespace ccf msg->data.tid, std::move(msg)); ::threading::ThreadMessaging::instance().run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } #ifndef VIRTUAL_ENCLAVE catch (const std::exception& e) { RINGBUFFER_WRITE_MESSAGE( AdminMessage::fatal_error_msg, to_host, std::string(e.what())); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return false; } #endif diff --git a/src/enclave/main.cpp b/src/enclave/main.cpp index 67abd7493613..04944823b967 100644 --- a/src/enclave/main.cpp +++ b/src/enclave/main.cpp @@ -132,7 +132,7 @@ extern "C" auto new_logger = std::make_unique( writer_factory->create_writer_to_outside()); auto ringbuffer_logger = new_logger.get(); - logger::config::loggers().push_back(std::move(new_logger)); + ccf::logger::config::loggers().push_back(std::move(new_logger)); ccf::pal::redirect_platform_logging(); @@ -234,7 +234,7 @@ extern "C" // minimum logging level (maximum verbosity) restricted at compile-time, // while other platforms can permit any level at compile-time and then bind // the run-time choice in attestations. - const auto mv = logger::MOST_VERBOSE; + const auto mv = ccf::logger::MOST_VERBOSE; const auto requested = enclave_log_level; const auto permitted = std::max(mv, requested); if (requested != permitted) @@ -242,12 +242,12 @@ extern "C" LOG_FAIL_FMT( "Unable to set requested enclave logging level '{}'. Most verbose " "permitted level is '{}', so setting level to '{}'.", - logger::to_string(requested), - logger::to_string(mv), - logger::to_string(permitted)); + ccf::logger::to_string(requested), + ccf::logger::to_string(mv), + ccf::logger::to_string(permitted)); } - logger::config::level() = permitted; + ccf::logger::config::level() = permitted; ccf::Enclave* enclave = nullptr; diff --git a/src/enclave/ringbuffer_logger.h b/src/enclave/ringbuffer_logger.h index db5c169a8b63..c39f0777db76 100644 --- a/src/enclave/ringbuffer_logger.h +++ b/src/enclave/ringbuffer_logger.h @@ -6,7 +6,7 @@ namespace ccf { - class RingbufferLogger : public logger::AbstractLogger + class RingbufferLogger : public ccf::logger::AbstractLogger { protected: ringbuffer::WriterPtr writer; @@ -19,7 +19,7 @@ namespace ccf RingbufferLogger(const ringbuffer::WriterPtr& writer_) : writer(writer_) {} void write( - const logger::LogLine& line, + const ccf::logger::LogLine& line, const std::optional& enclave_offset = std::nullopt) override { writer->write( diff --git a/src/enclave/rpc_handler.h b/src/enclave/rpc_handler.h index e6b9411bfca7..f4dd857798ff 100644 --- a/src/enclave/rpc_handler.h +++ b/src/enclave/rpc_handler.h @@ -11,7 +11,7 @@ #include #include -namespace kv +namespace ccf::kv { class CommittableTx; } diff --git a/src/enclave/rpc_sessions.h b/src/enclave/rpc_sessions.h index 15247c75d7c1..31fed0caab87 100644 --- a/src/enclave/rpc_sessions.h +++ b/src/enclave/rpc_sessions.h @@ -41,8 +41,8 @@ namespace ccf class RPCSessions : public std::enable_shared_from_this, public AbstractRPCResponder, - public http::ErrorReporter, - public http::ResponderLookup + public ::http::ErrorReporter, + public ::http::ResponderLookup { private: struct ListenInterface @@ -148,7 +148,7 @@ namespace ccf { if (app_protocol == "HTTP2") { - return std::make_shared( + return std::make_shared<::http::HTTP2ServerSession>( rpc_map, id, listen_interface_id, @@ -160,7 +160,7 @@ namespace ccf } else if (app_protocol == "HTTP1") { - return std::make_shared( + return std::make_shared<::http::HTTPServerSession>( rpc_map, id, listen_interface_id, @@ -287,20 +287,22 @@ namespace ccf return listening_interfaces.begin()->second.app_protocol; } - void set_node_cert(const crypto::Pem& cert_, const crypto::Pem& pk) + void set_node_cert( + const ccf::crypto::Pem& cert_, const ccf::crypto::Pem& pk) { set_cert(ccf::Authority::NODE, cert_, pk); } - void set_network_cert(const crypto::Pem& cert_, const crypto::Pem& pk) + void set_network_cert( + const ccf::crypto::Pem& cert_, const ccf::crypto::Pem& pk) { set_cert(ccf::Authority::SERVICE, cert_, pk); } void set_cert( ccf::Authority authority, - const crypto::Pem& cert_, - const crypto::Pem& pk, + const ccf::crypto::Pem& cert_, + const ccf::crypto::Pem& pk, const std::string& acme_configuration = "") { // Caller authentication is done by each frontend by looking up @@ -397,7 +399,7 @@ namespace ccf if (per_listen_interface.app_protocol == "HTTP2") { capped_session = - std::make_shared>( + std::make_shared>( rpc_map, id, listen_interface_id, @@ -410,7 +412,7 @@ namespace ccf else { capped_session = - std::make_shared>( + std::make_shared>( rpc_map, id, listen_interface_id, @@ -563,7 +565,7 @@ namespace ccf // want it to succeed even when we are busy. if (app_protocol == "HTTP2") { - auto session = std::make_shared( + auto session = std::make_shared<::http::HTTP2ClientSession>( id, writer_factory, std::move(ctx)); sessions.insert(std::make_pair(id, std::make_pair("", session))); sessions_peak = std::max(sessions_peak, sessions.size()); @@ -571,7 +573,7 @@ namespace ccf } else if (app_protocol == "HTTP1") { - auto session = std::make_shared( + auto session = std::make_shared<::http::HTTPClientSession>( id, writer_factory, std::move(ctx)); sessions.insert(std::make_pair(id, std::make_pair("", session))); sessions_peak = std::max(sessions_peak, sessions.size()); @@ -587,7 +589,7 @@ namespace ccf { std::lock_guard guard(lock); auto id = get_next_client_id(); - auto session = std::make_shared( + auto session = std::make_shared<::http::UnencryptedHTTPClientSession>( id, writer_factory); sessions.insert(std::make_pair(id, std::make_pair("", session))); sessions_peak = std::max(sessions_peak, sessions.size()); diff --git a/src/endpoints/authentication/all_of_auth.cpp b/src/endpoints/authentication/all_of_auth.cpp index b45b381e6b8e..ab1180cdce91 100644 --- a/src/endpoints/authentication/all_of_auth.cpp +++ b/src/endpoints/authentication/all_of_auth.cpp @@ -66,7 +66,7 @@ namespace ccf } std::unique_ptr AllOfAuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { @@ -103,7 +103,7 @@ namespace ccf void AllOfAuthnPolicy::set_unauthenticated_error( std::shared_ptr ctx, std::string&& error_reason) { - auto [pn, er] = nonstd::split_1(error_reason, ":"); + auto [pn, er] = ccf::nonstd::split_1(error_reason, ":"); std::string policy_name(pn); auto it = policies.find(policy_name); diff --git a/src/endpoints/authentication/cert_auth.cpp b/src/endpoints/authentication/cert_auth.cpp index 371b40f66f61..af23aaa89ea6 100644 --- a/src/endpoints/authentication/cert_auth.cpp +++ b/src/endpoints/authentication/cert_auth.cpp @@ -40,7 +40,7 @@ namespace ccf auto it = periods.find(der); if (it == periods.end()) { - auto verifier = crypto::make_unique_verifier(der); + auto verifier = ccf::crypto::make_unique_verifier(der); const auto [valid_from_timestring, valid_to_timestring] = verifier->validity_period(); @@ -104,7 +104,7 @@ namespace ccf UserCertAuthnPolicy::~UserCertAuthnPolicy() = default; std::unique_ptr UserCertAuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { @@ -120,7 +120,7 @@ namespace ccf return nullptr; } - auto caller_id = crypto::Sha256Hash(caller_cert).hex_str(); + auto caller_id = ccf::crypto::Sha256Hash(caller_cert).hex_str(); auto user_certs = tx.ro(Tables::USER_CERTS); if (user_certs->has(caller_id)) @@ -141,7 +141,7 @@ namespace ccf MemberCertAuthnPolicy::~MemberCertAuthnPolicy() = default; std::unique_ptr MemberCertAuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { @@ -152,7 +152,7 @@ namespace ccf return nullptr; } - auto caller_id = crypto::Sha256Hash(caller_cert).hex_str(); + auto caller_id = ccf::crypto::Sha256Hash(caller_cert).hex_str(); auto member_certs = tx.ro(Tables::MEMBER_CERTS); if (member_certs->has(caller_id)) @@ -167,7 +167,7 @@ namespace ccf } std::unique_ptr NodeCertAuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { diff --git a/src/endpoints/authentication/cose_auth.cpp b/src/endpoints/authentication/cose_auth.cpp index 1574982a3ae4..65c9c2d3d6f3 100644 --- a/src/endpoints/authentication/cose_auth.cpp +++ b/src/endpoints/authentication/cose_auth.cpp @@ -64,7 +64,7 @@ namespace ccf KID_INDEX, GOV_MSG_TYPE, GOV_MSG_PROPOSAL_ID, - GOV_MSG_CREATED_AT, + GOV_MSG_MSG_CREATED_AT, END_INDEX, }; QCBORItem header_items[END_INDEX + 1]; @@ -90,13 +90,13 @@ namespace ccf header_items[GOV_MSG_PROPOSAL_ID].uDataType = QCBOR_TYPE_TEXT_STRING; auto gov_msg_proposal_created_at = HEADER_PARAM_MSG_CREATED_AT; - header_items[GOV_MSG_CREATED_AT].label.string = + header_items[GOV_MSG_MSG_CREATED_AT].label.string = UsefulBuf_FromSZ(gov_msg_proposal_created_at); - header_items[GOV_MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; + header_items[GOV_MSG_MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; // Although this is really uint, specify QCBOR_TYPE_INT64 // QCBOR_TYPE_UINT64 only matches uint values that are greater than // INT64_MAX - header_items[GOV_MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; + header_items[GOV_MSG_MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE; @@ -121,7 +121,7 @@ namespace ccf } parsed.kid = qcbor_buf_to_string(header_items[KID_INDEX].val.string); - if (header_items[GOV_MSG_CREATED_AT].uDataType == QCBOR_TYPE_NONE) + if (header_items[GOV_MSG_MSG_CREATED_AT].uDataType == QCBOR_TYPE_NONE) { throw COSEDecodeError("Missing created_at in protected header"); } @@ -137,11 +137,12 @@ namespace ccf qcbor_buf_to_string(header_items[GOV_MSG_PROPOSAL_ID].val.string); } // Really uint, but the parser doesn't enforce that, so we must check - if (header_items[GOV_MSG_CREATED_AT].val.int64 < 0) + if (header_items[GOV_MSG_MSG_CREATED_AT].val.int64 < 0) { throw COSEDecodeError("Header parameter created_at must be positive"); } - parsed.gov_msg_created_at = header_items[GOV_MSG_CREATED_AT].val.int64; + parsed.gov_msg_created_at = + header_items[GOV_MSG_MSG_CREATED_AT].val.int64; QCBORDecode_ExitMap(&ctx); QCBORDecode_ExitBstrWrapped(&ctx); @@ -166,11 +167,13 @@ namespace ccf return {parsed, sig}; } - std::pair + std::pair extract_protected_header_and_signature( - const std::vector& cose_sign1) + const std::vector& cose_sign1, + const std::string& msg_type_name, + const std::string& created_at_name) { - ccf::ProtectedHeader parsed; + ccf::TimestampedProtectedHeader parsed; // Adapted from parse_cose_header_parameters in t_cose_parameters.c. // t_cose doesn't support custom header parameters yet. @@ -203,6 +206,8 @@ namespace ccf { ALG_INDEX, KID_INDEX, + MSG_TYPE, + MSG_CREATED_AT, END_INDEX, }; QCBORItem header_items[END_INDEX + 1]; @@ -215,6 +220,20 @@ namespace ccf header_items[KID_INDEX].uLabelType = QCBOR_TYPE_INT64; header_items[KID_INDEX].uDataType = QCBOR_TYPE_BYTE_STRING; + header_items[MSG_TYPE].label.string = + UsefulBuf_FromSZ(msg_type_name.c_str()); + header_items[MSG_TYPE].uLabelType = QCBOR_TYPE_TEXT_STRING; + header_items[MSG_TYPE].uDataType = QCBOR_TYPE_TEXT_STRING; + + auto gov_msg_proposal_created_at = HEADER_PARAM_MSG_CREATED_AT; + header_items[MSG_CREATED_AT].label.string = + UsefulBuf_FromSZ(created_at_name.c_str()); + header_items[MSG_CREATED_AT].uLabelType = QCBOR_TYPE_TEXT_STRING; + // Although this is really uint, specify QCBOR_TYPE_INT64 + // QCBOR_TYPE_UINT64 only matches uint values that are greater than + // INT64_MAX + header_items[MSG_CREATED_AT].uDataType = QCBOR_TYPE_INT64; + header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE; QCBORDecode_GetItemsInMap(&ctx, header_items); @@ -238,6 +257,19 @@ namespace ccf } parsed.kid = qcbor_buf_to_string(header_items[KID_INDEX].val.string); + if (header_items[MSG_TYPE].uDataType != QCBOR_TYPE_NONE) + { + parsed.msg_type = + qcbor_buf_to_string(header_items[MSG_TYPE].val.string); + } + if ( + header_items[MSG_CREATED_AT].uDataType != QCBOR_TYPE_NONE && + // Really uint, but the parser doesn't enforce that, so we must check + header_items[MSG_CREATED_AT].val.int64 > 0) + { + parsed.msg_created_at = header_items[MSG_CREATED_AT].val.int64; + } + QCBORDecode_ExitMap(&ctx); QCBORDecode_ExitBstrWrapped(&ctx); @@ -268,7 +300,7 @@ namespace ccf MemberCOSESign1AuthnPolicy::~MemberCOSESign1AuthnPolicy() = default; std::unique_ptr MemberCOSESign1AuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { @@ -302,7 +334,8 @@ namespace ccf auto member_cert = member_certs->get(phdr.kid); if (member_cert.has_value()) { - auto verifier = crypto::make_cose_verifier_from_cert(member_cert->raw()); + auto verifier = + ccf::crypto::make_cose_verifier_from_cert(member_cert->raw()); std::span body = { ctx->get_request_body().data(), ctx->get_request_body().size()}; @@ -373,7 +406,7 @@ namespace ccf "Signer must be a member identity registered with this service."}}); std::unique_ptr ActiveMemberCOSESign1AuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { @@ -404,13 +437,13 @@ namespace ccf return ident; } - UserCOSESign1AuthnPolicy::UserCOSESign1AuthnPolicy() = default; UserCOSESign1AuthnPolicy::~UserCOSESign1AuthnPolicy() = default; - std::unique_ptr UserCOSESign1AuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, - const std::shared_ptr& ctx, - std::string& error_reason) + std::unique_ptr UserCOSESign1AuthnPolicy:: + _authenticate( + ccf::kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) { const auto& headers = ctx->get_request_headers(); const auto content_type_it = headers.find(http::headers::CONTENT_TYPE); @@ -427,8 +460,8 @@ namespace ccf return nullptr; } - auto [phdr, cose_signature] = - cose::extract_protected_header_and_signature(ctx->get_request_body()); + auto [phdr, cose_signature] = cose::extract_protected_header_and_signature( + ctx->get_request_body(), msg_type_name, msg_created_at_name); if (!cose::is_ecdsa_alg(phdr.alg)) { @@ -441,7 +474,8 @@ namespace ccf auto user_cert = user_certs->get(phdr.kid); if (user_cert.has_value()) { - auto verifier = crypto::make_cose_verifier_from_cert(user_cert->raw()); + auto verifier = + ccf::crypto::make_cose_verifier_from_cert(user_cert->raw()); std::span body = { ctx->get_request_body().data(), ctx->get_request_body().size()}; @@ -467,6 +501,14 @@ namespace ccf } } + std::unique_ptr UserCOSESign1AuthnPolicy::authenticate( + ccf::kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) + { + return _authenticate(tx, ctx, error_reason); + } + void UserCOSESign1AuthnPolicy::set_unauthenticated_error( std::shared_ptr ctx, std::string&& error_reason) { @@ -489,4 +531,25 @@ namespace ccf "Request payload must be a COSE Sign1 document, with expected " "protected headers. " "Signer must be a user identity registered with this service."}}); + + std::unique_ptr TypedUserCOSESign1AuthnPolicy::authenticate( + ccf::kv::ReadOnlyTx& tx, + const std::shared_ptr& ctx, + std::string& error_reason) + { + auto identity = _authenticate(tx, ctx, error_reason); + + if ( + identity != nullptr && + identity->protected_header.msg_type != expected_msg_type) + { + error_reason = fmt::format( + "Unexpected message type: {}, expected: {}", + identity->protected_header.msg_type, + expected_msg_type); + return nullptr; + } + + return identity; + } } diff --git a/src/endpoints/authentication/empty_auth.cpp b/src/endpoints/authentication/empty_auth.cpp index 6b5246e3b41b..f94d57d9f6fd 100644 --- a/src/endpoints/authentication/empty_auth.cpp +++ b/src/endpoints/authentication/empty_auth.cpp @@ -6,7 +6,7 @@ namespace ccf { std::unique_ptr EmptyAuthnPolicy::authenticate( - kv::ReadOnlyTx&, const std::shared_ptr&, std::string&) + ccf::kv::ReadOnlyTx&, const std::shared_ptr&, std::string&) { return std::make_unique(); } diff --git a/src/endpoints/authentication/jwt_auth.cpp b/src/endpoints/authentication/jwt_auth.cpp index 9834c365a3e6..466adba15eb9 100644 --- a/src/endpoints/authentication/jwt_auth.cpp +++ b/src/endpoints/authentication/jwt_auth.cpp @@ -44,7 +44,7 @@ namespace ccf tid, constraint); - const auto issuer_url = http::parse_url_full(constraint); + const auto issuer_url = ::http::parse_url_full(constraint); if (issuer_url.host != microsoft_entra_domain) { return iss == constraint && @@ -75,8 +75,9 @@ namespace ccf // Check for details here: // https://learn.microsoft.com/en-us/entra/identity-platform/access-tokens#validate-the-issuer. - const auto url = http::parse_url_full(iss); - const auto tenant_id = first_non_empty_chunk(nonstd::split(url.path, "/")); + const auto url = ::http::parse_url_full(iss); + const auto tenant_id = + first_non_empty_chunk(ccf::nonstd::split(url.path, "/")); return tenant_id && tid && *tid == *tenant_id; } @@ -87,20 +88,20 @@ namespace ccf using DER = std::vector; ccf::pal::Mutex verifiers_lock; - LRU verifiers; + LRU verifiers; VerifiersCache(size_t max_verifiers = DEFAULT_MAX_VERIFIERS) : verifiers(max_verifiers) {} - crypto::VerifierPtr get_verifier(const DER& der) + ccf::crypto::VerifierPtr get_verifier(const DER& der) { std::lock_guard guard(verifiers_lock); auto it = verifiers.find(der); if (it == verifiers.end()) { - it = verifiers.insert(der, crypto::make_unique_verifier(der)); + it = verifiers.insert(der, ccf::crypto::make_unique_verifier(der)); } return it->second; @@ -114,14 +115,14 @@ namespace ccf JwtAuthnPolicy::~JwtAuthnPolicy() = default; std::unique_ptr JwtAuthnPolicy::authenticate( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::shared_ptr& ctx, std::string& error_reason) { const auto& headers = ctx->get_request_headers(); const auto token_opt = - http::JwtVerifier::extract_token(headers, error_reason); + ::http::JwtVerifier::extract_token(headers, error_reason); if (!token_opt) { @@ -162,7 +163,7 @@ namespace ccf for (const auto& metadata : *token_keys) { auto verifier = verifiers->get_verifier(metadata.cert); - if (!http::JwtVerifier::validate_token_signature(token, verifier)) + if (!::http::JwtVerifier::validate_token_signature(token, verifier)) { continue; } diff --git a/src/endpoints/base_endpoint_registry.cpp b/src/endpoints/base_endpoint_registry.cpp index 4d96f5b62a99..ac91edaa06e2 100644 --- a/src/endpoints/base_endpoint_registry.cpp +++ b/src/endpoints/base_endpoint_registry.cpp @@ -3,7 +3,6 @@ #include "ccf/base_endpoint_registry.h" -#include "ccf/pal/locking.h" #include "ccf/service/tables/members.h" #include "ccf/service/tables/nodes.h" #include "ccf/service/tables/users.h" @@ -13,13 +12,15 @@ namespace ccf { BaseEndpointRegistry::BaseEndpointRegistry( - const std::string& method_prefix_, ccfapp::AbstractNodeContext& context_) : + const std::string& method_prefix_, ccf::AbstractNodeContext& context_) : ccf::endpoints::EndpointRegistry(method_prefix_), context(context_) {} - ApiResult BaseEndpointRegistry::get_view_history_v1( - std::vector& history, ccf::View since) + ApiResult BaseEndpointRegistry::get_view_history_v2( + std::vector& history, + ccf::View since, + ccf::InvalidArgsReason& reason) { try { @@ -27,7 +28,7 @@ namespace ccf { if (since < 1) { - // views start at 1 + reason = ccf::InvalidArgsReason::ViewSmallerThanOne; return ApiResult::InvalidArgs; } auto latest_view = consensus->get_view(); @@ -54,6 +55,13 @@ namespace ccf } } + ApiResult BaseEndpointRegistry::get_view_history_v1( + std::vector& history, ccf::View since) + { + ccf::InvalidArgsReason ignored; + return get_view_history_v2(history, since, ignored); + } + ApiResult BaseEndpointRegistry::get_status_for_txid_v1( ccf::View view, ccf::SeqNo seqno, ccf::TxStatus& tx_status) { @@ -107,7 +115,7 @@ namespace ccf } ApiResult BaseEndpointRegistry::generate_openapi_document_v1( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const std::string& title, const std::string& description, const std::string& document_version, @@ -128,7 +136,7 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_quote_for_this_node_v1( - kv::ReadOnlyTx& tx, QuoteInfo& quote_info) + ccf::kv::ReadOnlyTx& tx, QuoteInfo& quote_info) { try { @@ -167,7 +175,7 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_quotes_for_all_trusted_nodes_v1( - kv::ReadOnlyTx& tx, std::map& quotes) + ccf::kv::ReadOnlyTx& tx, std::map& quotes) { try { @@ -222,7 +230,7 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_user_data_v1( - kv::ReadOnlyTx& tx, const UserId& user_id, nlohmann::json& user_data) + ccf::kv::ReadOnlyTx& tx, const UserId& user_id, nlohmann::json& user_data) { try { @@ -244,7 +252,9 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_member_data_v1( - kv::ReadOnlyTx& tx, const MemberId& member_id, nlohmann::json& member_data) + ccf::kv::ReadOnlyTx& tx, + const MemberId& member_id, + nlohmann::json& member_data) { try { @@ -266,7 +276,9 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_user_cert_v1( - kv::ReadOnlyTx& tx, const UserId& user_id, crypto::Pem& user_cert_pem) + ccf::kv::ReadOnlyTx& tx, + const UserId& user_id, + ccf::crypto::Pem& user_cert_pem) { try { @@ -288,7 +300,9 @@ namespace ccf } ApiResult BaseEndpointRegistry::get_member_cert_v1( - kv::ReadOnlyTx& tx, const MemberId& member_id, crypto::Pem& member_cert_pem) + ccf::kv::ReadOnlyTx& tx, + const MemberId& member_id, + ccf::crypto::Pem& member_cert_pem) { try { diff --git a/src/endpoints/common_endpoint_registry.cpp b/src/endpoints/common_endpoint_registry.cpp index 8023fbeb1055..356c2f7bcbe3 100644 --- a/src/endpoints/common_endpoint_registry.cpp +++ b/src/endpoints/common_endpoint_registry.cpp @@ -62,7 +62,7 @@ namespace ccf } CommonEndpointRegistry::CommonEndpointRegistry( - const std::string& method_prefix_, ccfapp::AbstractNodeContext& context_) : + const std::string& method_prefix_, ccf::AbstractNodeContext& context_) : BaseEndpointRegistry(method_prefix_, context_) {} @@ -285,13 +285,12 @@ namespace ccf auto get_receipt = [](auto& ctx, ccf::historical::StatePtr historical_state) { - const auto [pack, params] = - ccf::jsonhandler::get_json_params(ctx.rpc_ctx); + const auto params = ccf::jsonhandler::get_json_params(ctx.rpc_ctx); assert(historical_state->receipt); auto out = ccf::describe_receipt_v1(*historical_state->receipt); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); - ccf::jsonhandler::set_response(out, ctx.rpc_ctx, pack); + ccf::jsonhandler::set_response(out, ctx.rpc_ctx); }; make_read_only_endpoint( diff --git a/src/endpoints/endpoint_registry.cpp b/src/endpoints/endpoint_registry.cpp index b477764a2f62..32f0da7e776d 100644 --- a/src/endpoints/endpoint_registry.cpp +++ b/src/endpoints/endpoint_registry.cpp @@ -304,7 +304,8 @@ namespace ccf::endpoints default_endpoint = std::move(tmp); } - void EndpointRegistry::build_api(nlohmann::json& document, kv::ReadOnlyTx&) + void EndpointRegistry::build_api( + nlohmann::json& document, ccf::kv::ReadOnlyTx&) { // Add common components: // - Descriptions of each kind of forwarding @@ -389,7 +390,7 @@ namespace ccf::endpoints void EndpointRegistry::init_handlers() {} EndpointDefinitionPtr EndpointRegistry::find_endpoint( - kv::Tx&, ccf::RpcContext& rpc_ctx) + ccf::kv::Tx&, ccf::RpcContext& rpc_ctx) { auto method = rpc_ctx.get_method(); auto endpoints_for_exact_method = fully_qualified_endpoints.find(method); @@ -438,7 +439,7 @@ namespace ccf::endpoints const auto& template_name = endpoint->spec.template_component_names[i]; const auto& template_value = match[i + 1].str(); - auto decoded_value = http::url_decode(template_value); + auto decoded_value = ::http::url_decode(template_value); path_params[template_name] = template_value; decoded_path_params[template_name] = decoded_value; } @@ -497,7 +498,7 @@ namespace ccf::endpoints } std::set EndpointRegistry::get_allowed_verbs( - kv::Tx& tx, const ccf::RpcContext& rpc_ctx) + ccf::kv::Tx& tx, const ccf::RpcContext& rpc_ctx) { auto method = rpc_ctx.get_method(); @@ -555,12 +556,12 @@ namespace ccf::endpoints // Default implementation does nothing void EndpointRegistry::tick(std::chrono::milliseconds) {} - void EndpointRegistry::set_consensus(kv::Consensus* c) + void EndpointRegistry::set_consensus(ccf::kv::Consensus* c) { consensus = c; } - void EndpointRegistry::set_history(kv::TxHistory* h) + void EndpointRegistry::set_history(ccf::kv::TxHistory* h) { history = h; } diff --git a/src/endpoints/grpc/grpc.h b/src/endpoints/grpc/grpc.h index 3c6cd57d3a19..5cf134e3a18b 100644 --- a/src/endpoints/grpc/grpc.h +++ b/src/endpoints/grpc/grpc.h @@ -34,7 +34,7 @@ namespace ccf::grpc http::headervalues::contenttype::GRPC)); } - if constexpr (nonstd::is_std_vector::value) + if constexpr (ccf::nonstd::is_std_vector::value) { using Message = typename In::value_type; In messages; @@ -117,7 +117,7 @@ namespace ccf::grpc { std::vector v; - if constexpr (nonstd::is_std_vector::value) + if constexpr (ccf::nonstd::is_std_vector::value) { v = serialise_grpc_messages(success_response->body); } @@ -187,8 +187,8 @@ namespace ccf }; } - // Note: For now, only command endpoints (i.e. with no kv::Tx) support gRPC - // server streaming. + // Note: For now, only command endpoints (i.e. with no ccf::kv::Tx) support + // gRPC server streaming. template endpoints::CommandEndpointFunction grpc_command_unary_stream_adapter( const GrpcCommandUnaryStreamEndpoint& f) diff --git a/src/endpoints/grpc/stream.h b/src/endpoints/grpc/stream.h index b082763348ab..b83de213b9bc 100644 --- a/src/endpoints/grpc/stream.h +++ b/src/endpoints/grpc/stream.h @@ -18,10 +18,10 @@ namespace ccf::grpc class BaseStream { private: - std::shared_ptr http_responder; + std::shared_ptr http_responder; protected: - BaseStream(const std::shared_ptr& r) : + BaseStream(const std::shared_ptr& r) : http_responder(r) {} @@ -55,7 +55,7 @@ namespace ccf::grpc { public: Stream( - const std::shared_ptr& r, + const std::shared_ptr& r, http_status s = HTTP_STATUS_OK, const http::HeaderMap& h = default_response_headers) : BaseStream(r) @@ -116,7 +116,7 @@ namespace ccf::grpc template using DetachedStreamPtr = std::unique_ptr>; - static std::shared_ptr get_http_responder( + static std::shared_ptr get_http_responder( const std::shared_ptr& rpc_ctx) { auto http_responder = rpc_ctx->get_responder(); diff --git a/src/endpoints/json_handler.cpp b/src/endpoints/json_handler.cpp index bd7ecf8cb63a..6e9a7e76387f 100644 --- a/src/endpoints/json_handler.cpp +++ b/src/endpoints/json_handler.cpp @@ -15,128 +15,26 @@ namespace ccf { namespace jsonhandler { - char const* pack_to_content_type(serdes::Pack p) + nlohmann::json get_json_params(const std::shared_ptr& ctx) { - switch (p) - { - case serdes::Pack::Text: - { - return http::headervalues::contenttype::JSON; - } - case serdes::Pack::MsgPack: - { - return http::headervalues::contenttype::MSGPACK; - } - default: - { - return nullptr; - } - } - } - - serdes::Pack detect_json_pack(const std::shared_ptr& ctx) - { - std::optional packing = std::nullopt; - - const auto content_type_it = - ctx->get_request_header(http::headers::CONTENT_TYPE); - if (content_type_it.has_value()) - { - const auto& content_type = content_type_it.value(); - if (content_type == http::headervalues::contenttype::JSON) - { - packing = serdes::Pack::Text; - } - else if (content_type == http::headervalues::contenttype::MSGPACK) - { - packing = serdes::Pack::MsgPack; - } - else - { - throw RpcException( - HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE, - ccf::errors::UnsupportedContentType, - fmt::format( - "Unsupported content type {}. Only {} and {} are currently " - "supported", - content_type, - http::headervalues::contenttype::JSON, - http::headervalues::contenttype::MSGPACK)); - } - } - else - { - packing = serdes::detect_pack(ctx->get_request_body()); - } - - return packing.value_or(serdes::Pack::Text); - } - - serdes::Pack get_response_pack( - const std::shared_ptr& ctx, serdes::Pack request_pack) - { - const auto accept_it = ctx->get_request_header(http::headers::ACCEPT); - if (accept_it.has_value()) - { - const auto accept_options = - http::parse_accept_header(accept_it.value()); - for (const auto& option : accept_options) - { - if (option.matches(http::headervalues::contenttype::JSON)) - { - return serdes::Pack::Text; - } - if (option.matches(http::headervalues::contenttype::MSGPACK)) - { - return serdes::Pack::MsgPack; - } - } - - throw RpcException( - HTTP_STATUS_NOT_ACCEPTABLE, - ccf::errors::UnsupportedContentType, - fmt::format( - "No supported content type in accept header: {}\nOnly {} and {} " - "are currently supported", - accept_it.value(), - http::headervalues::contenttype::JSON, - http::headervalues::contenttype::MSGPACK)); - } - - return request_pack; - } - - nlohmann::json get_params_from_body( - const std::shared_ptr& ctx, serdes::Pack pack) - { - return serdes::unpack(ctx->get_request_body(), pack); - } - - std::pair get_json_params( - const std::shared_ptr& ctx) - { - const auto pack = detect_json_pack(ctx); - nlohmann::json params = nullptr; if ( !ctx->get_request_body().empty() // Body of GET is ignored && ctx->get_request_verb() != HTTP_GET) { - params = get_params_from_body(ctx, pack); + params = nlohmann::json::parse(ctx->get_request_body()); } else { params = nlohmann::json::object(); } - return std::make_pair(pack, params); + return params; } void set_response( - JsonAdapterResponse&& res, - std::shared_ptr& ctx, - serdes::Pack request_packing) + JsonAdapterResponse&& res, std::shared_ptr& ctx) { auto error = std::get_if(&res); if (error != nullptr) @@ -160,28 +58,41 @@ namespace ccf else { ctx->set_response_status(HTTP_STATUS_OK); - const auto packing = get_response_pack(ctx, request_packing); - switch (packing) + const auto accept_it = + ctx->get_request_header(http::headers::ACCEPT); + if (accept_it.has_value()) { - case serdes::Pack::Text: - { - const auto s = body->dump(); - ctx->set_response_body( - std::vector(s.begin(), s.end())); - break; - } - case serdes::Pack::MsgPack: + const auto accept_options = + ::http::parse_accept_header(accept_it.value()); + bool matched = false; + for (const auto& option : accept_options) { - ctx->set_response_body(nlohmann::json::to_msgpack(*body)); - break; + if (option.matches(http::headervalues::contenttype::JSON)) + { + matched = true; + break; + } } - default: + + if (!matched) { - throw std::logic_error("Unhandled serdes::Pack"); + throw RpcException( + HTTP_STATUS_NOT_ACCEPTABLE, + ccf::errors::UnsupportedContentType, + fmt::format( + "No supported content type in accept header: {}\nOnly {} " + "is currently supported", + accept_it.value(), + http::headervalues::contenttype::JSON)); } } + + const auto s = body->dump(); + ctx->set_response_body(std::vector(s.begin(), s.end())); + ctx->set_response_header( - http::headers::CONTENT_TYPE, pack_to_content_type(packing)); + http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); } } } @@ -220,9 +131,8 @@ namespace ccf endpoints::EndpointFunction json_adapter(const HandlerJsonParamsAndForward& f) { return [f](endpoints::EndpointContext& ctx) { - auto [packing, params] = jsonhandler::get_json_params(ctx.rpc_ctx); - jsonhandler::set_response( - f(ctx, std::move(params)), ctx.rpc_ctx, packing); + auto params = jsonhandler::get_json_params(ctx.rpc_ctx); + jsonhandler::set_response(f(ctx, std::move(params)), ctx.rpc_ctx); }; } @@ -230,9 +140,8 @@ namespace ccf const ReadOnlyHandlerWithJson& f) { return [f](endpoints::ReadOnlyEndpointContext& ctx) { - auto [packing, params] = jsonhandler::get_json_params(ctx.rpc_ctx); - jsonhandler::set_response( - f(ctx, std::move(params)), ctx.rpc_ctx, packing); + auto params = jsonhandler::get_json_params(ctx.rpc_ctx); + jsonhandler::set_response(f(ctx, std::move(params)), ctx.rpc_ctx); }; } @@ -240,9 +149,8 @@ namespace ccf const CommandHandlerWithJson& f) { return [f](endpoints::CommandEndpointContext& ctx) { - auto [packing, params] = jsonhandler::get_json_params(ctx.rpc_ctx); - jsonhandler::set_response( - f(ctx, std::move(params)), ctx.rpc_ctx, packing); + auto params = jsonhandler::get_json_params(ctx.rpc_ctx); + jsonhandler::set_response(f(ctx, std::move(params)), ctx.rpc_ctx); }; } } diff --git a/src/endpoints/test/endpoint_registry.cpp b/src/endpoints/test/endpoint_registry.cpp index 59e544aeb7ce..9a2d26495569 100644 --- a/src/endpoints/test/endpoint_registry.cpp +++ b/src/endpoints/test/endpoint_registry.cpp @@ -29,7 +29,7 @@ std::optional require_parsed_components( TEST_CASE("URL template parsing") { - logger::config::default_init(); + ccf::logger::config::default_init(); std::optional parsed; std::string path; diff --git a/src/host/handle_ring_buffer.h b/src/host/handle_ring_buffer.h index ea3d14ed0ae0..311d13552ad6 100644 --- a/src/host/handle_ring_buffer.h +++ b/src/host/handle_ring_buffer.h @@ -49,7 +49,7 @@ namespace asynchost thread_id, msg] = ringbuffer::read_message(data, size); - logger::LogLine ll( + ccf::logger::LogLine ll( log_level, tag, file_name.c_str(), line_number, thread_id); ll.msg = msg; @@ -76,7 +76,7 @@ namespace asynchost offset_time = enclave_time_s - host_time_s; } - auto& loggers = logger::config::loggers(); + auto& loggers = ccf::logger::config::loggers(); for (auto const& logger : loggers) { logger->write(ll, offset_time); diff --git a/src/host/ledger.h b/src/host/ledger.h index 40df74a39f8c..37ecd504f3e8 100644 --- a/src/host/ledger.h +++ b/src/host/ledger.h @@ -288,12 +288,13 @@ namespace asynchost total_len = sizeof(positions_offset_header_t); auto len = total_file_size - total_len; - kv::SerialisedEntryHeader entry_header = {}; + ccf::kv::SerialisedEntryHeader entry_header = {}; size_t current_idx = start_idx; - while (len >= kv::serialised_entry_header_size) + while (len >= ccf::kv::serialised_entry_header_size) { if ( - fread(&entry_header, kv::serialised_entry_header_size, 1, file) != + fread( + &entry_header, ccf::kv::serialised_entry_header_size, 1, file) != 1) { LOG_FAIL_FMT( @@ -303,7 +304,7 @@ namespace asynchost return; } - len -= kv::serialised_entry_header_size; + len -= ccf::kv::serialised_entry_header_size; const auto& entry_size = entry_header.size; if (len < entry_size) @@ -330,7 +331,7 @@ namespace asynchost current_idx++; positions.push_back(total_len); - total_len += (kv::serialised_entry_header_size + entry_size); + total_len += (ccf::kv::serialised_entry_header_size + entry_size); } completed = false; } @@ -1297,9 +1298,10 @@ namespace asynchost TimeBoundLogger log_if_slow(fmt::format( "Writing ledger entry - {} bytes, committable={}", size, committable)); - auto header = serialized::peek(data, size); + auto header = + serialized::peek(data, size); - if (header.flags & kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE) + if (header.flags & ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE) { LOG_TRACE_FMT( "Forcing ledger chunk before entry as required by the entry header " @@ -1314,7 +1316,7 @@ namespace asynchost } bool force_chunk_after = - header.flags & kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER; + header.flags & ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER; if (force_chunk_after) { if (!committable) diff --git a/src/host/main.cpp b/src/host/main.cpp index ffde70da8e1a..b3c13e018a1b 100644 --- a/src/host/main.cpp +++ b/src/host/main.cpp @@ -106,10 +106,11 @@ int main(int argc, char** argv) LoggerLevel enclave_log_level = LoggerLevel::INFO; std::map log_level_options; - for (size_t i = logger::MOST_VERBOSE; i < LoggerLevel::MAX_LOG_LEVEL; ++i) + for (size_t i = ccf::logger::MOST_VERBOSE; i < LoggerLevel::MAX_LOG_LEVEL; + ++i) { const auto l = (LoggerLevel)i; - log_level_options[logger::to_string(l)] = l; + log_level_options[ccf::logger::to_string(l)] = l; } app @@ -179,11 +180,11 @@ int main(int argc, char** argv) if (config.logging.format == host::LogFormat::JSON) { - logger::config::add_json_console_logger(); + ccf::logger::config::add_json_console_logger(); } else { - logger::config::add_text_console_logger(); + ccf::logger::config::add_text_console_logger(); } LOG_INFO_FMT("CCF version: {}", ccf::ccf_version); @@ -201,7 +202,7 @@ int main(int argc, char** argv) nlohmann::json environment; for (int i = 0; environ[i] != nullptr; i++) { - auto [k, v] = nonstd::split_1(environ[i], "="); + auto [k, v] = ccf::nonstd::split_1(environ[i], "="); environment[k] = v; } @@ -271,7 +272,7 @@ int main(int argc, char** argv) files::dump(fmt::format("{}", ::getpid()), config.output_files.pid_file); // set the host log level - logger::config::level() = config.logging.host_level; + ccf::logger::config::level() = config.logging.host_level; asynchost::TimeBoundLogger::default_max_time = config.slow_io_logging_threshold; @@ -511,7 +512,7 @@ int main(int argc, char** argv) LOG_DEBUG_FMT( "Resolving snp_security_policy_file: {}", security_policy_file); security_policy_file = - nonstd::expand_envvars_in_path(security_policy_file); + ccf::nonstd::expand_envvars_in_path(security_policy_file); LOG_DEBUG_FMT( "Resolved snp_security_policy_file: {}", security_policy_file); @@ -526,7 +527,7 @@ int main(int argc, char** argv) LOG_DEBUG_FMT( "Resolving snp_uvm_endorsements_file: {}", snp_uvm_endorsements_file); snp_uvm_endorsements_file = - nonstd::expand_envvars_in_path(snp_uvm_endorsements_file); + ccf::nonstd::expand_envvars_in_path(snp_uvm_endorsements_file); LOG_DEBUG_FMT( "Resolved snp_uvm_endorsements_file: {}", snp_uvm_endorsements_file); @@ -549,14 +550,14 @@ int main(int argc, char** argv) auto pos = url.find(':'); if (pos == std::string::npos) { - endorsement_servers_it->url = nonstd::expand_envvar(url); + endorsement_servers_it->url = ccf::nonstd::expand_envvar(url); } else { endorsement_servers_it->url = fmt::format( "{}:{}", - nonstd::expand_envvar(url.substr(0, pos)), - nonstd::expand_envvar(url.substr(pos + 1))); + ccf::nonstd::expand_envvar(url.substr(0, pos)), + ccf::nonstd::expand_envvar(url.substr(pos + 1))); } LOG_DEBUG_FMT( "Resolved snp_endorsements_server url: {}", @@ -591,19 +592,19 @@ int main(int argc, char** argv) LOG_INFO_FMT("Startup host time: {}", startup_host_time); startup_config.startup_host_time = - ds::to_x509_time_string(startup_host_time); + ::ds::to_x509_time_string(startup_host_time); if (config.command.type == StartType::Start) { for (auto const& m : config.command.start.members) { - std::optional public_encryption_key = std::nullopt; + std::optional public_encryption_key = std::nullopt; if ( m.encryption_public_key_file.has_value() && !m.encryption_public_key_file.value().empty()) { - public_encryption_key = - crypto::Pem(files::slurp(m.encryption_public_key_file.value())); + public_encryption_key = ccf::crypto::Pem( + files::slurp(m.encryption_public_key_file.value())); } nlohmann::json md = nullptr; @@ -613,7 +614,7 @@ int main(int argc, char** argv) } startup_config.start.members.emplace_back( - crypto::Pem(files::slurp(m.certificate_file)), + ccf::crypto::Pem(files::slurp(m.certificate_file)), public_encryption_key, md); } diff --git a/src/host/test/ledger.cpp b/src/host/test/ledger.cpp index 8f066b2d244f..f0320a0654c5 100644 --- a/src/host/test/ledger.cpp +++ b/src/host/test/ledger.cpp @@ -149,12 +149,12 @@ void verify_framed_entries_range( const uint8_t* data = &framed_entries[pos]; size_t size = framed_entries.size() - pos; - auto header = serialized::read(data, size); + auto header = serialized::read(data, size); auto header_size = header.size; REQUIRE(header_size == sizeof(TestLedgerEntry)); REQUIRE(TestLedgerEntry(data, size).value() == idx); - pos += kv::serialised_entry_header_size + sizeof(TestLedgerEntry); + pos += ccf::kv::serialised_entry_header_size + sizeof(TestLedgerEntry); idx++; } @@ -169,7 +169,7 @@ void read_entry_from_ledger(Ledger& ledger, size_t idx) auto& entry = framed_entry->data; const uint8_t* data = entry.data(); auto size = entry.size(); - auto header = serialized::read(data, size); + auto header = serialized::read(data, size); auto header_size = header.size; REQUIRE(header_size == sizeof(TestLedgerEntry)); @@ -195,14 +195,15 @@ size_t read_entries_range_from_ledger( } using LedgerDirCapture = - std::vector>; + std::vector>; LedgerDirCapture capture_ledger_dir() { LedgerDirCapture capture = {}; for (auto const& f : fs::directory_iterator(ledger_dir)) { capture.emplace_back( - f.path().filename(), crypto::Sha256Hash(files::slurp(f.path().string()))); + f.path().filename(), + ccf::crypto::Sha256Hash(files::slurp(f.path().string()))); } return capture; } @@ -211,11 +212,11 @@ std::vector make_ledger_entry(size_t idx, uint8_t header_flags = 0) { auto e = TestLedgerEntry(idx); std::vector framed_entry( - kv::serialised_entry_header_size + sizeof(TestLedgerEntry)); + ccf::kv::serialised_entry_header_size + sizeof(TestLedgerEntry)); auto data = framed_entry.data(); auto size = framed_entry.size(); - kv::SerialisedEntryHeader header; + ccf::kv::SerialisedEntryHeader header; header.set_size(sizeof(TestLedgerEntry)); header.flags = header_flags; @@ -282,7 +283,7 @@ size_t get_entries_per_chunk(size_t chunk_threshold) // size of each entry return ceil( (static_cast(chunk_threshold - sizeof(size_t))) / - (kv::serialised_entry_header_size + sizeof(TestLedgerEntry))); + (ccf::kv::serialised_entry_header_size + sizeof(TestLedgerEntry))); } // Assumes that no entries have been written yet @@ -384,7 +385,7 @@ TEST_CASE("Regular chunking") // Write a new committable entry that forces a new ledger chunk is_committable = true; - entry_submitter.write(is_committable, kv::FORCE_LEDGER_CHUNK_AFTER); + entry_submitter.write(is_committable, ccf::kv::FORCE_LEDGER_CHUNK_AFTER); REQUIRE(number_of_files_in_ledger_dir() == number_of_files_after); // Because of forcing a new chunk, the next entry will create a new chunk @@ -395,7 +396,7 @@ TEST_CASE("Regular chunking") REQUIRE(number_of_files_in_ledger_dir() == number_of_files_after + 1); is_committable = true; - entry_submitter.write(is_committable, kv::FORCE_LEDGER_CHUNK_BEFORE); + entry_submitter.write(is_committable, ccf::kv::FORCE_LEDGER_CHUNK_BEFORE); // A new chunk is created before, as the entry is committable and forced REQUIRE(number_of_files_in_ledger_dir() == number_of_files_after + 2); } @@ -1077,7 +1078,7 @@ void corrupt_ledger_file( else if (corrupt_first_hdr) { REQUIRE(fread(&table_offset, sizeof(table_offset), 1, file) == 1); - kv::SerialisedEntryHeader entry_header = {.size = 0xffffffff}; + ccf::kv::SerialisedEntryHeader entry_header = {.size = 0xffffffff}; fwrite(&entry_header, sizeof(entry_header), 1, file); } else if (corrupt_last_entry) @@ -1086,7 +1087,7 @@ void corrupt_ledger_file( std::vector last_entry = {}; while (true) { - kv::SerialisedEntryHeader entry_header = {}; + ccf::kv::SerialisedEntryHeader entry_header = {}; if (fread(&entry_header, sizeof(entry_header), 1, file) != 1) { break; @@ -1395,7 +1396,7 @@ TEST_CASE("Chunking according to entry header flag") INFO("Write an entry with the ledger chunking after header flag enabled"); { entry_submitter.write( - is_committable, kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER); + is_committable, ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER); REQUIRE(number_of_files_in_ledger_dir() == 1); @@ -1418,7 +1419,7 @@ TEST_CASE("Chunking according to entry header flag") { auto ledger_files_count = number_of_files_in_ledger_dir(); entry_submitter.write( - is_committable, kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE); + is_committable, ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE); // Forcing a new chunk before creating a new chunk to store this entry REQUIRE(number_of_files_in_ledger_dir() == ledger_files_count + 1); @@ -1725,12 +1726,12 @@ TEST_CASE("Ledger init with existing files") int main(int argc, char** argv) { - logger::config::default_init(); - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/http/http2_callbacks.h b/src/http/http2_callbacks.h index 45634231ca5a..3299003df8b3 100644 --- a/src/http/http2_callbacks.h +++ b/src/http/http2_callbacks.h @@ -186,8 +186,8 @@ namespace http2 auto* p = get_parser(user_data); const auto& configuration = p->get_configuration(); - auto const& max_header_size = - configuration.max_header_size.value_or(http::default_max_header_size); + auto const& max_header_size = configuration.max_header_size.value_or( + ccf::http::default_max_header_size); if (namelen > max_header_size) { throw http::RequestHeaderTooLargeException( @@ -209,8 +209,8 @@ namespace http2 } auto* stream_data = get_stream_data(session, stream_id); - const auto max_headers_count = - configuration.max_headers_count.value_or(http::default_max_headers_count); + const auto max_headers_count = configuration.max_headers_count.value_or( + ccf::http::default_max_headers_count); if (stream_data->incoming.headers.size() >= max_headers_count) { throw http::RequestHeaderTooLargeException( @@ -242,7 +242,7 @@ namespace http2 stream_data->incoming.body.end(), data, data + len); auto const& max_body_size = - configuration.max_body_size.value_or(http::default_max_body_size); + configuration.max_body_size.value_or(ccf::http::default_max_body_size); if (stream_data->incoming.body.size() > max_body_size) { throw http::RequestPayloadTooLargeException( @@ -282,7 +282,7 @@ namespace http2 auto* p = get_parser(user_data); const auto& configuration = p->get_configuration(); const auto& max_frame_size = - configuration.max_frame_size.value_or(http::default_max_frame_size); + configuration.max_frame_size.value_or(ccf::http::default_max_frame_size); LOG_TRACE_FMT( "http2::on_data_source_read_length_callback: {}, {}, allowed [1, " diff --git a/src/http/http2_parser.h b/src/http/http2_parser.h index 4e5b91a33893..acdb8d7b4089 100644 --- a/src/http/http2_parser.h +++ b/src/http/http2_parser.h @@ -21,7 +21,7 @@ namespace http2 // reject new streams ids less than this value. StreamId last_stream_id = 0; DataHandlerCB handle_outgoing_data; - http::ParserConfiguration configuration; + ccf::http::ParserConfiguration configuration; protected: std::map> streams; @@ -29,7 +29,8 @@ namespace http2 public: Parser( - const http::ParserConfiguration& configuration_, bool is_client = false) : + const ccf::http::ParserConfiguration& configuration_, + bool is_client = false) : configuration(configuration_) { LOG_TRACE_FMT("Creating HTTP2 parser"); @@ -74,25 +75,25 @@ namespace http2 settings.push_back( {NGHTTP2_SETTINGS_MAX_FRAME_SIZE, static_cast(configuration.max_frame_size.value_or( - http::default_max_frame_size))}); + ccf::http::default_max_frame_size))}); settings.push_back( {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, static_cast( configuration.max_concurrent_streams_count.value_or( - http::default_max_concurrent_streams_count))}); + ccf::http::default_max_concurrent_streams_count))}); settings.push_back( {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, static_cast(configuration.initial_window_size.value_or( - http::default_initial_window_size))}); + ccf::http::default_initial_window_size))}); // NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE is only a hint to client // (https://www.rfc-editor.org/rfc/rfc7540#section-10.5.1) settings.push_back( {NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, static_cast( configuration.max_headers_count.value_or( - http::default_max_headers_count) * + ccf::http::default_max_headers_count) * configuration.max_header_size.value_or( - http::default_max_header_size))}); + ccf::http::default_max_header_size))}); auto rv = nghttp2_submit_settings( session, NGHTTP2_FLAG_NONE, settings.data(), settings.size()); @@ -116,7 +117,7 @@ namespace http2 return last_stream_id; } - http::ParserConfiguration get_configuration() const override + ccf::http::ParserConfiguration get_configuration() const override { return configuration; } @@ -240,7 +241,7 @@ namespace http2 private: http::RequestProcessor& proc; - void submit_trailers(StreamId stream_id, http::HeaderMap&& trailers) + void submit_trailers(StreamId stream_id, ccf::http::HeaderMap&& trailers) { if (trailers.empty()) { @@ -267,14 +268,15 @@ namespace http2 void submit_response( StreamId stream_id, http_status status, - const http::HeaderMap& base_headers, - const http::HeaderMap& extra_headers = {}) + const ccf::http::HeaderMap& base_headers, + const ccf::http::HeaderMap& extra_headers = {}) { std::vector hdrs = {}; auto status_str = fmt::format( "{}", static_cast::type>(status)); - hdrs.emplace_back(make_nv(http2::headers::STATUS, status_str.data())); + hdrs.emplace_back( + make_nv(ccf::http2::headers::STATUS, status_str.data())); for (auto& [k, v] : base_headers) { @@ -301,7 +303,7 @@ namespace http2 public: ServerParser( http::RequestProcessor& proc_, - const http::ParserConfiguration& configuration_) : + const ccf::http::ParserConfiguration& configuration_) : Parser(configuration_, false), proc(proc_) {} @@ -324,8 +326,8 @@ namespace http2 void respond( StreamId stream_id, http_status status, - const http::HeaderMap& headers, - http::HeaderMap&& trailers, + const ccf::http::HeaderMap& headers, + ccf::http::HeaderMap&& trailers, std::span body) { LOG_TRACE_FMT( @@ -348,13 +350,13 @@ namespace http2 stream_data->outgoing.state = StreamResponseState::Closing; - http::HeaderMap extra_headers = {}; - extra_headers[http::headers::CONTENT_LENGTH] = + ccf::http::HeaderMap extra_headers = {}; + extra_headers[ccf::http::headers::CONTENT_LENGTH] = std::to_string(body.size()); auto thv = make_trailer_header_value(trailers); if (thv.has_value()) { - extra_headers[http::headers::TRAILER] = thv.value(); + extra_headers[ccf::http::headers::TRAILER] = thv.value(); } stream_data->outgoing.body = DataSource(body); @@ -371,7 +373,9 @@ namespace http2 } void start_stream( - StreamId stream_id, http_status status, const http::HeaderMap& headers) + StreamId stream_id, + http_status status, + const ccf::http::HeaderMap& headers) { LOG_TRACE_FMT( "http2::start_stream: stream {} - {} headers", @@ -427,7 +431,7 @@ namespace http2 send_all_submitted(); } - void close_stream(StreamId stream_id, http::HeaderMap&& trailers) + void close_stream(StreamId stream_id, ccf::http::HeaderMap&& trailers) { LOG_TRACE_FMT( "http2::close: stream {} - {} trailers ", stream_id, trailers.size()); @@ -467,7 +471,7 @@ namespace http2 std::string url = {}; { - const auto url_it = headers.find(http2::headers::PATH); + const auto url_it = headers.find(ccf::http2::headers::PATH); if (url_it != headers.end()) { url = url_it->second; @@ -476,7 +480,7 @@ namespace http2 llhttp_method method = {}; { - const auto method_it = headers.find(http2::headers::METHOD); + const auto method_it = headers.find(ccf::http2::headers::METHOD); if (method_it != headers.end()) { method = ccf::http_method_from_str(method_it->second.c_str()); @@ -495,24 +499,24 @@ namespace http2 class ClientParser : public Parser { private: - http::ResponseProcessor& proc; + ::http::ResponseProcessor& proc; public: - ClientParser(http::ResponseProcessor& proc_) : - Parser(http::ParserConfiguration{}, true), + ClientParser(::http::ResponseProcessor& proc_) : + Parser(ccf::http::ParserConfiguration{}, true), proc(proc_) {} void send_structured_request( llhttp_method method, const std::string& route, - const http::HeaderMap& headers, + const ccf::http::HeaderMap& headers, std::span body) { std::vector hdrs; hdrs.emplace_back( - make_nv(http2::headers::METHOD, llhttp_method_name(method))); - hdrs.emplace_back(make_nv(http2::headers::PATH, route.data())); + make_nv(ccf::http2::headers::METHOD, llhttp_method_name(method))); + hdrs.emplace_back(make_nv(ccf::http2::headers::PATH, route.data())); hdrs.emplace_back(make_nv(":scheme", "https")); hdrs.emplace_back(make_nv(":authority", "localhost:8080")); for (auto const& [k, v] : headers) @@ -557,7 +561,7 @@ namespace http2 http_status status = {}; { - const auto status_it = headers.find(http2::headers::STATUS); + const auto status_it = headers.find(ccf::http2::headers::STATUS); if (status_it != headers.end()) { status = http_status(std::stoi(status_it->second)); diff --git a/src/http/http2_session.h b/src/http/http2_session.h index 28d880deaa4d..1374ee44e05b 100644 --- a/src/http/http2_session.h +++ b/src/http/http2_session.h @@ -98,7 +98,7 @@ namespace http {} }; - class HTTP2StreamResponder : public http::HTTPResponder + class HTTP2StreamResponder : public ccf::http::HTTPResponder { private: http2::StreamId stream_id; @@ -118,8 +118,8 @@ namespace http bool send_response( http_status status_code, - http::HeaderMap&& headers, - http::HeaderMap&& trailers, + ccf::http::HeaderMap&& headers, + ccf::http::HeaderMap&& trailers, std::span body) override { auto sp = server_parser.lock(); @@ -143,7 +143,7 @@ namespace http } bool start_stream( - http_status status, const http::HeaderMap& headers) override + http_status status, const ccf::http::HeaderMap& headers) override { auto sp = server_parser.lock(); if (sp) @@ -166,7 +166,7 @@ namespace http return true; } - bool close_stream(http::HeaderMap&& trailers) override + bool close_stream(ccf::http::HeaderMap&& trailers) override { auto sp = server_parser.lock(); if (sp) @@ -214,7 +214,8 @@ namespace http return true; } - bool set_on_stream_close_callback(StreamOnCloseCallback cb) override + bool set_on_stream_close_callback( + ccf::http::StreamOnCloseCallback cb) override { auto sp = server_parser.lock(); if (sp) @@ -243,7 +244,7 @@ namespace http class HTTP2ServerSession : public HTTP2Session, public http::RequestProcessor, - public http::HTTPResponder + public ccf::http::HTTPResponder { private: std::shared_ptr server_parser; @@ -295,9 +296,9 @@ namespace http ccf::ODataError{std::move(error.code), std::move(error.msg)}}; const auto s = body.dump(); - http::HeaderMap headers; - headers[http::headers::CONTENT_TYPE] = - http::headervalues::contenttype::JSON; + ccf::http::HeaderMap headers; + headers[ccf::http::headers::CONTENT_TYPE] = + ccf::http::headervalues::contenttype::JSON; get_stream_responder(stream_id)->send_response( error.status, @@ -313,7 +314,7 @@ namespace http const ccf::ListenInterfaceID& interface_id, ringbuffer::AbstractWriterFactory& writer_factory, std::unique_ptr ctx, - const http::ParserConfiguration& configuration, + const ccf::http::ParserConfiguration& configuration, const std::shared_ptr& error_reporter, http::ResponderLookup& responder_lookup_) : HTTP2Session(session_id_, writer_factory, std::move(ctx), error_reporter), @@ -403,7 +404,7 @@ namespace http void handle_request( llhttp_method verb, const std::string_view& url, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body, int32_t stream_id) override { @@ -474,8 +475,8 @@ namespace http bool send_response( http_status status_code, - http::HeaderMap&& headers, - http::HeaderMap&& trailers, + ccf::http::HeaderMap&& headers, + ccf::http::HeaderMap&& trailers, std::span body) override { return get_stream_responder(http2::DEFAULT_STREAM_ID) @@ -484,7 +485,7 @@ namespace http } bool start_stream( - http_status status, const http::HeaderMap& headers) override + http_status status, const ccf::http::HeaderMap& headers) override { return get_stream_responder(http2::DEFAULT_STREAM_ID) ->start_stream(status, headers); @@ -495,13 +496,14 @@ namespace http return get_stream_responder(http2::DEFAULT_STREAM_ID)->stream_data(data); } - bool close_stream(http::HeaderMap&& trailers) override + bool close_stream(ccf::http::HeaderMap&& trailers) override { return get_stream_responder(http2::DEFAULT_STREAM_ID) ->close_stream(std::move(trailers)); } - bool set_on_stream_close_callback(StreamOnCloseCallback cb) override + bool set_on_stream_close_callback( + ccf::http::StreamOnCloseCallback cb) override { return get_stream_responder(http2::DEFAULT_STREAM_ID) ->set_on_stream_close_callback(cb); @@ -510,7 +512,7 @@ namespace http class HTTP2ClientSession : public HTTP2Session, public ccf::ClientSession, - public http::ResponseProcessor + public ::http::ResponseProcessor { private: http2::ClientParser client_parser; @@ -564,7 +566,7 @@ namespace http void handle_response( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body) override { handle_data_cb(status, std::move(headers), std::move(body)); diff --git a/src/http/http2_types.h b/src/http/http2_types.h index 9e0221cfab19..2a72a52b97eb 100644 --- a/src/http/http2_types.h +++ b/src/http/http2_types.h @@ -21,7 +21,7 @@ namespace http2 using StreamId = int32_t; constexpr static StreamId DEFAULT_STREAM_ID = 0; - using StreamCloseCB = http::StreamOnCloseCallback; + using StreamCloseCB = ccf::http::StreamOnCloseCallback; // Used to keep track of response state between nghttp2 callbacks and to // differentiate unary from streaming responses @@ -65,7 +65,7 @@ namespace http2 { struct Incoming { - http::HeaderMap headers; + ccf::http::HeaderMap headers; std::vector body; }; Incoming incoming; @@ -91,6 +91,6 @@ namespace http2 virtual std::shared_ptr get_stream(StreamId stream_id) = 0; virtual void destroy_stream(StreamId stream_id) = 0; virtual StreamId get_last_stream_id() const = 0; - virtual http::ParserConfiguration get_configuration() const = 0; + virtual ccf::http::ParserConfiguration get_configuration() const = 0; }; } \ No newline at end of file diff --git a/src/http/http2_utils.h b/src/http/http2_utils.h index 35b6d606b3c6..d8b94e88c8d3 100644 --- a/src/http/http2_utils.h +++ b/src/http/http2_utils.h @@ -33,14 +33,16 @@ namespace http2 } static inline std::optional make_trailer_header_value( - const http::HeaderMap& trailers) + const ccf::http::HeaderMap& trailers) { if (trailers.empty()) { return std::nullopt; } - using HeaderKeysIt = nonstd::KeyIterator; + using HeaderKeysIt = + ccf::nonstd::KeyIterator; + const auto trailer_header_val = fmt::format( "{}", fmt::join( diff --git a/src/http/http_accept.h b/src/http/http_accept.h index 39476ab91d56..496f463353dc 100644 --- a/src/http/http_accept.h +++ b/src/http/http_accept.h @@ -24,7 +24,7 @@ namespace http bool matches(const std::string& mime) const { - const auto [t, st] = nonstd::split_1(mime, "/"); + const auto [t, st] = ccf::nonstd::split_1(mime, "/"); if (is_wildcard(mime_type) || mime_type == t) { @@ -88,11 +88,11 @@ namespace http std::vector fields; - const auto elements = nonstd::split(s, ","); + const auto elements = ccf::nonstd::split(s, ","); for (const auto& element : elements) { - const auto [types, q_string] = nonstd::split_1(element, ";q="); - const auto [type, subtype] = nonstd::split_1(types, "/"); + const auto [types, q_string] = ccf::nonstd::split_1(element, ";q="); + const auto [type, subtype] = ccf::nonstd::split_1(types, "/"); if (type.empty() || subtype.empty()) { throw ccf::RpcException( diff --git a/src/http/http_builder.h b/src/http/http_builder.h index c2d6c053b4f8..3ce285749152 100644 --- a/src/http/http_builder.h +++ b/src/http/http_builder.h @@ -16,7 +16,7 @@ namespace http { - static std::string get_header_string(const HeaderMap& headers) + static std::string get_header_string(const ccf::http::HeaderMap& headers) { std::string header_string; for (const auto& [k, v] : headers) @@ -30,14 +30,14 @@ namespace http class Message { protected: - HeaderMap headers; + ccf::http::HeaderMap headers; const uint8_t* body = nullptr; size_t body_size = 0; Message() = default; public: - const HeaderMap& get_headers() const + const ccf::http::HeaderMap& get_headers() const { return headers; } @@ -45,7 +45,7 @@ namespace http void set_header(std::string k, const std::string& v) { // Store all headers lower-cased to simplify case-insensitive lookup - nonstd::to_lower(k); + ccf::nonstd::to_lower(k); headers[k] = v; } @@ -88,7 +88,7 @@ namespace http body = b; body_size = s; - headers[headers::CONTENT_LENGTH] = + headers[ccf::http::headers::CONTENT_LENGTH] = fmt::format("{}", get_content_length()); } @@ -97,7 +97,7 @@ namespace http body = (uint8_t*)s.data(); body_size = s.size(); - headers[headers::CONTENT_LENGTH] = + headers[ccf::http::headers::CONTENT_LENGTH] = fmt::format("{}", get_content_length()); } }; diff --git a/src/http/http_jwt.h b/src/http/http_jwt.h index 304f6866a18b..aecf1c71074e 100644 --- a/src/http/http_jwt.h +++ b/src/http/http_jwt.h @@ -62,11 +62,11 @@ namespace http return false; } auto auth_scheme = auth_header_value.substr(0, next_space); - if (auth_scheme != auth::BEARER_AUTH_SCHEME) + if (auth_scheme != ccf::http::auth::BEARER_AUTH_SCHEME) { error_reason = fmt::format( "Authorization header does not have {} scheme", - auth::BEARER_AUTH_SCHEME); + ccf::http::auth::BEARER_AUTH_SCHEME); return false; } auth_header_value = auth_header_value.substr(next_space + 1); @@ -101,9 +101,9 @@ namespace http std::string_view payload_b64url = token.substr(first_dot + 1, payload_size); std::string_view signature_b64url = token.substr(second_dot + 1); - auto header_raw = crypto::raw_from_b64url(header_b64url); - auto payload_raw = crypto::raw_from_b64url(payload_b64url); - auto signature_raw = crypto::raw_from_b64url(signature_b64url); + auto header_raw = ccf::crypto::raw_from_b64url(header_b64url); + auto payload_raw = ccf::crypto::raw_from_b64url(payload_b64url); + auto signature_raw = ccf::crypto::raw_from_b64url(signature_b64url); auto signed_content = token.substr(0, second_dot); nlohmann::json header; nlohmann::json payload; @@ -156,12 +156,13 @@ namespace http } static std::optional extract_token( - const http::HeaderMap& headers, std::string& error_reason) + const ccf::http::HeaderMap& headers, std::string& error_reason) { - const auto auth_it = headers.find(headers::AUTHORIZATION); + const auto auth_it = headers.find(ccf::http::headers::AUTHORIZATION); if (auth_it == headers.end()) { - error_reason = fmt::format("Missing {} header", headers::AUTHORIZATION); + error_reason = + fmt::format("Missing {} header", ccf::http::headers::AUTHORIZATION); return std::nullopt; } std::string_view token = auth_it->second; @@ -174,14 +175,14 @@ namespace http } static bool validate_token_signature( - const Token& token, const crypto::VerifierPtr& verifier) + const Token& token, const ccf::crypto::VerifierPtr& verifier) { return verifier->verify( (uint8_t*)token.signed_content.data(), token.signed_content.size(), token.signature.data(), token.signature.size(), - crypto::MDType::SHA256); + ccf::crypto::MDType::SHA256); } }; } diff --git a/src/http/http_parser.h b/src/http/http_parser.h index 6d2a8339ffdc..7094f60b7fc6 100644 --- a/src/http/http_parser.h +++ b/src/http/http_parser.h @@ -80,7 +80,7 @@ namespace http { llhttp_method method; std::string url; - http::HeaderMap headers; + ccf::http::HeaderMap headers; std::vector body; }; @@ -89,7 +89,7 @@ namespace http virtual void handle_request( llhttp_method method, const std::string_view& url, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body, int32_t) override { @@ -97,13 +97,13 @@ namespace http } }; - struct SimpleResponseProcessor : public http::ResponseProcessor + struct SimpleResponseProcessor : public ::http::ResponseProcessor { public: struct Response { http_status status; - http::HeaderMap headers; + ccf::http::HeaderMap headers; std::vector body; }; @@ -111,7 +111,7 @@ namespace http virtual void handle_response( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body) override { received.emplace(Response{status, headers, body}); @@ -183,11 +183,11 @@ namespace http protected: llhttp_t parser; llhttp_settings_t settings; - ParserConfiguration configuration; + ccf::http::ParserConfiguration configuration; State state = DONE; std::vector body_buf; - HeaderMap headers; + ccf::http::HeaderMap headers; std::pair partial_parsed_header = {}; @@ -200,7 +200,8 @@ namespace http Parser( llhttp_type_t type, - const ParserConfiguration& config = ParserConfiguration{}) : + const ccf::http::ParserConfiguration& config = + ccf::http::ParserConfiguration{}) : configuration(config) { llhttp_settings_init(&settings); @@ -245,8 +246,8 @@ namespace http LOG_TRACE_FMT("Appending chunk [{} bytes]", length); std::copy(at, at + length, std::back_inserter(body_buf)); - auto const& max_body_size = - configuration.max_body_size.value_or(default_max_body_size); + auto const& max_body_size = configuration.max_body_size.value_or( + ccf::http::default_max_body_size); if (body_buf.size() > max_body_size) { throw RequestPayloadTooLargeException(fmt::format( @@ -299,8 +300,8 @@ namespace http complete_header(); } - const auto max_headers_count = - configuration.max_headers_count.value_or(default_max_headers_count); + const auto max_headers_count = configuration.max_headers_count.value_or( + ccf::http::default_max_headers_count); if (headers.size() >= max_headers_count) { throw RequestHeaderTooLargeException(fmt::format( @@ -310,12 +311,12 @@ namespace http // HTTP headers are stored lowercase as it is easier to verify HTTP // signatures later on auto f = std::string(at, length); - nonstd::to_lower(f); + ccf::nonstd::to_lower(f); auto& partial_header_key = partial_parsed_header.first; partial_header_key.append(f); - auto const& max_header_size = - configuration.max_header_size.value_or(default_max_header_size); + auto const& max_header_size = configuration.max_header_size.value_or( + ccf::http::default_max_header_size); if (partial_header_key.size() > max_header_size) { throw RequestHeaderTooLargeException(fmt::format( @@ -329,8 +330,8 @@ namespace http { auto& partial_header_value = partial_parsed_header.second; partial_header_value.append(at, length); - auto const& max_header_size = - configuration.max_header_size.value_or(default_max_header_size); + auto const& max_header_size = configuration.max_header_size.value_or( + ccf::http::default_max_header_size); if (partial_header_value.size() > max_header_size) { throw RequestHeaderTooLargeException(fmt::format( @@ -399,7 +400,8 @@ namespace http public: RequestParser( RequestProcessor& proc_, - const ParserConfiguration& config = ParserConfiguration{}) : + const ccf::http::ParserConfiguration& config = + ccf::http::ParserConfiguration{}) : Parser(HTTP_REQUEST, config), proc(proc_) { @@ -453,7 +455,7 @@ namespace http public: ResponseParser(ResponseProcessor& proc_) : - Parser(HTTP_RESPONSE, ParserConfiguration{}), + Parser(HTTP_RESPONSE, ccf::http::ParserConfiguration{}), proc(proc_) {} diff --git a/src/http/http_proc.h b/src/http/http_proc.h index 371ea39cb199..efbfc4028112 100644 --- a/src/http/http_proc.h +++ b/src/http/http_proc.h @@ -22,7 +22,7 @@ namespace http virtual void handle_request( llhttp_method method, const std::string_view& url, - HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body, int32_t stream_id = http2::DEFAULT_STREAM_ID) = 0; }; @@ -31,6 +31,8 @@ namespace http { public: virtual void handle_response( - http_status status, HeaderMap&& headers, std::vector&& body) = 0; + http_status status, + ccf::http::HeaderMap&& headers, + std::vector&& body) = 0; }; } \ No newline at end of file diff --git a/src/http/http_rpc_context.h b/src/http/http_rpc_context.h index 2e98e303463d..0d8ca9c08224 100644 --- a/src/http/http_rpc_context.h +++ b/src/http/http_rpc_context.h @@ -18,10 +18,11 @@ namespace http const auto s = body.dump(); std::vector data(s.begin(), s.end()); - auto response = http::Response(error.status); + auto response = ::http::Response(error.status); response.set_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); response.set_body(&data); return response.build_response(); @@ -44,16 +45,16 @@ namespace http std::string query = {}; std::string fragment = {}; - http::HeaderMap request_headers = {}; + ccf::http::HeaderMap request_headers = {}; std::vector request_body = {}; - std::shared_ptr responder = nullptr; + std::shared_ptr responder = nullptr; std::vector serialised_request = {}; - http::HeaderMap response_headers; - http::HeaderMap response_trailers; + ccf::http::HeaderMap response_headers; + ccf::http::HeaderMap response_trailers; std::vector response_body = {}; http_status response_status = HTTP_STATUS_OK; @@ -71,7 +72,7 @@ namespace http "\r\n", verb.c_str(), url, - http::get_header_string(request_headers)); + ::http::get_header_string(request_headers)); serialised_request.resize(request_prefix.size() + request_body.size()); ::memcpy( @@ -96,9 +97,9 @@ namespace http ccf::HttpVersion http_version, llhttp_method verb_, const std::string_view& url_, - const http::HeaderMap& headers_, + const ccf::http::HeaderMap& headers_, const std::vector& body_, - const std::shared_ptr& responder_ = nullptr, + const std::shared_ptr& responder_ = nullptr, const std::vector& raw_request_ = {}) : RpcContextImpl(s, http_version), verb(verb_), @@ -120,12 +121,12 @@ namespace http } } - http::HeaderMap get_response_headers() const + ccf::http::HeaderMap get_response_headers() const { return response_headers; } - http::HeaderMap get_response_trailers() const + ccf::http::HeaderMap get_response_trailers() const { return response_trailers; } @@ -147,7 +148,7 @@ namespace http virtual void set_tx_id(const ccf::TxID& tx_id) override { - set_response_header(http::headers::CCF_TX_ID, tx_id.to_str()); + set_response_header(ccf::http::headers::CCF_TX_ID, tx_id.to_str()); } virtual const std::vector& get_request_body() const override @@ -186,7 +187,7 @@ namespace http path = p; } - virtual const http::HeaderMap& get_request_headers() const override + virtual const ccf::http::HeaderMap& get_request_headers() const override { return request_headers; } @@ -208,7 +209,8 @@ namespace http return url; } - virtual std::shared_ptr get_responder() const override + virtual std::shared_ptr get_responder() + const override { return responder; } @@ -286,7 +288,7 @@ namespace http virtual std::vector serialise_response() const override { - auto http_response = http::Response(response_status); + auto http_response = ::http::Response(response_status); for (const auto& [k, v] : response_headers) { @@ -370,11 +372,11 @@ namespace http namespace ccf { - inline std::shared_ptr make_rpc_context( + inline std::shared_ptr<::http::HttpRpcContext> make_rpc_context( std::shared_ptr s, const std::vector& packed) { - http::SimpleRequestProcessor processor; - http::RequestParser parser(processor, http::permissive_configuration()); + ::http::SimpleRequestProcessor processor; + ::http::RequestParser parser(processor, http::permissive_configuration()); parser.execute(packed.data(), packed.size()); if (processor.received.size() != 1) @@ -387,7 +389,7 @@ namespace ccf const auto& msg = processor.received.front(); - return std::make_shared( + return std::make_shared<::http::HttpRpcContext>( s, ccf::HttpVersion::HTTP1, msg.method, @@ -398,7 +400,7 @@ namespace ccf packed); } - inline std::shared_ptr make_fwd_rpc_context( + inline std::shared_ptr<::http::HttpRpcContext> make_fwd_rpc_context( std::shared_ptr s, const std::vector& packed, ccf::FrameFormat frame_format) diff --git a/src/http/http_session.h b/src/http/http_session.h index 60e108c044f1..c044736fe816 100644 --- a/src/http/http_session.h +++ b/src/http/http_session.h @@ -86,7 +86,7 @@ namespace http class HTTPServerSession : public HTTPSession, public http::RequestProcessor, - public http::HTTPResponder + public ccf::http::HTTPResponder { private: http::RequestParser request_parser; @@ -103,7 +103,7 @@ namespace http const ccf::ListenInterfaceID& interface_id, ringbuffer::AbstractWriterFactory& writer_factory, std::unique_ptr ctx, - const http::ParserConfiguration& configuration, + const ccf::http::ParserConfiguration& configuration, const std::shared_ptr& error_reporter = nullptr) : HTTPSession(session_id_, writer_factory, std::move(ctx), error_reporter), request_parser(*this, configuration), @@ -160,9 +160,9 @@ namespace http } LOG_DEBUG_FMT("Error parsing HTTP request: {}", e.what()); - http::HeaderMap headers; - headers[http::headers::CONTENT_TYPE] = - http::headervalues::contenttype::TEXT; + ccf::http::HeaderMap headers; + headers[ccf::http::headers::CONTENT_TYPE] = + ccf::http::headervalues::contenttype::TEXT; // NB: Avoid formatting input data a string, as it may contain null // bytes. Instead insert it at the end of this message, verbatim @@ -189,7 +189,7 @@ namespace http void handle_request( llhttp_method verb, const std::string_view& url, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body, int32_t) override { @@ -269,8 +269,8 @@ namespace http bool send_response( http_status status_code, - http::HeaderMap&& headers, - http::HeaderMap&& trailers, + ccf::http::HeaderMap&& headers, + ccf::http::HeaderMap&& trailers, std::span body) override { if (!trailers.empty()) @@ -278,7 +278,7 @@ namespace http throw std::logic_error("Cannot return trailers over HTTP/1"); } - auto response = http::Response(status_code); + auto response = ::http::Response(status_code); for (const auto& [k, v] : headers) { response.set_header(k, v); @@ -291,7 +291,7 @@ namespace http } bool start_stream( - http_status status, const http::HeaderMap& headers) override + http_status status, const ccf::http::HeaderMap& headers) override { throw std::logic_error("Not implemented!"); } @@ -301,12 +301,13 @@ namespace http throw std::logic_error("Not implemented!"); } - bool close_stream(http::HeaderMap&&) override + bool close_stream(ccf::http::HeaderMap&&) override { throw std::logic_error("Not implemented!"); } - bool set_on_stream_close_callback(StreamOnCloseCallback cb) override + bool set_on_stream_close_callback( + ccf::http::StreamOnCloseCallback cb) override { throw std::logic_error("Not implemented!"); } @@ -314,10 +315,10 @@ namespace http class HTTPClientSession : public HTTPSession, public ccf::ClientSession, - public http::ResponseProcessor + public ::http::ResponseProcessor { private: - http::ResponseParser response_parser; + ::http::ResponseParser response_parser; public: HTTPClientSession( @@ -380,7 +381,7 @@ namespace http void handle_response( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body) override { handle_data_cb(status, std::move(headers), std::move(body)); @@ -452,10 +453,10 @@ namespace http class UnencryptedHTTPClientSession : public UnencryptedHTTPSession, public ccf::ClientSession, - public http::ResponseProcessor + public ::http::ResponseProcessor { private: - http::ResponseParser response_parser; + ::http::ResponseParser response_parser; public: UnencryptedHTTPClientSession( @@ -504,7 +505,7 @@ namespace http void handle_response( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& body) override { handle_data_cb(status, std::move(headers), std::move(body)); diff --git a/src/http/responder_lookup.h b/src/http/responder_lookup.h index 688ef8f5a5f5..5eb7b089485e 100644 --- a/src/http/responder_lookup.h +++ b/src/http/responder_lookup.h @@ -13,8 +13,8 @@ namespace http class ResponderLookup { protected: - using ByStream = - std::unordered_map>; + using ByStream = std:: + unordered_map>; std::unordered_map<::tls::ConnID, ByStream> all_responders; @@ -22,7 +22,7 @@ namespace http ccf::pal::Mutex lock; public: - std::shared_ptr lookup_responder( + std::shared_ptr lookup_responder( ::tls::ConnID session_id, http2::StreamId stream_id) { std::unique_lock guard(lock); @@ -43,7 +43,7 @@ namespace http void add_responder( ::tls::ConnID session_id, http2::StreamId stream_id, - std::shared_ptr responder) + std::shared_ptr responder) { std::unique_lock guard(lock); all_responders[session_id][stream_id] = responder; diff --git a/src/http/test/http_test.cpp b/src/http/test/http_test.cpp index b3e93f355fb7..202111459e38 100644 --- a/src/http/test/http_test.cpp +++ b/src/http/test/http_test.cpp @@ -26,7 +26,7 @@ std::vector s_to_v(char const* s) std::string to_lowercase(std::string s) { - nonstd::to_lower(s); + ccf::nonstd::to_lower(s); return s; } @@ -62,10 +62,10 @@ DOCTEST_TEST_CASE("Complete response") { const std::vector r = {0, 1, 2, 3}; - http::SimpleResponseProcessor sp; - http::ResponseParser p(sp); + ::http::SimpleResponseProcessor sp; + ::http::ResponseParser p(sp); - auto response = http::Response(status); + auto response = ::http::Response(status); response.set_body(&r); auto res = response.build_response(); p.execute(res.data(), res.size()); @@ -277,17 +277,17 @@ DOCTEST_TEST_CASE("URL parsing") DOCTEST_TEST_CASE("Pessimal transport") { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - const http::HeaderMap h1 = {{"foo", "bar"}, {"baz", "42"}}; - const http::HeaderMap h2 = { + const ccf::http::HeaderMap h1 = {{"foo", "bar"}, {"baz", "42"}}; + const ccf::http::HeaderMap h2 = { {"foo", "barbar"}, {"content-type", "application/json"}, {"x-custom-header", "custom user data"}, {"x-MixedCASE", "DontCARE"}}; - http::SimpleRequestProcessor sp; - http::RequestParser p(sp); + ::http::SimpleRequestProcessor sp; + ::http::RequestParser p(sp); // Use the same processor and test repeatedly to make sure headers are for // only the current request @@ -487,7 +487,7 @@ DOCTEST_TEST_CASE("Query parser") "&multi=maintains-order!&multi=twice&multi=2&multi=three&multi=1&multi=" "twice"; - const auto parsed = http::parse_query(query); + const auto parsed = ccf::http::parse_query(query); std::vector checked_keys; diff --git a/src/indexing/enclave_lfs_access.h b/src/indexing/enclave_lfs_access.h index d7e18200f5f9..3bdfcf2bad78 100644 --- a/src/indexing/enclave_lfs_access.h +++ b/src/indexing/enclave_lfs_access.h @@ -27,12 +27,12 @@ namespace ccf::indexing { static inline bool verify_and_decrypt( - crypto::KeyAesGcm& encryption_key, + ccf::crypto::KeyAesGcm& encryption_key, const LFSKey& key, LFSEncryptedContents&& encrypted, std::vector& plaintext) { - crypto::GcmCipher gcm; + ccf::crypto::GcmCipher gcm; gcm.deserialise(encrypted); #ifdef PLAINTEXT_CACHE @@ -91,8 +91,8 @@ namespace ccf::indexing ringbuffer::WriterPtr to_host; - crypto::EntropyPtr entropy_src; - std::unique_ptr encryption_key; + ccf::crypto::EntropyPtr entropy_src; + std::unique_ptr encryption_key; LFSEncryptedContents encrypt(const LFSKey& key, LFSContents&& contents) { @@ -105,7 +105,7 @@ namespace ccf::indexing contents.insert(contents.begin(), key_prefix.begin(), key_prefix.end()); } - crypto::GcmCipher gcm(contents.size()); + ccf::crypto::GcmCipher gcm(contents.size()); // Use a random IV for each call gcm.hdr.set_random_iv(); @@ -123,12 +123,12 @@ namespace ccf::indexing public: EnclaveLFSAccess(const ringbuffer::WriterPtr& writer) : to_host(writer), - entropy_src(crypto::get_entropy()) + entropy_src(ccf::crypto::get_entropy()) { // Generate a fresh random key. Only this specific instance, in this // enclave, can read these files! - encryption_key = crypto::make_key_aes_gcm( - entropy_src->random(crypto::GCM_DEFAULT_KEY_SIZE)); + encryption_key = ccf::crypto::make_key_aes_gcm( + entropy_src->random(ccf::crypto::GCM_DEFAULT_KEY_SIZE)); } void register_message_handlers( @@ -242,7 +242,8 @@ namespace ccf::indexing #ifdef PLAINTEXT_CACHE return key; #else - const auto h = crypto::sha256((const uint8_t*)key.data(), key.size()); + const auto h = + ccf::crypto::sha256((const uint8_t*)key.data(), key.size()); return ds::to_hex(h); #endif } diff --git a/src/indexing/historical_transaction_fetcher.h b/src/indexing/historical_transaction_fetcher.h index 024f8f43a9f1..cf357d824c38 100644 --- a/src/indexing/historical_transaction_fetcher.h +++ b/src/indexing/historical_transaction_fetcher.h @@ -19,15 +19,15 @@ namespace ccf::indexing historical_cache(sc) {} - kv::ReadOnlyStorePtr deserialise_transaction( + ccf::kv::ReadOnlyStorePtr deserialise_transaction( ccf::SeqNo seqno, const uint8_t* data, size_t size) override { - kv::ApplyResult result; + ccf::kv::ApplyResult result; ccf::ClaimsDigest claims_digest; bool has_commit_evidence; auto store = historical_cache->deserialise_ledger_entry( seqno, data, size, result, claims_digest, has_commit_evidence); - if (store != nullptr && result != kv::ApplyResult::FAIL) + if (store != nullptr && result != ccf::kv::ApplyResult::FAIL) { return store; } @@ -39,7 +39,7 @@ namespace ccf::indexing return nullptr; } - std::vector fetch_transactions( + std::vector fetch_transactions( const SeqNoCollection& seqnos) override { const ccf::historical::CompoundHandle handle{ diff --git a/src/indexing/strategies/seqnos_by_key_bucketed.cpp b/src/indexing/strategies/seqnos_by_key_bucketed.cpp index 020ca9a31eef..2882b9e129a1 100644 --- a/src/indexing/strategies/seqnos_by_key_bucketed.cpp +++ b/src/indexing/strategies/seqnos_by_key_bucketed.cpp @@ -431,12 +431,14 @@ namespace ccf::indexing::strategies SeqnosByKey_Bucketed_Untyped::SeqnosByKey_Bucketed_Untyped( const std::string& map_name_, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, size_t seqnos_per_bucket_, size_t max_buckets_) : VisitEachEntryInMap(map_name_, "SeqnosByKey") { - if (kv::get_security_domain(map_name_) != kv::SecurityDomain::PUBLIC) + if ( + ccf::kv::get_security_domain(map_name_) != + ccf::kv::SecurityDomain::PUBLIC) { throw std::logic_error(fmt::format( "This Strategy ({}) is currently only implemented for public tables, " diff --git a/src/indexing/strategies/visit_each_entry_in_map.cpp b/src/indexing/strategies/visit_each_entry_in_map.cpp index b8ea9bc08e9a..7c947b6eb74d 100644 --- a/src/indexing/strategies/visit_each_entry_in_map.cpp +++ b/src/indexing/strategies/visit_each_entry_in_map.cpp @@ -14,12 +14,12 @@ namespace ccf::indexing::strategies {} void VisitEachEntryInMap::handle_committed_transaction( - const ccf::TxID& tx_id, const kv::ReadOnlyStorePtr& store) + const ccf::TxID& tx_id, const ccf::kv::ReadOnlyStorePtr& store) { // NB: Get an untyped view over the map with the same name. This saves // deserialisation here, where we hand on the raw key and value. auto tx = store->create_read_only_tx(); - auto handle = tx.ro(map_name); + auto handle = tx.ro(map_name); handle->foreach([this, &tx_id](const auto& k, const auto& v) { visit_entry(tx_id, k, v); diff --git a/src/indexing/test/common.h b/src/indexing/test/common.h index 37a24f5e7275..37dd75f3f88e 100644 --- a/src/indexing/test/common.h +++ b/src/indexing/test/common.h @@ -8,16 +8,16 @@ // Needed by TestTransactionFetcher #include "kv/test/null_encryptor.h" -using MapA = kv::Map; +using MapA = ccf::kv::Map; static MapA map_a("public:map_a"); -using MapB = kv::Map; +using MapB = ccf::kv::Map; static MapB map_b("public:map_b"); -using ValueA = kv::Value; +using ValueA = ccf::kv::Value; static ValueA value_a("public:value_a"); -using SetA = kv::Set; +using SetA = ccf::kv::Set; static SetA set_a("public:set_a"); static const std::chrono::milliseconds step_time(10); @@ -25,16 +25,16 @@ static const std::chrono::milliseconds step_time(10); class TestTransactionFetcher : public ccf::indexing::TransactionFetcher { public: - std::shared_ptr encryptor = - std::make_shared(); + std::shared_ptr encryptor = + std::make_shared(); ccf::SeqNoCollection requested; - std::unordered_map fetched_stores; + std::unordered_map fetched_stores; - kv::ReadOnlyStorePtr deserialise_transaction( + ccf::kv::ReadOnlyStorePtr deserialise_transaction( ccf::SeqNo seqno, const uint8_t* data, size_t size) { - auto store = std::make_shared( + auto store = std::make_shared( false /* Do not start from very first seqno */, true /* Make use of historical secrets */); @@ -48,7 +48,7 @@ class TestTransactionFetcher : public ccf::indexing::TransactionFetcher } auto result = exec->apply(); - if (result == kv::ApplyResult::FAIL) + if (result == ccf::kv::ApplyResult::FAIL) { return nullptr; } @@ -56,10 +56,10 @@ class TestTransactionFetcher : public ccf::indexing::TransactionFetcher return store; } - std::vector fetch_transactions( + std::vector fetch_transactions( const ccf::SeqNoCollection& seqnos) { - std::vector stores; + std::vector stores; for (auto seqno : seqnos) { @@ -87,11 +87,11 @@ class AllCommittableWrapper : public TConsensus public: using TConsensus::TConsensus; - bool replicate(const kv::BatchVector& entries_, ccf::View view) override + bool replicate(const ccf::kv::BatchVector& entries_, ccf::View view) override { // Rather than building a history that produces real signatures, we just // overwrite the entries here to say that everything is committable - kv::BatchVector entries(entries_); + ccf::kv::BatchVector entries(entries_); for (auto& [seqno, data, committable, hooks] : entries) { committable = true; @@ -101,7 +101,8 @@ class AllCommittableWrapper : public TConsensus } }; -using AllCommittableConsensus = AllCommittableWrapper; +using AllCommittableConsensus = + AllCommittableWrapper; using ExpectedSeqNos = std::set; @@ -161,7 +162,7 @@ static inline bool check_seqnos( return true; } -using Action = std::function; +using Action = std::function; struct ActionDesc { ExpectedSeqNos& expected; @@ -169,7 +170,7 @@ struct ActionDesc }; static inline bool create_transactions( - kv::Store& kv_store, + ccf::kv::Store& kv_store, const std::vector& actions, size_t count = ccf::indexing::Indexer::MAX_REQUESTABLE * 3) { @@ -185,7 +186,7 @@ static inline bool create_transactions( } } - if (tx.commit() != kv::CommitResult::SUCCESS) + if (tx.commit() != ccf::kv::CommitResult::SUCCESS) { return false; } diff --git a/src/indexing/test/indexing.cpp b/src/indexing/test/indexing.cpp index 65114b08ff44..e5aefd88f45e 100644 --- a/src/indexing/test/indexing.cpp +++ b/src/indexing/test/indexing.cpp @@ -30,8 +30,8 @@ using IndexB = ccf::indexing::strategies::SeqnosByKey_InMemory; constexpr size_t certificate_validity_period_days = 365; using namespace std::literals; auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); -auto valid_to = crypto::compute_cert_valid_to_string( + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); +auto valid_to = ccf::crypto::compute_cert_valid_to_string( valid_from, certificate_validity_period_days); static std::vector create_actions( @@ -41,11 +41,11 @@ static std::vector create_actions( ExpectedSeqNos& seqnos_2) { std::vector actions; - actions.push_back({seqnos_hello, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_hello, [](size_t i, ccf::kv::Tx& tx) { tx.wo(map_a)->put("hello", "value doesn't matter"); return true; }}); - actions.push_back({seqnos_saluton, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_saluton, [](size_t i, ccf::kv::Tx& tx) { if (i % 2 == 0) { tx.wo(map_a)->put("saluton", "value doesn't matter"); @@ -53,7 +53,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_1, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_1, [](size_t i, ccf::kv::Tx& tx) { if (i % 3 == 0) { tx.wo(map_b)->put(1, 42); @@ -61,7 +61,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_2, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_2, [](size_t i, ccf::kv::Tx& tx) { if (i % 4 == 0) { tx.wo(map_b)->put(2, 42); @@ -75,7 +75,7 @@ static std::vector create_actions( template void run_tests( const std::function& tick_until_caught_up, - kv::Store& kv_store, + ccf::kv::Store& kv_store, ccf::indexing::Indexer& indexer, ExpectedSeqNos& seqnos_hello, ExpectedSeqNos& seqnos_saluton, @@ -172,7 +172,7 @@ void run_tests( // Uses stub classes to test just indexing logic in isolation TEST_CASE("basic indexing" * doctest::test_suite("indexing")) { - kv::Store kv_store; + ccf::kv::Store kv_store; auto consensus = std::make_shared(); kv_store.set_consensus(consensus); @@ -180,7 +180,7 @@ TEST_CASE("basic indexing" * doctest::test_suite("indexing")) auto fetcher = std::make_shared(); ccf::indexing::Indexer indexer(fetcher); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); REQUIRE_THROWS(indexer.install_strategy(nullptr)); @@ -253,8 +253,8 @@ TEST_CASE("basic indexing" * doctest::test_suite("indexing")) index_b); } -kv::Version rekey( - kv::Store& kv_store, +ccf::kv::Version rekey( + ccf::kv::Store& kv_store, const std::shared_ptr& ledger_secrets) { ccf::ShareManager share_manager(ledger_secrets); @@ -262,7 +262,7 @@ kv::Version rekey( auto tx = kv_store.create_tx(); auto new_ledger_secret = ccf::make_ledger_secret(); share_manager.issue_recovery_shares(tx, new_ledger_secret); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_version = tx.commit_version(); @@ -275,7 +275,7 @@ kv::Version rekey( } aft::LedgerStubProxy* add_raft_consensus( - std::shared_ptr kv_store, + std::shared_ptr kv_store, std::shared_ptr indexer) { using TRaft = aft::Aft; @@ -285,7 +285,7 @@ aft::LedgerStubProxy* add_raft_consensus( const ccf::consensus::Configuration settings{{"20ms"}, {"100ms"}}; auto consensus = std::make_shared( settings, - std::make_unique>(kv_store), + std::make_unique>(kv_store), std::make_unique(node_id), std::make_shared(), std::make_shared(node_id), @@ -309,7 +309,7 @@ TEST_CASE_TEMPLATE( IndexA, LazyIndexA) { - auto kv_store_p = std::make_shared(); + auto kv_store_p = std::make_shared(); auto& kv_store = *kv_store_p; auto ledger_secrets = std::make_shared(); @@ -342,15 +342,15 @@ TEST_CASE_TEMPLATE( auto member_public_encryption_keys = tx.rw( ccf::Tables::MEMBER_ENCRYPTION_PUBLIC_KEYS); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto cert = kp->self_sign("CN=member", valid_from, valid_to); auto member_id = - crypto::Sha256Hash(crypto::cert_pem_to_der(cert)).hex_str(); + ccf::crypto::Sha256Hash(ccf::crypto::cert_pem_to_der(cert)).hex_str(); member_info->put(member_id, {ccf::MemberStatus::ACTIVE}); member_public_encryption_keys->put( - member_id, crypto::make_rsa_key_pair()->public_key_pem()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + member_id, ccf::crypto::make_rsa_key_pair()->public_key_pem()); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } ExpectedSeqNos seqnos_hello, seqnos_saluton, seqnos_1, seqnos_2; @@ -457,7 +457,7 @@ const auto max_multithread_run_time = 100s; TEST_CASE( "multi-threaded indexing - in memory" * doctest::test_suite("indexing")) { - auto kv_store_p = std::make_shared(); + auto kv_store_p = std::make_shared(); auto& kv_store = *kv_store_p; auto ledger_secrets = std::make_shared(); @@ -493,15 +493,15 @@ TEST_CASE( auto member_public_encryption_keys = tx.rw( ccf::Tables::MEMBER_ENCRYPTION_PUBLIC_KEYS); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto cert = kp->self_sign("CN=member", valid_from, valid_to); auto member_id = - crypto::Sha256Hash(crypto::cert_pem_to_der(cert)).hex_str(); + ccf::crypto::Sha256Hash(ccf::crypto::cert_pem_to_der(cert)).hex_str(); member_info->put(member_id, {ccf::MemberStatus::ACTIVE}); member_public_encryption_keys->put( - member_id, crypto::make_rsa_key_pair()->public_key_pem()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + member_id, ccf::crypto::make_rsa_key_pair()->public_key_pem()); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } std::atomic finished = false; @@ -510,7 +510,7 @@ TEST_CASE( std::atomic writes_to_42 = 0; auto tx_advancer = [&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); size_t i = 0; while (i < 1'000) { @@ -528,11 +528,11 @@ TEST_CASE( tx.wo(map_b)->put(42, i); } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); ++i; std::this_thread::yield(); } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); finished = true; }; @@ -572,7 +572,7 @@ TEST_CASE( std::atomic work_done = false; std::thread index_ticker([&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); while (!work_done) { size_t post_work_done_loops = 0; @@ -626,7 +626,7 @@ TEST_CASE( std::this_thread::yield(); } } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }); std::vector threads; @@ -661,19 +661,20 @@ TEST_CASE( class MockTransactionFetcher : public ccf::indexing::TransactionFetcher { - std::shared_ptr encryptor; + std::shared_ptr encryptor; public: aft::LedgerStubProxy* ledger; - MockTransactionFetcher(const std::shared_ptr& e) : + MockTransactionFetcher( + const std::shared_ptr& e) : encryptor(e) {} - kv::ReadOnlyStorePtr deserialise_transaction( + ccf::kv::ReadOnlyStorePtr deserialise_transaction( ccf::SeqNo seqno, const uint8_t* data, size_t size) override { - auto store = std::make_shared( + auto store = std::make_shared( false /* Do not start from very first seqno */, true /* Make use of historical secrets */); @@ -687,7 +688,7 @@ class MockTransactionFetcher : public ccf::indexing::TransactionFetcher } auto result = exec->apply(); - if (result == kv::ApplyResult::FAIL) + if (result == ccf::kv::ApplyResult::FAIL) { return nullptr; } @@ -695,10 +696,10 @@ class MockTransactionFetcher : public ccf::indexing::TransactionFetcher return store; } - std::vector fetch_transactions( + std::vector fetch_transactions( const ccf::SeqNoCollection& seqnos) override { - std::vector ret; + std::vector ret; for (const auto& seqno : seqnos) { @@ -723,7 +724,7 @@ TEST_CASE( INFO("Using seed: ", seed); srand(seed); - auto kv_store_p = std::make_shared(); + auto kv_store_p = std::make_shared(); auto& kv_store = *kv_store_p; auto ledger_secrets = std::make_shared(); @@ -755,7 +756,7 @@ TEST_CASE( std::make_shared(outbound_reader)); enclave_lfs->register_message_handlers(enclave_bp.get_dispatcher()); - ccfapp::AbstractNodeContext node_context; + ccf::AbstractNodeContext node_context; node_context.install_subsystem(enclave_lfs); using IndexA_Bucketed = @@ -782,15 +783,15 @@ TEST_CASE( auto member_public_encryption_keys = tx.rw( ccf::Tables::MEMBER_ENCRYPTION_PUBLIC_KEYS); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto cert = kp->self_sign("CN=member", valid_from, valid_to); auto member_id = - crypto::Sha256Hash(crypto::cert_pem_to_der(cert)).hex_str(); + ccf::crypto::Sha256Hash(ccf::crypto::cert_pem_to_der(cert)).hex_str(); member_info->put(member_id, {ccf::MemberStatus::ACTIVE}); member_public_encryption_keys->put( - member_id, crypto::make_rsa_key_pair()->public_key_pem()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + member_id, ccf::crypto::make_rsa_key_pair()->public_key_pem()); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } std::atomic all_submitted = false; @@ -799,7 +800,7 @@ TEST_CASE( std::atomic writes_to_42 = 0; auto tx_advancer = [&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); size_t i = 0; constexpr auto tx_count = #if NDEBUG @@ -824,17 +825,17 @@ TEST_CASE( tx.wo(map_b)->put(42, i); } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); ++i; std::this_thread::yield(); } all_submitted = true; - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; auto get_all = [&](const std::string& key) -> std::optional { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); const auto max_range = index_a->max_requestable_range(); auto range_start = 0; @@ -866,7 +867,7 @@ TEST_CASE( if (range_end == end_seqno) { - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return all_results; } else @@ -874,7 +875,7 @@ TEST_CASE( range_start = range_end + 1; } } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; auto fetch_index_a = [&]() { @@ -931,7 +932,7 @@ TEST_CASE( }); std::thread index_ticker([&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); while (!work_done) { while (indexer.update_strategies(step_time, kv_store.current_txid())) @@ -939,7 +940,7 @@ TEST_CASE( std::this_thread::yield(); } } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }); std::thread watchdog([&]() { @@ -967,11 +968,11 @@ TEST_CASE( int main(int argc, char** argv) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/indexing/test/lfs.cpp b/src/indexing/test/lfs.cpp index a14c7467eda9..2832224bdafc 100644 --- a/src/indexing/test/lfs.cpp +++ b/src/indexing/test/lfs.cpp @@ -42,11 +42,11 @@ static std::vector create_actions( ExpectedSeqNos& seqnos_value) { std::vector actions; - actions.push_back({seqnos_hello, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_hello, [](size_t i, ccf::kv::Tx& tx) { tx.wo(map_a)->put("hello", "value doesn't matter"); return true; }}); - actions.push_back({seqnos_saluton, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_saluton, [](size_t i, ccf::kv::Tx& tx) { if (i % 2 == 0) { tx.wo(map_a)->put("saluton", "value doesn't matter"); @@ -54,7 +54,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_1, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_1, [](size_t i, ccf::kv::Tx& tx) { if (i % 3 == 0) { tx.wo(map_b)->put(1, 42); @@ -62,7 +62,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_2, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_2, [](size_t i, ccf::kv::Tx& tx) { if (i % 4 == 0) { tx.wo(map_b)->put(2, 42); @@ -70,7 +70,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_set, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_set, [](size_t i, ccf::kv::Tx& tx) { if (i % 5 == 0) { tx.wo(set_a)->insert("set key"); @@ -78,7 +78,7 @@ static std::vector create_actions( } return false; }}); - actions.push_back({seqnos_value, [](size_t i, kv::Tx& tx) { + actions.push_back({seqnos_value, [](size_t i, ccf::kv::Tx& tx) { if (i % 6 == 0) { tx.wo(value_a)->put("value doesn't matter"); @@ -181,7 +181,7 @@ TEST_CASE("Basic cache" * doctest::test_suite("lfs")) TEST_CASE("Integrated cache" * doctest::test_suite("lfs")) { - kv::Store kv_store; + ccf::kv::Store kv_store; auto consensus = std::make_shared(); kv_store.set_consensus(consensus); @@ -189,7 +189,7 @@ TEST_CASE("Integrated cache" * doctest::test_suite("lfs")) auto fetcher = std::make_shared(); ccf::indexing::Indexer indexer(fetcher); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); messaging::BufferProcessor host_bp("lfs_host"); @@ -209,7 +209,7 @@ TEST_CASE("Integrated cache" * doctest::test_suite("lfs")) std::make_shared(outbound_reader)); enclave_lfs->register_message_handlers(enclave_bp.get_dispatcher()); - ccfapp::AbstractNodeContext node_context; + ccf::AbstractNodeContext node_context; node_context.install_subsystem(enclave_lfs); auto flush_ringbuffers = [&]() { @@ -394,7 +394,7 @@ TEST_CASE("Integrated cache" * doctest::test_suite("lfs")) auto index_b = std::make_shared(map_b, node_context, 100, 4); REQUIRE(indexer.install_strategy(index_b)); - kv::TxID current_ = kv_store.current_txid(); + ccf::kv::TxID current_ = kv_store.current_txid(); ccf::TxID current{current_.term, current_.version}; REQUIRE(index_a->get_indexed_watermark() == current); REQUIRE(index_b->get_indexed_watermark() == ccf::TxID()); @@ -505,7 +505,7 @@ TEST_CASE("Integrated cache" * doctest::test_suite("lfs")) void run_sparse_index_test(size_t bucket_size, size_t num_buckets) { - kv::Store kv_store; + ccf::kv::Store kv_store; auto consensus = std::make_shared(); kv_store.set_consensus(consensus); @@ -513,7 +513,7 @@ void run_sparse_index_test(size_t bucket_size, size_t num_buckets) auto fetcher = std::make_shared(); ccf::indexing::Indexer indexer(fetcher); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); messaging::BufferProcessor host_bp("lfs_host"); @@ -533,7 +533,7 @@ void run_sparse_index_test(size_t bucket_size, size_t num_buckets) std::make_shared(outbound_reader)); enclave_lfs->register_message_handlers(enclave_bp.get_dispatcher()); - ccfapp::AbstractNodeContext node_context; + ccf::AbstractNodeContext node_context; node_context.install_subsystem(enclave_lfs); auto flush_ringbuffers = [&]() { @@ -565,7 +565,7 @@ void run_sparse_index_test(size_t bucket_size, size_t num_buckets) { handle_b->put(k, k); } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto seqno = tx.get_txid()->version; for (const auto& k : keys) { @@ -580,7 +580,7 @@ void run_sparse_index_test(size_t bucket_size, size_t num_buckets) auto tx = kv_store.create_tx(); auto handle_a = tx.wo(map_a); handle_a->put("ignore", "ignore"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } }; diff --git a/src/indexing/transaction_fetcher_interface.h b/src/indexing/transaction_fetcher_interface.h index 6309d3be37ed..6743ab6b644f 100644 --- a/src/indexing/transaction_fetcher_interface.h +++ b/src/indexing/transaction_fetcher_interface.h @@ -12,10 +12,10 @@ namespace ccf::indexing public: virtual ~TransactionFetcher() = default; - virtual kv::ReadOnlyStorePtr deserialise_transaction( + virtual ccf::kv::ReadOnlyStorePtr deserialise_transaction( ccf::SeqNo seqno, const uint8_t* data, size_t size) = 0; - virtual std::vector fetch_transactions( + virtual std::vector fetch_transactions( const SeqNoCollection& seqnos) = 0; }; } diff --git a/src/js/core/context.cpp b/src/js/core/context.cpp index 7b2996eee387..cfd736bb051f 100644 --- a/src/js/core/context.cpp +++ b/src/js/core/context.cpp @@ -8,7 +8,6 @@ #include "ccf/js/core/wrapped_value.h" #include "ccf/js/extensions/console.h" #include "ccf/js/tx_access.h" -#include "ccf/pal/locking.h" #include "enclave/enclave_time.h" #include "js/ffi_plugins.h" #include "js/global_class_ids.h" diff --git a/src/js/extensions/ccf/converters.cpp b/src/js/extensions/ccf/converters.cpp index 41958eaf5aac..b53649e9e7bf 100644 --- a/src/js/extensions/ccf/converters.cpp +++ b/src/js/extensions/ccf/converters.cpp @@ -179,9 +179,9 @@ namespace ccf::js::extensions try { - auto pem = crypto::Pem(*pem_str); - auto der = crypto::make_verifier(pem)->cert_der(); - auto id = crypto::Sha256Hash(der).hex_str(); + auto pem = ccf::crypto::Pem(*pem_str); + auto der = ccf::crypto::make_verifier(pem)->cert_der(); + auto id = ccf::crypto::Sha256Hash(der).hex_str(); return JS_NewString(ctx, id.c_str()); } diff --git a/src/js/extensions/ccf/crypto.cpp b/src/js/extensions/ccf/crypto.cpp index e69044eb2976..abd1d935828e 100644 --- a/src/js/extensions/ccf/crypto.cpp +++ b/src/js/extensions/ccf/crypto.cpp @@ -40,7 +40,8 @@ namespace ccf::js::extensions try { - std::vector key = crypto::get_entropy()->random(key_size / 8); + std::vector key = + ccf::crypto::get_entropy()->random(key_size / 8); return JS_NewArrayBufferCopy(ctx, key.data(), key.size()); } catch (const std::exception& exc) @@ -68,16 +69,16 @@ namespace ccf::js::extensions return ccf::js::core::constants::Exception; } - std::shared_ptr k; + std::shared_ptr k; try { if (argc == 1) { - k = crypto::make_rsa_key_pair(key_size); + k = ccf::crypto::make_rsa_key_pair(key_size); } else { - k = crypto::make_rsa_key_pair(key_size, key_exponent); + k = ccf::crypto::make_rsa_key_pair(key_size, key_exponent); } } catch (const std::exception& exc) @@ -90,8 +91,8 @@ namespace ccf::js::extensions try { - crypto::Pem prv = k->private_key_pem(); - crypto::Pem pub = k->public_key_pem(); + ccf::crypto::Pem prv = k->private_key_pem(); + ccf::crypto::Pem pub = k->public_key_pem(); auto r = jsctx.new_obj(); JS_CHECK_EXC(r); @@ -126,18 +127,18 @@ namespace ccf::js::extensions return ccf::js::core::constants::Exception; } - crypto::CurveID cid; + ccf::crypto::CurveID cid; if (curve == "secp256r1") { - cid = crypto::CurveID::SECP256R1; + cid = ccf::crypto::CurveID::SECP256R1; } else if (curve == "secp256k1") { - cid = crypto::CurveID::SECP256K1; + cid = ccf::crypto::CurveID::SECP256K1; } else if (curve == "secp384r1") { - cid = crypto::CurveID::SECP384R1; + cid = ccf::crypto::CurveID::SECP384R1; } else { @@ -148,10 +149,10 @@ namespace ccf::js::extensions try { - auto k = crypto::make_key_pair(cid); + auto k = ccf::crypto::make_key_pair(cid); - crypto::Pem prv = k->private_key_pem(); - crypto::Pem pub = k->public_key_pem(); + ccf::crypto::Pem prv = k->private_key_pem(); + ccf::crypto::Pem pub = k->public_key_pem(); auto r = jsctx.new_obj(); JS_CHECK_EXC(r); @@ -186,14 +187,14 @@ namespace ccf::js::extensions return ccf::js::core::constants::Exception; } - crypto::CurveID cid; + ccf::crypto::CurveID cid; if (curve == "curve25519") { - cid = crypto::CurveID::CURVE25519; + cid = ccf::crypto::CurveID::CURVE25519; } else if (curve == "x25519") { - cid = crypto::CurveID::X25519; + cid = ccf::crypto::CurveID::X25519; } else { @@ -203,10 +204,10 @@ namespace ccf::js::extensions try { - auto k = crypto::make_eddsa_key_pair(cid); + auto k = ccf::crypto::make_eddsa_key_pair(cid); - crypto::Pem prv = k->private_key_pem(); - crypto::Pem pub = k->public_key_pem(); + ccf::crypto::Pem prv = k->private_key_pem(); + ccf::crypto::Pem pub = k->public_key_pem(); auto r = jsctx.new_obj(); JS_CHECK_EXC(r); @@ -256,7 +257,7 @@ namespace ccf::js::extensions try { - auto h = crypto::sha256(data, data_size); + auto h = ccf::crypto::sha256(data, data_size); return JS_NewArrayBufferCopy(ctx, h.data(), h.size()); } catch (const std::exception& exc) @@ -325,8 +326,8 @@ namespace ccf::js::extensions try { - auto chain_vec = crypto::split_x509_cert_bundle(*chain_str); - auto trusted_vec = crypto::split_x509_cert_bundle(*trusted_str); + auto chain_vec = ccf::crypto::split_x509_cert_bundle(*chain_str); + auto trusted_vec = ccf::crypto::split_x509_cert_bundle(*trusted_str); if (chain_vec.empty() || trusted_vec.empty()) { throw std::runtime_error( @@ -334,18 +335,18 @@ namespace ccf::js::extensions } auto& target_pem = chain_vec[0]; - std::vector chain_ptr; + std::vector chain_ptr; for (auto it = chain_vec.begin() + 1; it != chain_vec.end(); it++) { chain_ptr.push_back(&*it); } - std::vector trusted_ptr; + std::vector trusted_ptr; for (auto& pem : trusted_vec) { trusted_ptr.push_back(&pem); } - auto verifier = crypto::make_unique_verifier(target_pem); + auto verifier = ccf::crypto::make_unique_verifier(target_pem); if (!verifier->verify_certificate(trusted_ptr, chain_ptr)) { throw std::runtime_error("certificate chain is invalid"); @@ -395,39 +396,41 @@ namespace ccf::js::extensions T jwk; try { - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { - auto pubk = crypto::make_public_key(*pem_str); + auto pubk = ccf::crypto::make_public_key(*pem_str); jwk = pubk->public_key_jwk(kid); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto kp = crypto::make_key_pair(*pem_str); + auto kp = ccf::crypto::make_key_pair(*pem_str); jwk = kp->private_key_jwk(kid); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto pubk = crypto::make_rsa_public_key(*pem_str); + auto pubk = ccf::crypto::make_rsa_public_key(*pem_str); jwk = pubk->public_key_jwk_rsa(kid); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto kp = crypto::make_rsa_key_pair(*pem_str); + auto kp = ccf::crypto::make_rsa_key_pair(*pem_str); jwk = kp->private_key_jwk_rsa(kid); } - else if constexpr (std::is_same_v) + else if constexpr (std:: + is_same_v) { - auto pubk = crypto::make_eddsa_public_key(*pem_str); + auto pubk = ccf::crypto::make_eddsa_public_key(*pem_str); jwk = pubk->public_key_jwk_eddsa(kid); } - else if constexpr (std::is_same_v) + else if constexpr (std:: + is_same_v) { - auto kp = crypto::make_eddsa_key_pair(*pem_str); + auto kp = ccf::crypto::make_eddsa_key_pair(*pem_str); jwk = kp->private_key_jwk_eddsa(kid); } else { - static_assert(nonstd::dependent_false_v, "Unknown type"); + static_assert(ccf::nonstd::dependent_false_v, "Unknown type"); } } catch (const std::exception& ex) @@ -464,45 +467,47 @@ namespace ccf::js::extensions return ccf::js::core::constants::Exception; } - crypto::Pem pem; + ccf::crypto::Pem pem; try { T jwk = nlohmann::json::parse(jwk_str.value()); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { - auto pubk = crypto::make_public_key(jwk); + auto pubk = ccf::crypto::make_public_key(jwk); pem = pubk->public_key_pem(); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto kp = crypto::make_key_pair(jwk); + auto kp = ccf::crypto::make_key_pair(jwk); pem = kp->private_key_pem(); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto pubk = crypto::make_rsa_public_key(jwk); + auto pubk = ccf::crypto::make_rsa_public_key(jwk); pem = pubk->public_key_pem(); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { - auto kp = crypto::make_rsa_key_pair(jwk); + auto kp = ccf::crypto::make_rsa_key_pair(jwk); pem = kp->private_key_pem(); } - else if constexpr (std::is_same_v) + else if constexpr (std:: + is_same_v) { - auto pubk = crypto::make_eddsa_public_key(jwk); + auto pubk = ccf::crypto::make_eddsa_public_key(jwk); pem = pubk->public_key_pem(); } - else if constexpr (std::is_same_v) + else if constexpr (std:: + is_same_v) { - auto kp = crypto::make_eddsa_key_pair(jwk); + auto kp = ccf::crypto::make_eddsa_key_pair(jwk); pem = kp->private_key_pem(); } else { - static_assert(nonstd::dependent_false_v, "Unknown type"); + static_assert(ccf::nonstd::dependent_false_v, "Unknown type"); } } catch (const std::exception& ex) @@ -573,8 +578,8 @@ namespace ccf::js::extensions label_opt = {label_buf, label_buf + label_buf_size}; } - auto wrapped_key = crypto::ckm_rsa_pkcs_oaep_wrap( - crypto::Pem(wrapping_key, wrapping_key_size), + auto wrapped_key = ccf::crypto::ckm_rsa_pkcs_oaep_wrap( + ccf::crypto::Pem(wrapping_key, wrapping_key_size), {key, key + key_size}, label_opt); @@ -585,8 +590,8 @@ namespace ccf::js::extensions { std::vector privateKey( wrapping_key, wrapping_key + wrapping_key_size); - std::vector wrapped_key = - crypto::ckm_aes_key_wrap_pad(privateKey, {key, key + key_size}); + std::vector wrapped_key = ccf::crypto::ckm_aes_key_wrap_pad( + privateKey, {key, key + key_size}); OPENSSL_cleanse(privateKey.data(), privateKey.size()); @@ -618,9 +623,9 @@ namespace ccf::js::extensions label_opt = {label_buf, label_buf + label_buf_size}; } - auto wrapped_key = crypto::ckm_rsa_aes_key_wrap( + auto wrapped_key = ccf::crypto::ckm_rsa_aes_key_wrap( aes_key_size, - crypto::Pem(wrapping_key, wrapping_key_size), + ccf::crypto::Pem(wrapping_key, wrapping_key_size), {key, key + key_size}, label_opt); @@ -704,8 +709,8 @@ namespace ccf::js::extensions } auto pemPrivateUnwrappingKey = - crypto::Pem(unwrapping_key, unwrapping_key_size); - auto unwrapped_key = crypto::ckm_rsa_pkcs_oaep_unwrap( + ccf::crypto::Pem(unwrapping_key, unwrapping_key_size); + auto unwrapped_key = ccf::crypto::ckm_rsa_pkcs_oaep_unwrap( pemPrivateUnwrappingKey, {key, key + key_size}, label_opt); OPENSSL_cleanse( @@ -719,7 +724,8 @@ namespace ccf::js::extensions std::vector privateKey( unwrapping_key, unwrapping_key + unwrapping_key_size); std::vector unwrapped_key = - crypto::ckm_aes_key_unwrap_pad(privateKey, {key, key + key_size}); + ccf::crypto::ckm_aes_key_unwrap_pad( + privateKey, {key, key + key_size}); OPENSSL_cleanse(privateKey.data(), privateKey.size()); @@ -752,8 +758,8 @@ namespace ccf::js::extensions } auto privPemUnwrappingKey = - crypto::Pem(unwrapping_key, unwrapping_key_size); - auto unwrapped_key = crypto::ckm_rsa_aes_key_unwrap( + ccf::crypto::Pem(unwrapping_key, unwrapping_key_size); + auto unwrapped_key = ccf::crypto::ckm_rsa_aes_key_unwrap( privPemUnwrappingKey, {key, key + key_size}, label_opt); OPENSSL_cleanse( @@ -827,8 +833,8 @@ namespace ccf::js::extensions { try { - crypto::Pem key_pem(key); - auto key_pair = crypto::make_eddsa_key_pair(key_pem); + ccf::crypto::Pem key_pem(key); + auto key_pair = ccf::crypto::make_eddsa_key_pair(key_pem); auto sig = key_pair->sign(contents); return JS_NewArrayBufferCopy(ctx, sig.data(), sig.size()); } @@ -850,18 +856,18 @@ namespace ccf::js::extensions auto algo_name = *algo_name_str; auto algo_hash = *algo_hash_str; - crypto::MDType mdtype; + ccf::crypto::MDType mdtype; if (algo_hash == "SHA-256") { - mdtype = crypto::MDType::SHA256; + mdtype = ccf::crypto::MDType::SHA256; } else if (algo_hash == "SHA-384") { - mdtype = crypto::MDType::SHA384; + mdtype = ccf::crypto::MDType::SHA384; } else if (algo_hash == "SHA-512") { - mdtype = crypto::MDType::SHA512; + mdtype = ccf::crypto::MDType::SHA512; } else { @@ -872,22 +878,22 @@ namespace ccf::js::extensions if (algo_name == "ECDSA") { - auto key_pair = crypto::make_key_pair(key); + auto key_pair = ccf::crypto::make_key_pair(key); auto sig_der = key_pair->sign(contents, mdtype); - auto sig = - crypto::ecdsa_sig_der_to_p1363(sig_der, key_pair->get_curve_id()); + auto sig = ccf::crypto::ecdsa_sig_der_to_p1363( + sig_der, key_pair->get_curve_id()); return JS_NewArrayBufferCopy(ctx, sig.data(), sig.size()); } else if (algo_name == "RSASSA-PKCS1-v1_5") { - auto key_pair = crypto::make_rsa_key_pair(key); + auto key_pair = ccf::crypto::make_rsa_key_pair(key); auto sig = key_pair->sign(contents, mdtype); return JS_NewArrayBufferCopy(ctx, sig.data(), sig.size()); } else if (algo_name == "HMAC") { std::vector vkey(key.begin(), key.end()); - const auto sig = crypto::hmac(mdtype, vkey, contents); + const auto sig = ccf::crypto::hmac(mdtype, vkey, contents); return JS_NewArrayBufferCopy(ctx, sig.data(), sig.size()); } else @@ -911,7 +917,7 @@ namespace ccf::js::extensions size_t signature_size, const std::string& pub_key) { - auto public_key = crypto::make_eddsa_public_key(pub_key); + auto public_key = ccf::crypto::make_eddsa_public_key(pub_key); return public_key->verify( contents, contents_size, signature, signature_size); } @@ -993,10 +999,10 @@ namespace ccf::js::extensions auto algo_hash = *algo_hash_str; auto key = *key_str; - crypto::MDType mdtype; + ccf::crypto::MDType mdtype; if (algo_hash == "SHA-256") { - mdtype = crypto::MDType::SHA256; + mdtype = ccf::crypto::MDType::SHA256; } else { @@ -1016,7 +1022,7 @@ namespace ccf::js::extensions if (algo_name == "ECDSA") { - sig = crypto::ecdsa_sig_p1363_to_der(sig); + sig = ccf::crypto::ecdsa_sig_p1363_to_der(sig); } auto is_cert = key.starts_with("-----BEGIN CERTIFICATE"); @@ -1025,13 +1031,13 @@ namespace ccf::js::extensions if (is_cert) { - auto verifier = crypto::make_unique_verifier(key); + auto verifier = ccf::crypto::make_unique_verifier(key); valid = verifier->verify(data, data_size, sig.data(), sig.size(), mdtype); } else { - auto public_key = crypto::make_public_key(key); + auto public_key = ccf::crypto::make_public_key(key); valid = public_key->verify(data, data_size, sig.data(), sig.size(), mdtype); } @@ -1062,32 +1068,38 @@ namespace ccf::js::extensions crypto, "pubPemToJwk", JS_NewCFunction( - ctx, js_pem_to_jwk, "pubPemToJwk", 1)); + ctx, js_pem_to_jwk, "pubPemToJwk", 1)); JS_SetPropertyStr( ctx, crypto, "pemToJwk", JS_NewCFunction( - ctx, js_pem_to_jwk, "pemToJwk", 1)); + ctx, js_pem_to_jwk, "pemToJwk", 1)); JS_SetPropertyStr( ctx, crypto, "pubRsaPemToJwk", JS_NewCFunction( - ctx, js_pem_to_jwk, "pubRsaPemToJwk", 1)); + ctx, + js_pem_to_jwk, + "pubRsaPemToJwk", + 1)); JS_SetPropertyStr( ctx, crypto, "rsaPemToJwk", JS_NewCFunction( - ctx, js_pem_to_jwk, "rsaPemToJwk", 1)); + ctx, + js_pem_to_jwk, + "rsaPemToJwk", + 1)); JS_SetPropertyStr( ctx, crypto, "pubEddsaPemToJwk", JS_NewCFunction( ctx, - js_pem_to_jwk, + js_pem_to_jwk, "pubEddsaPemToJwk", 1)); JS_SetPropertyStr( @@ -1096,7 +1108,7 @@ namespace ccf::js::extensions "eddsaPemToJwk", JS_NewCFunction( ctx, - js_pem_to_jwk, + js_pem_to_jwk, "eddsaPemToJwk", 1)); JS_SetPropertyStr( @@ -1104,32 +1116,38 @@ namespace ccf::js::extensions crypto, "pubJwkToPem", JS_NewCFunction( - ctx, js_jwk_to_pem, "pubJwkToPem", 1)); + ctx, js_jwk_to_pem, "pubJwkToPem", 1)); JS_SetPropertyStr( ctx, crypto, "jwkToPem", JS_NewCFunction( - ctx, js_jwk_to_pem, "jwkToPem", 1)); + ctx, js_jwk_to_pem, "jwkToPem", 1)); JS_SetPropertyStr( ctx, crypto, "pubRsaJwkToPem", JS_NewCFunction( - ctx, js_jwk_to_pem, "pubRsaJwkToPem", 1)); + ctx, + js_jwk_to_pem, + "pubRsaJwkToPem", + 1)); JS_SetPropertyStr( ctx, crypto, "rsaJwkToPem", JS_NewCFunction( - ctx, js_jwk_to_pem, "rsaJwkToPem", 1)); + ctx, + js_jwk_to_pem, + "rsaJwkToPem", + 1)); JS_SetPropertyStr( ctx, crypto, "pubEddsaJwkToPem", JS_NewCFunction( ctx, - js_jwk_to_pem, + js_jwk_to_pem, "pubEddsaJwkToPem", 1)); JS_SetPropertyStr( @@ -1138,7 +1156,7 @@ namespace ccf::js::extensions "eddsaJwkToPem", JS_NewCFunction( ctx, - js_jwk_to_pem, + js_jwk_to_pem, "eddsaJwkToPem", 1)); JS_SetPropertyStr( diff --git a/src/js/extensions/ccf/historical.cpp b/src/js/extensions/ccf/historical.cpp index 406cdcdaed32..29531960ab5d 100644 --- a/src/js/extensions/ccf/historical.cpp +++ b/src/js/extensions/ccf/historical.cpp @@ -19,8 +19,8 @@ namespace ccf::js::extensions struct HistoricalHandle { ccf::historical::StatePtr state; - std::unique_ptr tx; - std::unordered_map + std::unique_ptr tx; + std::unordered_map kv_handles = {}; }; std::unordered_map historical_handles; @@ -172,7 +172,7 @@ namespace ccf::js::extensions std::string sig_b64; try { - sig_b64 = crypto::b64_from_raw(receipt_out.signature); + sig_b64 = ccf::crypto::b64_from_raw(receipt_out.signature); } catch (const std::exception& e) { @@ -305,7 +305,7 @@ namespace ccf::js::extensions if (hit->second == nullptr) { - kv::ReadOnlyTx* tx = it->second.tx.get(); + ccf::kv::ReadOnlyTx* tx = it->second.tx.get(); if (tx == nullptr) { LOG_FAIL_FMT("Can't rehydrate MapHandle - no transaction"); diff --git a/src/js/extensions/ccf/kv.cpp b/src/js/extensions/ccf/kv.cpp index 85f3d63a72fb..68cb664c2e30 100644 --- a/src/js/extensions/ccf/kv.cpp +++ b/src/js/extensions/ccf/kv.cpp @@ -17,10 +17,11 @@ namespace ccf::js::extensions { struct KvExtension::Impl { - kv::Tx* tx; - std::unordered_map kv_handles = {}; + ccf::kv::Tx* tx; + std::unordered_map kv_handles = + {}; - Impl(kv::Tx* t) : tx(t){}; + Impl(ccf::kv::Tx* t) : tx(t){}; }; namespace @@ -54,7 +55,7 @@ namespace ccf::js::extensions if (it->second == nullptr) { - kv::Tx* tx = extension->impl->tx; + ccf::kv::Tx* tx = extension->impl->tx; if (tx == nullptr) { LOG_FAIL_FMT("Can't rehydrate MapHandle - no transaction context"); @@ -129,7 +130,8 @@ namespace ccf::js::extensions } } - KvExtension::KvExtension(kv::Tx* t, const ccf::js::NamespaceRestriction& nr) : + KvExtension::KvExtension( + ccf::kv::Tx* t, const ccf::js::NamespaceRestriction& nr) : namespace_restriction(nr) { impl = std::make_unique(t); diff --git a/src/js/extensions/ccf/kv_helpers.h b/src/js/extensions/ccf/kv_helpers.h index c81224e26faf..c288c1510de8 100644 --- a/src/js/extensions/ccf/kv_helpers.h +++ b/src/js/extensions/ccf/kv_helpers.h @@ -8,7 +8,7 @@ namespace ccf::js::extensions::kvhelpers { - using KVMap = ::kv::untyped::Map; + using KVMap = ::ccf::kv::untyped::Map; using ROHandleGetter = KVMap::ReadOnlyHandle* (*)(js::core::Context& jsctx, JSValueConst this_val); diff --git a/src/js/extensions/ccf/network.cpp b/src/js/extensions/ccf/network.cpp index e8cf84c0464b..96ab920be0d5 100644 --- a/src/js/extensions/ccf/network.cpp +++ b/src/js/extensions/ccf/network.cpp @@ -85,10 +85,10 @@ namespace ccf::js::extensions { return ccf::js::core::constants::Exception; } - crypto::Pem csr; + ccf::crypto::Pem csr; try { - csr = crypto::Pem(*csr_cstr); + csr = ccf::crypto::Pem(*csr_cstr); } catch (const std::exception& e) { diff --git a/src/js/extensions/ccf/network.h b/src/js/extensions/ccf/network.h index e1cb42fbdc4b..f722c5c0044c 100644 --- a/src/js/extensions/ccf/network.h +++ b/src/js/extensions/ccf/network.h @@ -19,9 +19,9 @@ namespace ccf::js::extensions { public: ccf::NetworkState* network_state; - kv::Tx* tx; + ccf::kv::Tx* tx; - NetworkExtension(ccf::NetworkState* ns, kv::Tx* t) : + NetworkExtension(ccf::NetworkState* ns, ccf::kv::Tx* t) : network_state(ns), tx(t) {} diff --git a/src/js/extensions/ccf/node.cpp b/src/js/extensions/ccf/node.cpp index 592836c8da77..802c620e95c7 100644 --- a/src/js/extensions/ccf/node.cpp +++ b/src/js/extensions/ccf/node.cpp @@ -109,7 +109,7 @@ namespace ccf::js::extensions return JS_ThrowTypeError( ctx, "Previous service identity argument is not an array buffer"); } - identities.previous = crypto::Pem(prev_bytes, prev_bytes_sz); + identities.previous = ccf::crypto::Pem(prev_bytes, prev_bytes_sz); GOV_DEBUG_FMT( "previous service identity: {}", identities.previous->str()); } @@ -129,7 +129,7 @@ namespace ccf::js::extensions ctx, "Next service identity argument is not an array buffer"); } - identities.next = crypto::Pem(next_bytes, next_bytes_sz); + identities.next = ccf::crypto::Pem(next_bytes, next_bytes_sz); GOV_DEBUG_FMT("next service identity: {}", identities.next.str()); gov_effects->transition_service_to_open(*tx_ptr, identities); diff --git a/src/js/extensions/ccf/node.h b/src/js/extensions/ccf/node.h index 87272f21dcda..f19ff6169ab2 100644 --- a/src/js/extensions/ccf/node.h +++ b/src/js/extensions/ccf/node.h @@ -22,9 +22,9 @@ namespace ccf::js::extensions { public: ccf::AbstractGovernanceEffects* gov_effects; - kv::Tx* tx; + ccf::kv::Tx* tx; - NodeExtension(ccf::AbstractGovernanceEffects* ge, kv::Tx* t) : + NodeExtension(ccf::AbstractGovernanceEffects* ge, ccf::kv::Tx* t) : gov_effects(ge), tx(t) {} diff --git a/src/js/extensions/ccf/request.cpp b/src/js/extensions/ccf/request.cpp index f0ee8dbca59c..103c0c587813 100644 --- a/src/js/extensions/ccf/request.cpp +++ b/src/js/extensions/ccf/request.cpp @@ -125,8 +125,7 @@ namespace ccf::js::extensions dynamic_cast(ident.get())) { caller.set( - "policy", - ctx.new_string(ccfapp::get_policy_name_from_ident(jwt_ident))); + "policy", ctx.new_string(ccf::get_policy_name_from_ident(jwt_ident))); auto jwt = ctx.new_obj(); jwt.set( @@ -145,7 +144,7 @@ namespace ccf::js::extensions { caller.set( "policy", - ctx.new_string(ccfapp::get_policy_name_from_ident(empty_ident))); + ctx.new_string(ccf::get_policy_name_from_ident(empty_ident))); return caller; } if ( @@ -173,7 +172,7 @@ namespace ccf::js::extensions auto user_cert_ident = dynamic_cast(ident.get())) { - policy_name = ccfapp::get_policy_name_from_ident(user_cert_ident); + policy_name = ccf::get_policy_name_from_ident(user_cert_ident); id = user_cert_ident->user_id; is_member = false; } @@ -181,7 +180,7 @@ namespace ccf::js::extensions auto member_cert_ident = dynamic_cast(ident.get())) { - policy_name = ccfapp::get_policy_name_from_ident(member_cert_ident); + policy_name = ccf::get_policy_name_from_ident(member_cert_ident); id = member_cert_ident->member_id; is_member = true; } @@ -189,7 +188,7 @@ namespace ccf::js::extensions auto user_cose_ident = dynamic_cast(ident.get())) { - policy_name = ccfapp::get_policy_name_from_ident(user_cose_ident); + policy_name = ccf::get_policy_name_from_ident(user_cose_ident); id = user_cose_ident->user_id; is_member = false; @@ -225,7 +224,7 @@ namespace ccf::js::extensions fmt::format("Failed to get data for caller {}", id)); } - crypto::Pem cert; + ccf::crypto::Pem cert; if (is_member) { result = registry->get_member_cert_v1(endpoint_ctx.tx, id, cert); diff --git a/src/js/extensions/math/random.cpp b/src/js/extensions/math/random.cpp index b4714c8b339a..5f74c1706820 100644 --- a/src/js/extensions/math/random.cpp +++ b/src/js/extensions/math/random.cpp @@ -15,7 +15,7 @@ namespace ccf::js::extensions JSValue js_random_impl( JSContext* ctx, JSValueConst this_val, int argc, JSValueConst* argv) { - crypto::EntropyPtr entropy = crypto::get_entropy(); + ccf::crypto::EntropyPtr entropy = ccf::crypto::get_entropy(); // Generate a random 64 bit unsigned int, and transform that to a double // between 0 and 1. Note this is non-uniform, and not cryptographically diff --git a/src/js/no_plugins.cpp b/src/js/no_plugins.cpp index 5880db2d0f4b..d16663ec5ed8 100644 --- a/src/js/no_plugins.cpp +++ b/src/js/no_plugins.cpp @@ -5,7 +5,7 @@ #include -namespace ccfapp +namespace ccf { std::vector __attribute__((weak)) get_js_plugins() { diff --git a/src/js/permissions_checks.h b/src/js/permissions_checks.h index a9e39c4711a0..f0418262d708 100644 --- a/src/js/permissions_checks.h +++ b/src/js/permissions_checks.h @@ -17,11 +17,11 @@ namespace ccf::js // source of truth for these restrictions. This code is formatted to attempt // to make it clear how it maps directly to that table. const auto [privacy_of_table, namespace_of_table] = - kv::parse_map_name(table_name); + ccf::kv::parse_map_name(table_name); switch (privacy_of_table) { - case (kv::SecurityDomain::PRIVATE): + case (ccf::kv::SecurityDomain::PRIVATE): { // The only time private tables can be used, is on private application // tables in an application context. Governance should neither read from @@ -29,13 +29,13 @@ namespace ccf::js // tables exist then applications should not be able to read them. if ( execution_context == TxAccess::APP_RW && - namespace_of_table == kv::AccessCategory::APPLICATION) + namespace_of_table == ccf::kv::AccessCategory::APPLICATION) { return KVAccessPermissions::READ_WRITE; } else if ( execution_context == TxAccess::APP_RO && - namespace_of_table == kv::AccessCategory::APPLICATION) + namespace_of_table == ccf::kv::AccessCategory::APPLICATION) { return KVAccessPermissions::READ_ONLY; } @@ -45,16 +45,16 @@ namespace ccf::js } } - case (kv::SecurityDomain::PUBLIC): + case (ccf::kv::SecurityDomain::PUBLIC): { switch (namespace_of_table) { - case kv::AccessCategory::INTERNAL: + case ccf::kv::AccessCategory::INTERNAL: { return KVAccessPermissions::READ_ONLY; } - case kv::AccessCategory::GOVERNANCE: + case ccf::kv::AccessCategory::GOVERNANCE: { if (execution_context == TxAccess::GOV_RW) { @@ -66,7 +66,7 @@ namespace ccf::js } } - case kv::AccessCategory::APPLICATION: + case ccf::kv::AccessCategory::APPLICATION: { switch (execution_context) { @@ -87,7 +87,7 @@ namespace ccf::js } } - case (kv::SecurityDomain::SECURITY_DOMAIN_MAX): + case (ccf::kv::SecurityDomain::SECURITY_DOMAIN_MAX): { throw std::logic_error(fmt::format( "Unexpected security domain (max) for table {}", table_name)); diff --git a/src/js/registry.cpp b/src/js/registry.cpp index 1cffc532f590..1675c870c43d 100644 --- a/src/js/registry.cpp +++ b/src/js/registry.cpp @@ -17,6 +17,8 @@ #include // Custom Endpoints +#include "ccf/crypto/sha256.h" +#include "ccf/ds/hex.h" #include "ccf/endpoint.h" #include "ccf/endpoints/authentication/js.h" #include "ccf/historical_queries_adapter.h" @@ -107,7 +109,8 @@ namespace ccf::js // Extensions with a dependency on this endpoint context (invocation), // which must be removed after execution. - ccf::js::extensions::Extensions local_extensions; + ccf::js::extensions::Extensions local_extensions = + get_extensions(endpoint_ctx); // ccf.kv.* local_extensions.emplace_back( @@ -449,7 +452,7 @@ namespace ccf::js } DynamicJSEndpointRegistry::DynamicJSEndpointRegistry( - ccfapp::AbstractNodeContext& context, const std::string& kv_prefix) : + ccf::AbstractNodeContext& context, const std::string& kv_prefix) : ccf::UserEndpointRegistry(context), modules_map(fmt::format("{}.modules", kv_prefix)), metadata_map(fmt::format("{}.metadata", kv_prefix)), @@ -458,7 +461,10 @@ namespace ccf::js fmt::format("{}.modules_quickjs_version", kv_prefix)), modules_quickjs_bytecode_map( fmt::format("{}.modules_quickjs_bytecode", kv_prefix)), - runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)) + runtime_options_map(fmt::format("{}.runtime_options", kv_prefix)), + recent_actions_map(fmt::format("{}.recent_actions", kv_prefix)), + audit_input_map(fmt::format("{}.audit.input", kv_prefix)), + audit_info_map(fmt::format("{}.audit.info", kv_prefix)) { interpreter_cache = context.get_subsystem(); @@ -499,7 +505,7 @@ namespace ccf::js } ccf::ApiResult DynamicJSEndpointRegistry::install_custom_endpoints_v1( - kv::Tx& tx, const ccf::js::Bundle& bundle) + ccf::kv::Tx& tx, const ccf::js::Bundle& bundle) { try { @@ -511,7 +517,7 @@ namespace ccf::js for (const auto& [method, metadata] : methods) { std::string method_upper = method; - nonstd::to_upper(method_upper); + ccf::nonstd::to_upper(method_upper); const auto key = ccf::endpoints::EndpointKey{url, method_upper}; endpoints->put(key, metadata); } @@ -579,7 +585,7 @@ namespace ccf::js } ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoints_v1( - ccf::js::Bundle& bundle, kv::ReadOnlyTx& tx) + ccf::js::Bundle& bundle, ccf::kv::ReadOnlyTx& tx) { try { @@ -625,7 +631,7 @@ namespace ccf::js ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoint_properties_v1( ccf::endpoints::EndpointProperties& properties, - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const ccf::RESTVerb& verb, const ccf::endpoints::URI& uri) { @@ -653,7 +659,7 @@ namespace ccf::js } ccf::ApiResult DynamicJSEndpointRegistry::get_custom_endpoint_module_v1( - std::string& code, kv::ReadOnlyTx& tx, const std::string& module_name) + std::string& code, ccf::kv::ReadOnlyTx& tx, const std::string& module_name) { try { @@ -684,7 +690,7 @@ namespace ccf::js } ccf::ApiResult DynamicJSEndpointRegistry::set_js_runtime_options_v1( - kv::Tx& tx, const ccf::JSRuntimeOptions& options) + ccf::kv::Tx& tx, const ccf::JSRuntimeOptions& options) { try { @@ -698,7 +704,7 @@ namespace ccf::js } ccf::ApiResult DynamicJSEndpointRegistry::get_js_runtime_options_v1( - ccf::JSRuntimeOptions& options, kv::ReadOnlyTx& tx) + ccf::JSRuntimeOptions& options, ccf::kv::ReadOnlyTx& tx) { try { @@ -715,7 +721,7 @@ namespace ccf::js } ccf::endpoints::EndpointDefinitionPtr DynamicJSEndpointRegistry:: - find_endpoint(kv::Tx& tx, ccf::RpcContext& rpc_ctx) + find_endpoint(ccf::kv::Tx& tx, ccf::RpcContext& rpc_ctx) { // Look up the endpoint definition // First in the user-defined endpoints, and then fall-back to built-ins @@ -841,7 +847,7 @@ namespace ccf::js // Since we do our own dispatch (overriding find_endpoint), make sure we // describe those operations in the auto-generated OpenAPI void DynamicJSEndpointRegistry::build_api( - nlohmann::json& document, kv::ReadOnlyTx& tx) + nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) { ccf::UserEndpointRegistry::build_api(document, tx); @@ -875,4 +881,101 @@ namespace ccf::js return true; }); } + + ccf::ApiResult DynamicJSEndpointRegistry::check_action_not_replayed_v1( + ccf::kv::Tx& tx, + uint64_t created_at, + const std::span action, + ccf::InvalidArgsReason& reason) + { + try + { + const auto created_at_str = fmt::format("{:0>10}", created_at); + const auto action_digest = + ccf::crypto::sha256(action.data(), action.size()); + + using RecentActions = ccf::kv::Set; + + auto recent_actions = tx.rw(recent_actions_map); + auto key = + fmt::format("{}:{}", created_at_str, ds::to_hex(action_digest)); + + if (recent_actions->contains(key)) + { + reason = ccf::InvalidArgsReason::ActionAlreadyApplied; + return ApiResult::InvalidArgs; + } + + // In the absence of in-KV support for sorted sets, we need + // to extract them and sort them here. + std::vector replay_keys; + recent_actions->foreach([&replay_keys](const std::string& replay_key) { + replay_keys.push_back(replay_key); + return true; + }); + std::sort(replay_keys.begin(), replay_keys.end()); + + // Actions must be more recent than the median of recent actions + if (!replay_keys.empty()) + { + const auto [min_created_at, _] = + ccf::nonstd::split_1(replay_keys[replay_keys.size() / 2], ":"); + auto [key_ts, __] = ccf::nonstd::split_1(key, ":"); + if (key_ts < min_created_at) + { + reason = ccf::InvalidArgsReason::StaleActionCreatedTimestamp; + return ApiResult::InvalidArgs; + } + } + + // The action is neither stale, nor a replay + recent_actions->insert(key); + + // Only keep the most recent window_size proposals, do not + // allow the set to grow indefinitely. + // Should this be configurable through runtime options? + size_t window_size = 100; + if (replay_keys.size() >= (window_size - 1) /* We just added one */) + { + for (size_t i = 0; i < (replay_keys.size() - (window_size - 1)); i++) + { + recent_actions->remove(replay_keys[i]); + } + } + + return ApiResult::OK; + } + catch (const std::exception& e) + { + LOG_FAIL_FMT("{}", e.what()); + return ApiResult::InternalError; + } + } + + ccf::ApiResult DynamicJSEndpointRegistry::record_action_for_audit_v1( + ccf::kv::Tx& tx, + ccf::ActionFormat format, + const std::string& user_id, + const std::string& action_name, + const std::vector& action_body) + { + try + { + using AuditInputValue = ccf::kv::Value>; + using AuditInfoValue = ccf::kv::Value; + + auto audit_input = tx.template rw(audit_input_map); + audit_input->put(action_body); + + auto audit_info = tx.template rw(audit_info_map); + audit_info->put({format, user_id, action_name}); + + return ApiResult::OK; + } + catch (const std::exception& e) + { + LOG_FAIL_FMT("{}", e.what()); + return ApiResult::InternalError; + } + } } diff --git a/src/kv/apply_changes.h b/src/kv/apply_changes.h index ce6bf58be3bc..841878c8378c 100644 --- a/src/kv/apply_changes.h +++ b/src/kv/apply_changes.h @@ -9,7 +9,7 @@ #include #include -namespace kv +namespace ccf::kv { // All collections of Map must be ordered so that we lock their contained // maps in a stable order. The order here is by map name @@ -37,7 +37,7 @@ namespace kv static inline std::optional apply_changes( OrderedChanges& changes, VersionResolver version_resolver_fn, - kv::ConsensusHookPtrs& hooks, + ccf::kv::ConsensusHookPtrs& hooks, const MapCollection& new_maps, const std::optional& new_maps_conflict_version, bool track_read_versions, @@ -115,7 +115,7 @@ namespace kv // This is to avoid recursively locking version_lock by calling // current_version() in the commit_reserved case. - kv::Version current_v; + ccf::kv::Version current_v; if (new_maps_conflict_version.has_value()) { current_v = *new_maps_conflict_version; @@ -135,7 +135,7 @@ namespace kv if (ok && has_writes) { // Get the version number to be used for this commit. - kv::Version version_last_new_map; + ccf::kv::Version version_last_new_map; std::tie(version, version_last_new_map) = version_resolver_fn(!new_maps.empty()); diff --git a/src/kv/committable_tx.h b/src/kv/committable_tx.h index 8e654a947374..7f73f5902531 100644 --- a/src/kv/committable_tx.h +++ b/src/kv/committable_tx.h @@ -12,7 +12,7 @@ #include -namespace kv +namespace ccf::kv { class CommittableTx : public Tx, public AbstractChangeContainer { @@ -32,13 +32,13 @@ namespace kv Version version = NoVersion; - kv::TxHistory::RequestID req_id; + ccf::kv::TxHistory::RequestID req_id; TxFlags flags = 0; SerialisedEntryFlags entry_flags = 0; std::vector serialise( - crypto::Sha256Hash& commit_evidence_digest, + ccf::crypto::Sha256Hash& commit_evidence_digest, std::string& commit_evidence, const ccf::ClaimsDigest& claims_digest_, bool include_reads = false) @@ -76,7 +76,7 @@ namespace kv version, ccf::ds::to_hex(commit_nonce)); LOG_TRACE_FMT("Commit evidence: {}", commit_evidence); - crypto::Sha256Hash tx_commit_evidence_digest(commit_evidence); + ccf::crypto::Sha256Hash tx_commit_evidence_digest(commit_evidence); commit_evidence_digest = tx_commit_evidence_digest; auto entry_type = EntryType::WriteSetWithCommitEvidenceAndClaims; @@ -119,9 +119,10 @@ namespace kv * replication * * A transaction can either succeed and replicate - * (`kv::CommitResult::SUCCESS`), fail because of a conflict with other - * transactions (`kv::CommitResult::FAIL_CONFLICT`), or succeed locally, but - * fail to replicate (`kv::CommitResult::FAIL_NO_REPLICATE`). + * (`ccf::kv::CommitResult::SUCCESS`), fail because of a conflict with other + * transactions (`ccf::kv::CommitResult::FAIL_CONFLICT`), or succeed + * locally, but fail to replicate + * (`ccf::kv::CommitResult::FAIL_NO_REPLICATE`). * * Transactions that fail are rolled back, no matter the reason. * @@ -154,7 +155,7 @@ namespace kv this->pimpl->store->lock_map_set(); } - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; std::optional new_maps_conflict_version = std::nullopt; @@ -219,7 +220,7 @@ namespace kv // recover. try { - crypto::Sha256Hash commit_evidence_digest; + ccf::crypto::Sha256Hash commit_evidence_digest; std::string commit_evidence; auto data = serialise(commit_evidence_digest, commit_evidence, claims); @@ -297,7 +298,7 @@ namespace kv /** Version for the transaction set * - * @return Committed version, or `kv::NoVersion` otherwise + * @return Committed version, or `ccf::kv::NoVersion` otherwise */ Version get_version() { @@ -344,12 +345,12 @@ namespace kv pimpl->commit_view = view_; } - void set_req_id(const kv::TxHistory::RequestID& req_id_) + void set_req_id(const ccf::kv::TxHistory::RequestID& req_id_) { req_id = req_id_; } - const kv::TxHistory::RequestID& get_req_id() + const ccf::kv::TxHistory::RequestID& get_req_id() { return req_id; } @@ -364,7 +365,7 @@ namespace kv pimpl->commit_view = commit_view_; } - void set_root_at_read_version(const crypto::Sha256Hash& r) + void set_root_at_read_version(const ccf::crypto::Sha256Hash& r) { root_at_read_version = r; } @@ -437,7 +438,7 @@ namespace kv if (!success) throw std::logic_error("Failed to commit reserved transaction"); - crypto::Sha256Hash commit_evidence_digest; + ccf::crypto::Sha256Hash commit_evidence_digest; std::string commit_evidence; // This is a signature and, if the ledger chunking or snapshot flags are diff --git a/src/kv/compacted_version_conflict.h b/src/kv/compacted_version_conflict.h index f41fcb3cae39..9c0605621819 100644 --- a/src/kv/compacted_version_conflict.h +++ b/src/kv/compacted_version_conflict.h @@ -4,7 +4,7 @@ #include -namespace kv +namespace ccf::kv { class CompactedVersionConflict { diff --git a/src/kv/deserialise.h b/src/kv/deserialise.h index 7f34b6eda78f..f3c4e8855cd0 100644 --- a/src/kv/deserialise.h +++ b/src/kv/deserialise.h @@ -11,7 +11,7 @@ #include -namespace kv +namespace ccf::kv { class ExecutionWrapperStore { @@ -19,20 +19,20 @@ namespace kv virtual bool fill_maps( const std::vector& data, bool public_only, - kv::Version& v, - kv::Term& view, - kv::OrderedChanges& changes, - kv::MapCollection& new_maps, + ccf::kv::Version& v, + ccf::kv::Term& view, + ccf::kv::OrderedChanges& changes, + ccf::kv::MapCollection& new_maps, ccf::ClaimsDigest& claims_digest, - std::optional& commit_evidence_digest, + std::optional& commit_evidence_digest, bool ignore_strict_versions = false) = 0; virtual bool commit_deserialised( - kv::OrderedChanges& changes, - kv::Version v, - kv::Term term, + ccf::kv::OrderedChanges& changes, + ccf::kv::Version v, + ccf::kv::Term term, const MapCollection& new_maps, - kv::ConsensusHookPtrs& hooks, + ccf::kv::ConsensusHookPtrs& hooks, bool track_deletes_on_missing_keys) = 0; }; @@ -43,13 +43,13 @@ namespace kv std::shared_ptr history; const std::vector data; bool public_only; - kv::Version version; + ccf::kv::Version version; Term term; OrderedChanges changes; MapCollection new_maps; - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; ccf::ClaimsDigest claims_digest; - std::optional commit_evidence_digest = {}; + std::optional commit_evidence_digest = {}; const std::optional expected_txid; @@ -72,7 +72,7 @@ namespace kv return std::move(claims_digest); } - std::optional&& consume_commit_evidence_digest() + std::optional&& consume_commit_evidence_digest() override { return std::move(commit_evidence_digest); @@ -162,7 +162,7 @@ namespace kv return success; } - kv::ConsensusHookPtrs& get_hooks() override + ccf::kv::ConsensusHookPtrs& get_hooks() override { return hooks; } @@ -177,7 +177,7 @@ namespace kv return term; } - kv::Version get_index() override + ccf::kv::Version get_index() override { return version; } diff --git a/src/kv/encryptor.h b/src/kv/encryptor.h index f4bc4c351cf1..a872d8052a66 100644 --- a/src/kv/encryptor.h +++ b/src/kv/encryptor.h @@ -8,7 +8,7 @@ #include #include -namespace kv +namespace ccf::kv { template class TxEncryptor : public AbstractTxEncryptor @@ -142,7 +142,7 @@ namespace kv return ret; } - crypto::HashBytes get_commit_nonce( + ccf::crypto::HashBytes get_commit_nonce( const TxID& tx_id, bool historical_hint = false) override { auto secret = @@ -155,8 +155,8 @@ namespace kv std::vector txid = { txid_str.data(), txid_str.data() + txid_str.size()}; - auto commit_nonce = - crypto::hmac(crypto::MDType::SHA256, secret->get_commit_secret(), txid); + auto commit_nonce = ccf::crypto::hmac( + ccf::crypto::MDType::SHA256, secret->get_commit_secret(), txid); return commit_nonce; } diff --git a/src/kv/generic_serialise_wrapper.h b/src/kv/generic_serialise_wrapper.h index 8d3b88769c17..1fe9722b011a 100644 --- a/src/kv/generic_serialise_wrapper.h +++ b/src/kv/generic_serialise_wrapper.h @@ -10,10 +10,10 @@ #include -namespace kv +namespace ccf::kv { - using SerialisedKey = kv::serialisers::SerialisedEntry; - using SerialisedValue = kv::serialisers::SerialisedEntry; + using SerialisedKey = ccf::kv::serialisers::SerialisedEntry; + using SerialisedValue = ccf::kv::serialisers::SerialisedEntry; template class GenericSerialiseWrapper @@ -65,7 +65,7 @@ namespace kv SerialisedEntryFlags header_flags_, // The evidence and claims digest must be systematically present // in regular transactions, but absent in snapshots. - const crypto::Sha256Hash& commit_evidence_digest_ = {}, + const ccf::crypto::Sha256Hash& commit_evidence_digest_ = {}, const ccf::ClaimsDigest& claims_digest_ = ccf::no_claims(), bool historical_hint_ = false) : tx_id(tx_id_), @@ -246,7 +246,8 @@ namespace kv // Present systematically in regular transactions, but absent from snapshots ccf::ClaimsDigest claims_digest = ccf::no_claims(); // Present systematically in regular transactions, but absent from snapshots - std::optional commit_evidence_digest = std::nullopt; + std::optional commit_evidence_digest = + std::nullopt; Version version; std::shared_ptr crypto_util; std::optional domain_restriction; @@ -268,9 +269,9 @@ namespace kv { auto digest_array = public_reader - .template read_next(); + .template read_next(); commit_evidence_digest = - crypto::Sha256Hash::from_representation(digest_array); + ccf::crypto::Sha256Hash::from_representation(digest_array); } // max_conflict_version is included for compatibility, but currently // ignored @@ -290,7 +291,7 @@ namespace kv return std::move(claims_digest); } - std::optional&& consume_commit_evidence_digest() + std::optional&& consume_commit_evidence_digest() { return std::move(commit_evidence_digest); } @@ -298,7 +299,7 @@ namespace kv std::optional init( const uint8_t* data, size_t size, - kv::Term& term, + ccf::kv::Term& term, bool historical_hint = false) { current_reader = &public_reader; diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index c9bf986538dc..fbad8485a76a 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -36,7 +36,7 @@ namespace aft struct Request; } -namespace kv +namespace ccf::kv { // Term describes an epoch of Versions. It is incremented when global kv's // writer(s) changes. Term and Version combined give a unique identifier for @@ -112,7 +112,7 @@ namespace kv inline void from_json(const nlohmann::json& j, Configuration::NodeInfo& ni) { const std::string addr(j["address"]); - const auto& [h, p] = nonstd::split_1(addr, ":"); + const auto& [h, p] = ccf::nonstd::split_1(addr, ":"); ni.hostname = h; ni.port = p; } @@ -380,7 +380,7 @@ namespace kv { RequestID rid; Version version; - crypto::Sha256Hash replicated_state_merkle_root; + ccf::crypto::Sha256Hash replicated_state_merkle_root; }; struct ResponseCallbackArgs @@ -401,16 +401,16 @@ namespace kv virtual Result verify_and_sign( ccf::PrimarySignature& signature, Term* term, - kv::Configuration::Nodes& nodes) = 0; + ccf::kv::Configuration::Nodes& nodes) = 0; virtual bool verify( Term* term = nullptr, ccf::PrimarySignature* sig = nullptr) = 0; virtual void try_emit_signature() = 0; virtual void emit_signature() = 0; - virtual crypto::Sha256Hash get_replicated_state_root() = 0; + virtual ccf::crypto::Sha256Hash get_replicated_state_root() = 0; virtual std::tuple< - kv::TxID /* TxID of last transaction seen by history */, - crypto::Sha256Hash /* root as of TxID */, - kv::Term /* term_of_next_version */> + ccf::kv::TxID /* TxID of last transaction seen by history */, + ccf::crypto::Sha256Hash /* root as of TxID */, + ccf::kv::Term /* term_of_next_version */> get_replicated_state_txid_and_root() = 0; virtual std::vector get_proof(Version v) = 0; virtual bool verify_proof(const std::vector& proof) = 0; @@ -419,14 +419,14 @@ namespace kv virtual std::vector get_raw_leaf(uint64_t index) = 0; virtual void append(const std::vector& data) = 0; virtual void append_entry( - const crypto::Sha256Hash& digest, - std::optional expected_term = std::nullopt) = 0; + const ccf::crypto::Sha256Hash& digest, + std::optional expected_term = std::nullopt) = 0; virtual void rollback( - const kv::TxID& tx_id, kv::Term term_of_next_version_) = 0; + const ccf::kv::TxID& tx_id, ccf::kv::Term term_of_next_version_) = 0; virtual void compact(Version v) = 0; - virtual void set_term(kv::Term) = 0; + virtual void set_term(ccf::kv::Term) = 0; virtual std::vector serialise_tree(size_t to) = 0; - virtual void set_endorsed_certificate(const crypto::Pem& cert) = 0; + virtual void set_endorsed_certificate(const ccf::crypto::Pem& cert) = 0; virtual void start_signature_emit_timer() = 0; }; @@ -486,14 +486,14 @@ namespace kv CommitResult success; std::vector data; ccf::ClaimsDigest claims_digest; - crypto::Sha256Hash commit_evidence_digest; + ccf::crypto::Sha256Hash commit_evidence_digest; std::vector hooks; PendingTxInfo( CommitResult success_, std::vector&& data_, ccf::ClaimsDigest&& claims_digest_, - crypto::Sha256Hash&& commit_evidence_digest_, + ccf::crypto::Sha256Hash&& commit_evidence_digest_, std::vector&& hooks_) : success(success_), data(std::move(data_)), @@ -515,14 +515,14 @@ namespace kv private: std::vector data; ccf::ClaimsDigest claims_digest; - crypto::Sha256Hash commit_evidence_digest; + ccf::crypto::Sha256Hash commit_evidence_digest; ConsensusHookPtrs hooks; public: MovePendingTx( std::vector&& data_, ccf::ClaimsDigest&& claims_digest_, - crypto::Sha256Hash&& commit_evidence_digest_, + ccf::crypto::Sha256Hash&& commit_evidence_digest_, ConsensusHookPtrs&& hooks_) : data(std::move(data_)), claims_digest(std::move(claims_digest_)), @@ -568,7 +568,7 @@ namespace kv virtual size_t get_header_length() = 0; virtual uint64_t get_term(const uint8_t* data, size_t size) = 0; - virtual crypto::HashBytes get_commit_nonce( + virtual ccf::crypto::HashBytes get_commit_nonce( const TxID& tx_id, bool historical_hint = false) = 0; }; using EncryptorPtr = std::shared_ptr; @@ -578,9 +578,9 @@ namespace kv public: virtual ~AbstractSnapshotter(){}; - virtual bool record_committable(kv::Version v) = 0; - virtual void commit(kv::Version v, bool generate_snapshot) = 0; - virtual void rollback(kv::Version v) = 0; + virtual bool record_committable(ccf::kv::Version v) = 0; + virtual void commit(ccf::kv::Version v, bool generate_snapshot) = 0; + virtual void rollback(ccf::kv::Version v) = 0; }; using SnapshotterPtr = std::shared_ptr; @@ -649,16 +649,16 @@ namespace kv { public: virtual ~AbstractExecutionWrapper() = default; - virtual kv::ApplyResult apply( + virtual ccf::kv::ApplyResult apply( bool track_deletes_on_missing_keys = false) = 0; - virtual kv::ConsensusHookPtrs& get_hooks() = 0; + virtual ccf::kv::ConsensusHookPtrs& get_hooks() = 0; virtual const std::vector& get_entry() = 0; - virtual kv::Term get_term() = 0; - virtual kv::Version get_index() = 0; + virtual ccf::kv::Term get_term() = 0; + virtual ccf::kv::Version get_index() = 0; virtual bool support_async_execution() = 0; virtual bool is_public_only() = 0; virtual ccf::ClaimsDigest&& consume_claims_digest() = 0; - virtual std::optional&& + virtual std::optional&& consume_commit_evidence_digest() = 0; // Setting a short rollback is a work around that should be fixed @@ -782,7 +782,7 @@ namespace kv FMT_BEGIN_NAMESPACE template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -791,8 +791,8 @@ struct formatter } template - auto format(const kv::Configuration::Nodes& nodes, FormatContext& ctx) const - -> decltype(ctx.out()) + auto format(const ccf::kv::Configuration::Nodes& nodes, FormatContext& ctx) + const -> decltype(ctx.out()) { std::set node_ids; for (auto& [nid, _] : nodes) @@ -804,7 +804,7 @@ struct formatter }; template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -813,7 +813,7 @@ struct formatter } template - auto format(const kv::MembershipState& state, FormatContext& ctx) const + auto format(const ccf::kv::MembershipState& state, FormatContext& ctx) const -> decltype(ctx.out()) { const auto s = nlohmann::json(state).get(); @@ -822,7 +822,7 @@ struct formatter }; template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -831,7 +831,7 @@ struct formatter } template - auto format(const kv::LeadershipState& state, FormatContext& ctx) const + auto format(const ccf::kv::LeadershipState& state, FormatContext& ctx) const -> decltype(ctx.out()) { const auto s = nlohmann::json(state).get(); diff --git a/src/kv/raw_serialise.h b/src/kv/raw_serialise.h index 585d12e5f55f..e3ef83a18e98 100644 --- a/src/kv/raw_serialise.h +++ b/src/kv/raw_serialise.h @@ -9,7 +9,7 @@ #include #include -namespace kv +namespace ccf::kv { class RawWriter { @@ -81,8 +81,8 @@ namespace kv void append(const T& entry) { if constexpr ( - nonstd::is_std_vector::value || - std::is_same_v) + ccf::nonstd::is_std_vector::value || + std::is_same_v) { serialise_entry(entry.size() * sizeof(typename T::value_type)); if (entry.size() > 0) @@ -90,7 +90,7 @@ namespace kv serialise_vector(entry); } } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { serialise_array(entry.h); } @@ -109,7 +109,7 @@ namespace kv else { static_assert( - nonstd::dependent_false::value, "Can't serialise this type"); + ccf::nonstd::dependent_false::value, "Can't serialise this type"); } } @@ -183,8 +183,8 @@ namespace kv T read_next() { if constexpr ( - nonstd::is_std_vector::value || - std::is_same_v) + ccf::nonstd::is_std_vector::value || + std::is_same_v) { size_t entry_offset = 0; size_t entry_size = read_size_prefixed_entry(entry_offset); @@ -196,7 +196,7 @@ namespace kv return ret; } - else if constexpr (nonstd::is_std_array::value) + else if constexpr (ccf::nonstd::is_std_array::value) { T ret; auto data_ = reinterpret_cast(ret.data()); @@ -207,14 +207,14 @@ namespace kv return ret; } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { uint8_t entry_type = read_entry(); - if (entry_type > static_cast(kv::EntryType::MAX)) + if (entry_type > static_cast(ccf::kv::EntryType::MAX)) throw std::logic_error( fmt::format("Invalid EntryType: {}", entry_type)); - return kv::EntryType(entry_type); + return ccf::kv::EntryType(entry_type); } else if constexpr (std::is_same_v) { @@ -230,7 +230,8 @@ namespace kv else { static_assert( - nonstd::dependent_false::value, "Can't deserialise this type"); + ccf::nonstd::dependent_false::value, + "Can't deserialise this type"); } } diff --git a/src/kv/serialised_entry_format.h b/src/kv/serialised_entry_format.h index ab5a78c0f739..0426861f9d28 100644 --- a/src/kv/serialised_entry_format.h +++ b/src/kv/serialised_entry_format.h @@ -6,7 +6,7 @@ #include -namespace kv +namespace ccf::kv { static constexpr auto entry_format_v1 = 1; using SerialisedEntryFlags = uint8_t; diff --git a/src/kv/serialiser_declare.h b/src/kv/serialiser_declare.h index 1959c5308588..73fe5758bd42 100644 --- a/src/kv/serialiser_declare.h +++ b/src/kv/serialiser_declare.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -namespace kv +namespace ccf::kv { // This awkward forward declaration allows the -templated serialisers to // depend on kv_types.h, and removes the reverse dependency. Once these diff --git a/src/kv/snapshot.h b/src/kv/snapshot.h index d50302fb1f7e..af306e77e387 100644 --- a/src/kv/snapshot.h +++ b/src/kv/snapshot.h @@ -4,21 +4,22 @@ #include "kv/kv_types.h" -namespace kv +namespace ccf::kv { class StoreSnapshot : public AbstractStore::AbstractSnapshot { private: Version version; - std::vector> snapshots; + std::vector> snapshots; std::optional> hash_at_snapshot = std::nullopt; std::optional> view_history = std::nullopt; public: StoreSnapshot(Version version_) : version(version_) {} - void add_map_snapshot(std::unique_ptr snapshot) + void add_map_snapshot( + std::unique_ptr snapshot) { snapshots.push_back(std::move(snapshot)); } @@ -53,7 +54,7 @@ namespace kv KvStoreSerialiser serialiser( encryptor, {0, version}, - kv::EntryType::Snapshot, + ccf::kv::EntryType::Snapshot, 0, {}, ccf::no_claims(), diff --git a/src/kv/store.h b/src/kv/store.h index a7750762ea3e..e1789e4ef663 100644 --- a/src/kv/store.h +++ b/src/kv/store.h @@ -18,7 +18,7 @@ #include #include -namespace kv +namespace ccf::kv { class StoreState { @@ -26,14 +26,15 @@ namespace kv // All collections of Map must be ordered so that we lock their contained // maps in a stable order. The order here is by map name. The version // indicates the version at which the Map was created. - using Maps = std:: - map>>; + using Maps = std::map< + std::string, + std::pair>>; ccf::pal::Mutex maps_lock; Maps maps; ccf::pal::Mutex version_lock; std::atomic version = 0; - Version last_new_map = kv::NoVersion; + Version last_new_map = ccf::kv::NoVersion; std::atomic compacted = 0; // Calls to Store::commit are made atomic by taking this lock. @@ -68,7 +69,7 @@ namespace kv pending_txs.clear(); version = 0; - last_new_map = kv::NoVersion; + last_new_map = ccf::kv::NoVersion; compacted = 0; term_of_next_version = 0; term_of_last_version = 0; @@ -85,8 +86,8 @@ namespace kv public ReadOnlyStore { private: - using Hooks = std::map; - using MapHooks = std::map; + using Hooks = std::map; + using MapHooks = std::map; Hooks global_hooks; MapHooks map_hooks; @@ -113,7 +114,7 @@ namespace kv Version v, Term term, const MapCollection& new_maps, - kv::ConsensusHookPtrs& hooks, + ccf::kv::ConsensusHookPtrs& hooks, bool track_deletes_on_missing_keys) override { auto c = apply_changes( @@ -232,20 +233,20 @@ namespace kv * such map exists */ std::shared_ptr get_map( - kv::Version v, const std::string& map_name) override + ccf::kv::Version v, const std::string& map_name) override { std::lock_guard mguard(maps_lock); return get_map_internal(v, map_name); } std::shared_ptr get_map_unsafe( - kv::Version v, const std::string& map_name) override + ccf::kv::Version v, const std::string& map_name) override { return get_map_internal(v, map_name); } - std::shared_ptr get_map_internal( - kv::Version v, const std::string& map_name) + std::shared_ptr get_map_internal( + ccf::kv::Version v, const std::string& map_name) { auto search = maps.find(map_name); if (search != maps.end()) @@ -271,9 +272,9 @@ namespace kv * @param map_ Map to add */ void add_dynamic_map( - kv::Version v, const std::shared_ptr& map_) override + ccf::kv::Version v, const std::shared_ptr& map_) override { - auto map = std::dynamic_pointer_cast(map_); + auto map = std::dynamic_pointer_cast(map_); if (map == nullptr) { throw std::logic_error(fmt::format( @@ -383,17 +384,17 @@ namespace kv ApplyResult deserialise_snapshot( const uint8_t* data, size_t size, - kv::ConsensusHookPtrs& hooks, + ccf::kv::ConsensusHookPtrs& hooks, std::vector* view_history = nullptr, bool public_only = false) override { auto e = get_encryptor(); auto d = KvStoreDeserialiser( e, - public_only ? kv::SecurityDomain::PUBLIC : - std::optional()); + public_only ? ccf::kv::SecurityDomain::PUBLIC : + std::optional()); - kv::Term term; + ccf::kv::Term term; auto v_ = d.init(data, size, term, is_historical); if (!v_.has_value()) { @@ -431,12 +432,12 @@ namespace kv { const auto map_name = r.value(); - std::shared_ptr map = nullptr; + std::shared_ptr map = nullptr; auto search = maps.find(map_name); if (search == maps.end()) { - map = std::make_shared( + map = std::make_shared( this, map_name, get_security_domain(map_name)); new_maps[map_name] = map; LOG_DEBUG_FMT( @@ -688,12 +689,12 @@ namespace kv bool fill_maps( const std::vector& data, bool public_only, - kv::Version& v, - kv::Term& view, + ccf::kv::Version& v, + ccf::kv::Term& view, OrderedChanges& changes, MapCollection& new_maps, ccf::ClaimsDigest& claims_digest, - std::optional& commit_evidence_digest, + std::optional& commit_evidence_digest, bool ignore_strict_versions = false) override { // This will return FAILED if the serialised transaction is being @@ -705,8 +706,8 @@ namespace kv auto d = KvStoreDeserialiser( e, - public_only ? kv::SecurityDomain::PUBLIC : - std::optional()); + public_only ? ccf::kv::SecurityDomain::PUBLIC : + std::optional()); auto v_ = d.init(data.data(), data.size(), view, is_historical); if (!v_.has_value()) @@ -757,7 +758,7 @@ namespace kv auto map = get_map_internal(v, map_name); if (map == nullptr) { - auto new_map = std::make_shared( + auto new_map = std::make_shared( this, map_name, get_security_domain(map_name)); map = new_map; new_maps[map_name] = new_map; @@ -795,7 +796,7 @@ namespace kv return true; } - std::unique_ptr deserialize( + std::unique_ptr deserialize( const std::vector& data, bool public_only = false, const std::optional& expected_txid = std::nullopt) override @@ -839,7 +840,7 @@ namespace kv return version; } - kv::TxID current_txid() override + ccf::kv::TxID current_txid() override { // Must lock in case the version or read term is being incremented. std::lock_guard vguard(version_lock); @@ -965,7 +966,7 @@ namespace kv auto data_shared = std::make_shared>(std::move(data_)); auto hooks_shared = - std::make_shared(std::move(hooks_)); + std::make_shared(std::move(hooks_)); // NB: this cannot happen currently. Regular Tx only make it here if // they did succeed, and signatures cannot conflict because they @@ -1147,7 +1148,7 @@ namespace kv // rollback auto new_map = std::make_pair( NoVersion, - std::make_shared( + std::make_shared( this, name, SecurityDomain::PRIVATE)); maps[name] = new_map; map = new_map.second; @@ -1181,7 +1182,7 @@ namespace kv } void set_map_hook( - const std::string& map_name, const kv::untyped::Map::MapHook& hook) + const std::string& map_name, const ccf::kv::untyped::Map::MapHook& hook) { map_hooks[map_name] = hook; @@ -1204,7 +1205,8 @@ namespace kv } void set_global_hook( - const std::string& map_name, const kv::untyped::Map::CommitHook& hook) + const std::string& map_name, + const ccf::kv::untyped::Map::CommitHook& hook) { global_hooks[map_name] = hook; @@ -1292,5 +1294,5 @@ namespace kv } }; - using StorePtr = std::shared_ptr; + using StorePtr = std::shared_ptr; } diff --git a/src/kv/test/kv_bench.cpp b/src/kv/test/kv_bench.cpp index b4fe69e6a960..5107156c1e45 100644 --- a/src/kv/test/kv_bench.cpp +++ b/src/kv/test/kv_bench.cpp @@ -10,9 +10,9 @@ #include #include -using KeyType = kv::serialisers::SerialisedEntry; -using ValueType = kv::serialisers::SerialisedEntry; -using MapType = kv::untyped::Map; +using KeyType = ccf::kv::serialisers::SerialisedEntry; +using ValueType = ccf::kv::serialisers::SerialisedEntry; +using MapType = ccf::kv::untyped::Map; inline void clobber_memory() { @@ -39,23 +39,24 @@ std::shared_ptr create_ledger_secrets() return secrets; } -std::string build_map_name(const std::string& core_name, kv::SecurityDomain sd) +std::string build_map_name( + const std::string& core_name, ccf::kv::SecurityDomain sd) { - if (sd == kv::SecurityDomain::PUBLIC) + if (sd == ccf::kv::SecurityDomain::PUBLIC) { - return fmt::format("{}{}", kv::public_domain_prefix, core_name); + return fmt::format("{}{}", ccf::kv::public_domain_prefix, core_name); } return core_name; } // Test functions -template +template static void serialise(picobench::state& s) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - kv::Store kv_store; + ccf::kv::Store kv_store; auto secrets = create_ledger_secrets(); auto encryptor = std::make_shared(secrets); kv_store.set_encryptor(encryptor); @@ -77,20 +78,20 @@ static void serialise(picobench::state& s) s.start_timer(); auto rc = tx.commit(); - if (rc != kv::CommitResult::SUCCESS) + if (rc != ccf::kv::CommitResult::SUCCESS) throw std::logic_error("Transaction commit failed: " + std::to_string(rc)); s.stop_timer(); } -template +template static void deserialise(picobench::state& s) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - kv::Store kv_store; - kv::Store kv_store2; + ccf::kv::Store kv_store; + ccf::kv::Store kv_store2; - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); kv_store.set_consensus(consensus); auto secrets = create_ledger_secrets(); @@ -117,7 +118,7 @@ static void deserialise(picobench::state& s) s.start_timer(); auto rc = kv_store2.deserialize(consensus->get_latest_data().value())->apply(); - if (rc != kv::ApplyResult::PASS) + if (rc != ccf::kv::ApplyResult::PASS) throw std::logic_error( "Transaction deserialisation failed: " + std::to_string(rc)); s.stop_timer(); @@ -126,9 +127,9 @@ static void deserialise(picobench::state& s) template static void commit_latency(picobench::state& s) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - kv::Store kv_store; + ccf::kv::Store kv_store; auto secrets = create_ledger_secrets(); auto encryptor = std::make_shared(secrets); kv_store.set_encryptor(encryptor); @@ -150,7 +151,7 @@ static void commit_latency(picobench::state& s) } auto rc = tx.commit(); - if (rc != kv::CommitResult::SUCCESS) + if (rc != ccf::kv::CommitResult::SUCCESS) { throw std::logic_error( "Transaction commit failed: " + std::to_string(rc)); @@ -164,9 +165,9 @@ static void commit_latency(picobench::state& s) template static void ser_snap(picobench::state& s) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - kv::Store kv_store; + ccf::kv::Store kv_store; auto secrets = create_ledger_secrets(); auto encryptor = std::make_shared(secrets); kv_store.set_encryptor(encryptor); @@ -185,14 +186,14 @@ static void ser_snap(picobench::state& s) } auto rc = tx.commit(); - if (rc != kv::CommitResult::SUCCESS) + if (rc != ccf::kv::CommitResult::SUCCESS) throw std::logic_error("Transaction commit failed: " + std::to_string(rc)); s.start_timer(); - std::unique_ptr snap = nullptr; + std::unique_ptr snap = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&kv_store); + ccf::kv::ScopedStoreMapsLock maps_lock(&kv_store); snap = kv_store.snapshot_unsafe_maps(tx.commit_version()); } kv_store.serialise_snapshot(std::move(snap)); @@ -202,10 +203,10 @@ static void ser_snap(picobench::state& s) template static void des_snap(picobench::state& s) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; - kv::Store kv_store; - kv::Store kv_store2; + ccf::kv::Store kv_store; + ccf::kv::Store kv_store2; auto secrets = create_ledger_secrets(); auto encryptor = std::make_shared(secrets); kv_store.set_encryptor(encryptor); @@ -225,17 +226,17 @@ static void des_snap(picobench::state& s) } auto rc = tx.commit(); - if (rc != kv::CommitResult::SUCCESS) + if (rc != ccf::kv::CommitResult::SUCCESS) throw std::logic_error("Transaction commit failed: " + std::to_string(rc)); - std::unique_ptr snap = nullptr; + std::unique_ptr snap = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&kv_store); + ccf::kv::ScopedStoreMapsLock maps_lock(&kv_store); snap = kv_store.snapshot_unsafe_maps(tx.commit_version()); } auto serialised_snap = kv_store.serialise_snapshot(std::move(snap)); - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; s.start_timer(); kv_store2.deserialise_snapshot( serialised_snap.data(), serialised_snap.size(), hooks); @@ -245,7 +246,7 @@ static void des_snap(picobench::state& s) const std::vector tx_count = {10, 100, 1000}; const uint32_t sample_size = 100; -using SD = kv::SecurityDomain; +using SD = ccf::kv::SecurityDomain; PICOBENCH_SUITE("commit_latency"); PICOBENCH(commit_latency<10>).iterations(tx_count).samples(10).baseline(); @@ -284,7 +285,7 @@ PICOBENCH(des_snap<1000>).iterations(map_count).samples(snapshot_sample_size); int main(int argc, char** argv) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); picobench::runner runner; runner.parse_cmd_line(argc, argv); diff --git a/src/kv/test/kv_contention.cpp b/src/kv/test/kv_contention.cpp index 65d74c68603b..62a4178b2244 100644 --- a/src/kv/test/kv_contention.cpp +++ b/src/kv/test/kv_contention.cpp @@ -17,12 +17,12 @@ #include #include -class SlowStubConsensus : public kv::test::StubConsensus +class SlowStubConsensus : public ccf::kv::test::StubConsensus { public: - using kv::test::StubConsensus::StubConsensus; + using ccf::kv::test::StubConsensus::StubConsensus; - bool replicate(const kv::BatchVector& entries, ccf::View view) override + bool replicate(const ccf::kv::BatchVector& entries, ccf::View view) override { if (rand() % 2 == 0) { @@ -30,19 +30,19 @@ class SlowStubConsensus : public kv::test::StubConsensus std::this_thread::sleep_for(std::chrono::milliseconds(delay)); } - return kv::test::StubConsensus::replicate(entries, view); + return ccf::kv::test::StubConsensus::replicate(entries, view); } }; DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) { - logger::config::level() = LoggerLevel::INFO; + ccf::logger::config::level() = LoggerLevel::INFO; // Multiple threads write random entries into random tables, and attempt to // commit them. A single thread continually compacts the kv to the latest // entry. The goal is for these commits and compactions to avoid deadlock - using MapType = kv::Map; + using MapType = ccf::kv::Map; constexpr size_t max_k = 32; constexpr size_t thread_count = 16; @@ -54,7 +54,7 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) struct ThreadArgs { std::vector maps; - kv::Store* kv_store; + ccf::kv::Store* kv_store; std::atomic* counter; }; ThreadArgs args[thread_count] = {}; @@ -80,7 +80,7 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) } auto thread_fn = [](void* a) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto args = static_cast(a); for (size_t i = 0u; i < tx_count; ++i) @@ -124,12 +124,12 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) // Try to commit const auto result = tx.commit(); - if (result == kv::CommitResult::SUCCESS) + if (result == ccf::kv::CommitResult::SUCCESS) { break; } } - catch (const kv::CompactedVersionConflict& e) + catch (const ccf::kv::CompactedVersionConflict& e) { // Retry on conflict continue; @@ -139,7 +139,7 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) // Notify that this thread has finished --*args->counter; - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; // Start a thread which continually compacts at the latest version, until all @@ -154,7 +154,7 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) struct CompactArgs { - kv::Store* kv_store; + ccf::kv::Store* kv_store; std::atomic* tx_counter; std::atomic* compact_state; }; @@ -162,10 +162,10 @@ DOCTEST_TEST_CASE("Concurrent kv access" * doctest::test_suite("concurrency")) static constexpr auto iterations = 20; for (auto i = 0; i < iterations; ++i) { - kv::Store kv_store; + ccf::kv::Store kv_store; auto consensus = std::make_shared(); kv_store.set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); // Keep atomic count of running threads @@ -254,13 +254,13 @@ DOCTEST_TEST_CASE( { // Many threads attempt to produce a chain of transactions pointing at the // previous write to a single key, at that key. - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto store_commit_term = 2; kv_store.initialise_term(store_commit_term); - using MapType = kv::Map; + using MapType = ccf::kv::Map; MapType map("public:foo"); constexpr size_t k = 42; @@ -268,7 +268,7 @@ DOCTEST_TEST_CASE( std::atomic conflict_count = 0; auto point_at_previous_write = [&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto sleep_time = std::chrono::microseconds(5); while (true) { @@ -293,13 +293,13 @@ DOCTEST_TEST_CASE( h->put(k, j); const auto result = tx.commit(); - if (result == kv::CommitResult::SUCCESS) + if (result == ccf::kv::CommitResult::SUCCESS) { // Succeeded break; } - DOCTEST_REQUIRE(result == kv::CommitResult::FAIL_CONFLICT); + DOCTEST_REQUIRE(result == ccf::kv::CommitResult::FAIL_CONFLICT); ++conflict_count; // Sleep before retrying @@ -310,7 +310,7 @@ DOCTEST_TEST_CASE( sleep_time = std::chrono::microseconds((size_t)(sleep_time.count() * factor)); } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; std::vector threads; diff --git a/src/kv/test/kv_dynamic_tables.cpp b/src/kv/test/kv_dynamic_tables.cpp index 04b2ccc993ff..4d7553be8817 100644 --- a/src/kv/test/kv_dynamic_tables.cpp +++ b/src/kv/test/kv_dynamic_tables.cpp @@ -8,17 +8,17 @@ struct MapTypes { - using StringString = kv::Map; - using NumNum = kv::Map; - using NumString = kv::Map; - using StringNum = kv::Map; + using StringString = ccf::kv::Map; + using NumNum = ccf::kv::Map; + using NumString = ccf::kv::Map; + using StringNum = ccf::kv::Map; }; TEST_CASE("Basic dynamic table" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto map_name = "mapA"; @@ -31,7 +31,7 @@ TEST_CASE("Basic dynamic table" * doctest::test_suite("dynamic")) auto handle = tx.rw(map_name); handle->put("foo", "bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -79,7 +79,7 @@ TEST_CASE("Basic dynamic table" * doctest::test_suite("dynamic")) auto a = tx.rw(map_name); a->put("foo", "baz"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); { auto check_tx = kv_store.create_tx(); @@ -118,9 +118,9 @@ TEST_CASE("Basic dynamic table" * doctest::test_suite("dynamic")) TEST_CASE("Dynamic table opacity" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto map_name = "dynamic_map"; @@ -138,7 +138,7 @@ TEST_CASE("Dynamic table opacity" * doctest::test_suite("dynamic")) { INFO("First transaction commits successfully"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -152,7 +152,7 @@ TEST_CASE("Dynamic table opacity" * doctest::test_suite("dynamic")) { INFO("Second transaction conflicts"); - REQUIRE(tx2.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); } { @@ -164,7 +164,7 @@ TEST_CASE("Dynamic table opacity" * doctest::test_suite("dynamic")) handle3->put("foo", "baz"); REQUIRE(handle3->get("foo").value() == "baz"); - REQUIRE(tx3.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx3.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -180,9 +180,9 @@ TEST_CASE("Dynamic table opacity" * doctest::test_suite("dynamic")) TEST_CASE( "Dynamic table visibility by version" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto map_name = "dynamic_map"; @@ -203,7 +203,7 @@ TEST_CASE( // tx3 takes a read dependency at an early version, before the map is visible auto handle3_static = tx3.rw(other_map); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); // Even after commit, the new map is not visible to tx3 because it is reading // from an earlier version @@ -218,9 +218,9 @@ TEST_CASE( TEST_CASE("Read only handles" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto dynamic_map_a = "dynamic_map_a"; @@ -253,7 +253,7 @@ TEST_CASE("Read only handles" * doctest::test_suite("dynamic")) a->put("foo", "bar"); b->put("foo", "baz"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -273,9 +273,9 @@ TEST_CASE("Read only handles" * doctest::test_suite("dynamic")) TEST_CASE("Mixed map dependencies" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto key = "foo"; @@ -285,7 +285,7 @@ TEST_CASE("Mixed map dependencies" * doctest::test_suite("dynamic")) auto tx = kv_store.create_tx(); auto handle = tx.rw(prior_map); handle->put(key, "bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } constexpr auto dynamic_map_a = "dynamic_map_a"; @@ -302,8 +302,8 @@ TEST_CASE("Mixed map dependencies" * doctest::test_suite("dynamic")) handle1->put(42, "hello"); handle2->put("hello", 42); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); } SUBCASE("Map creation blocked by standard conflict") @@ -326,8 +326,8 @@ TEST_CASE("Mixed map dependencies" * doctest::test_suite("dynamic")) dynamic_handle->put("hello world", 42); } - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); - REQUIRE(tx2.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); tx2 = kv_store.create_tx(); @@ -349,14 +349,14 @@ TEST_CASE("Mixed map dependencies" * doctest::test_suite("dynamic")) TEST_CASE("Dynamic map serialisation" * doctest::test_suite("dynamic")) { - auto consensus = std::make_shared(); - auto encryptor = std::make_shared(); - kv::Store kv_store; + auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; kv_store.set_encryptor(encryptor); kv_store.set_consensus(consensus); - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); const auto map_name = "new_map"; @@ -368,7 +368,7 @@ TEST_CASE("Dynamic map serialisation" * doctest::test_suite("dynamic")) auto tx = kv_store.create_tx(); auto handle = tx.rw(map_name); handle->put(key, value); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -377,7 +377,7 @@ TEST_CASE("Dynamic map serialisation" * doctest::test_suite("dynamic")) REQUIRE(latest_data.has_value()); REQUIRE( kv_store_target.deserialize(latest_data.value())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.rw(map_name); @@ -389,40 +389,40 @@ TEST_CASE("Dynamic map serialisation" * doctest::test_suite("dynamic")) TEST_CASE("Dynamic map snapshot serialisation" * doctest::test_suite("dynamic")) { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); constexpr auto map_name = "string_map"; - kv::Version snapshot_version; + ccf::kv::Version snapshot_version; INFO("Create maps in original store"); { auto tx1 = store.create_tx(); auto handle_1 = tx1.rw(map_name); handle_1->put("foo", "foo"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); auto tx2 = store.create_tx(); auto handle_2 = tx2.rw(map_name); handle_2->put("bar", "bar"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); snapshot_version = tx2.commit_version(); } INFO("Create snapshot of original store"); - std::unique_ptr snapshot = nullptr; + std::unique_ptr snapshot = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); snapshot = store.snapshot_unsafe_maps(snapshot_version); } auto serialised_snapshot = store.serialise_snapshot(std::move(snapshot)); INFO("Apply snapshot to create maps in new store"); { - kv::ConsensusHookPtrs hooks; - kv::Store new_store; + ccf::kv::ConsensusHookPtrs hooks; + ccf::kv::Store new_store; new_store.set_encryptor(encryptor); new_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks); @@ -442,9 +442,9 @@ TEST_CASE("Dynamic map snapshot serialisation" * doctest::test_suite("dynamic")) TEST_CASE("Mid rollback safety" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto map_name = "my_new_map"; @@ -457,7 +457,7 @@ TEST_CASE("Mid rollback safety" * doctest::test_suite("dynamic")) auto handle = tx.rw(map_name); handle->put("foo", "bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -481,16 +481,16 @@ TEST_CASE("Mid rollback safety" * doctest::test_suite("dynamic")) handle->put("foo", "baz"); const auto result = tx.commit(); - REQUIRE(result == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(result == ccf::kv::CommitResult::FAIL_CONFLICT); } } TEST_CASE( "Security domain is determined by map name" * doctest::test_suite("dynamic")) { - kv::Store kv_store; + ccf::kv::Store kv_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); { @@ -498,7 +498,7 @@ TEST_CASE( auto handle = tx.rw("public:foo"); handle->put("foo", "bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -506,7 +506,7 @@ TEST_CASE( auto handle = tx.rw("foo"); handle->put("hello", "world"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -530,9 +530,9 @@ TEST_CASE( TEST_CASE("Swapping dynamic maps" * doctest::test_suite("dynamic")) { - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); - kv::Store s1; + ccf::kv::Store s1; s1.set_encryptor(encryptor); { @@ -541,7 +541,7 @@ TEST_CASE("Swapping dynamic maps" * doctest::test_suite("dynamic")) auto v1 = tx.rw("bar"); v0->put("hello", "world"); v1->put(42, "everything"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -550,7 +550,7 @@ TEST_CASE("Swapping dynamic maps" * doctest::test_suite("dynamic")) auto v1 = tx.rw("baz"); v0->put("hello", "goodbye"); v1->put("saluton", 100); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -558,10 +558,10 @@ TEST_CASE("Swapping dynamic maps" * doctest::test_suite("dynamic")) auto tx = s1.create_tx(); auto v0 = tx.rw("public:source_state"); v0->put("store", "source"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } - kv::Store s2; + ccf::kv::Store s2; s2.set_encryptor(encryptor); // Ensure source store is at _at least_ the same version as source store @@ -571,7 +571,7 @@ TEST_CASE("Swapping dynamic maps" * doctest::test_suite("dynamic")) auto tx = s2.create_tx(); auto v0 = tx.rw("public:target_state"); v0->put("store", "target"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } s1.compact(s1.current_version()); diff --git a/src/kv/test/kv_serialisation.cpp b/src/kv/test/kv_serialisation.cpp index ce04f3001cdd..3baa402eec83 100644 --- a/src/kv/test/kv_serialisation.cpp +++ b/src/kv/test/kv_serialisation.cpp @@ -13,10 +13,10 @@ struct MapTypes { - using StringString = kv::Map; - using NumNum = kv::Map; - using NumString = kv::Map; - using StringNum = kv::Map; + using StringString = ccf::kv::Map; + using NumNum = ccf::kv::Map; + using NumString = ccf::kv::Map; + using StringNum = ccf::kv::Map; }; TEST_CASE( @@ -26,14 +26,14 @@ TEST_CASE( // No need for an encryptor here as all maps are public. Both serialisation // and deserialisation should succeed. - kv::Store kv_store; + ccf::kv::Store kv_store; - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); kv_store.set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); MapTypes::StringString map("public:pub_map"); @@ -53,7 +53,7 @@ TEST_CASE( handle0->put(k1, v1); handle0->put(k2, v1); handle0->put(k3, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto first_version = kv_store.current_version(); @@ -68,7 +68,7 @@ TEST_CASE( // no change to k1, write to k2, remove k3 handle0->put(k2, v2); handle0->remove(k3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto second_version = kv_store.current_version(); @@ -80,7 +80,7 @@ TEST_CASE( INFO("Deserialise first transaction in target store"); REQUIRE( kv_store_target.deserialize(first_tx_serialised.value())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.ro(map); @@ -105,7 +105,7 @@ TEST_CASE( INFO("Deserialise second transaction in target store"); REQUIRE( kv_store_target.deserialize(second_tx_serialised.value())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.ro(map); @@ -130,13 +130,13 @@ TEST_CASE( "Serialise/deserialise private map only" * doctest::test_suite("serialisation")) { - auto consensus = std::make_shared(); - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); - kv::Store kv_store; + ccf::kv::Store kv_store; kv_store.set_consensus(consensus); - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); SUBCASE( @@ -145,7 +145,7 @@ TEST_CASE( auto tx = kv_store.create_tx(); auto handle0 = tx.rw("priv_map"); handle0->put("privk1", "privv1"); - REQUIRE_THROWS_AS(tx.commit(), kv::KvSerialiserException); + REQUIRE_THROWS_AS(tx.commit(), ccf::kv::KvSerialiserException); } SUBCASE("Commit private transaction with encryptor") @@ -156,7 +156,7 @@ TEST_CASE( auto tx = kv_store.create_tx(); auto handle0 = tx.rw("priv_map"); handle0->put("privk1", "privv1"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Deserialise transaction in target store"); @@ -165,7 +165,7 @@ TEST_CASE( REQUIRE(latest_data.has_value()); REQUIRE( kv_store_target.deserialize(latest_data.value())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.rw("priv_map"); @@ -178,17 +178,17 @@ TEST_CASE( "Serialise/deserialise private map and public maps" * doctest::test_suite("serialisation")) { - auto consensus = std::make_shared(); - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); - kv::Store kv_store; + ccf::kv::Store kv_store; kv_store.set_consensus(consensus); kv_store.set_encryptor(encryptor); constexpr auto priv_map = "priv_map"; constexpr auto pub_map = "public:pub_map"; - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); INFO("Commit to public and private map in source store"); @@ -200,7 +200,7 @@ TEST_CASE( handle_priv->put("privk1", "privv1"); handle_pub->put("pubk1", "pubv1"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Deserialise transaction in target store"); @@ -209,7 +209,7 @@ TEST_CASE( REQUIRE(latest_data.has_value()); REQUIRE( kv_store_target.deserialize(latest_data.value())->apply() != - kv::ApplyResult::FAIL); + ccf::kv::ApplyResult::FAIL); auto tx_target = kv_store_target.create_tx(); auto handle_priv = tx_target.rw(priv_map); @@ -223,14 +223,14 @@ TEST_CASE( TEST_CASE( "Serialise/deserialise removed keys" * doctest::test_suite("serialisation")) { - auto consensus = std::make_shared(); - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); - kv::Store kv_store; + ccf::kv::Store kv_store; kv_store.set_consensus(consensus); kv_store.set_encryptor(encryptor); - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); INFO("Commit new keys in source store and deserialise in target store"); @@ -241,13 +241,13 @@ TEST_CASE( handle->put("key1", "value1"); handle2->put("key2", "value2"); handle2->put("key3", "value3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto latest_data = consensus->get_latest_data(); REQUIRE(latest_data.has_value()); REQUIRE( kv_store_target.deserialize(latest_data.value())->apply() != - kv::ApplyResult::FAIL); + ccf::kv::ApplyResult::FAIL); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.rw("map"); @@ -281,7 +281,7 @@ TEST_CASE( handle_->remove("key3"); handle_->put("key3", "value3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Make sure keys have been marked as deleted in source store auto tx2 = kv_store.create_tx(); @@ -296,7 +296,7 @@ TEST_CASE( REQUIRE(latest_data.has_value()); REQUIRE( kv_store_target.deserialize(latest_data.value())->apply() != - kv::ApplyResult::FAIL); + ccf::kv::ApplyResult::FAIL); auto tx_target = kv_store_target.create_tx(); auto handle_target = tx_target.rw("map"); @@ -322,7 +322,7 @@ DECLARE_JSON_REQUIRED_FIELDS(CustomClass, s, n); // Not really intended to be extended, but lets us use the BlitSerialiser for // this specific type -namespace kv::serialisers +namespace ccf::kv::serialisers { template <> struct BlitSerialiser @@ -373,11 +373,12 @@ struct CustomSerialiser static constexpr auto size_of_n = 8; static constexpr auto size_of_size_of_s = 8; - static kv::serialisers::SerialisedEntry to_serialised(const CustomClass& cc) + static ccf::kv::serialisers::SerialisedEntry to_serialised( + const CustomClass& cc) { const auto s_size = cc.s.size(); const auto total_size = size_of_n + size_of_size_of_s + s_size; - kv::serialisers::SerialisedEntry serialised(total_size); + ccf::kv::serialisers::SerialisedEntry serialised(total_size); uint8_t* data = serialised.data(); memcpy(data, (const uint8_t*)&cc.n, size_of_n); data += size_of_n; @@ -388,7 +389,7 @@ struct CustomSerialiser } static CustomClass from_serialised( - const kv::serialisers::SerialisedEntry& ser) + const ccf::kv::serialisers::SerialisedEntry& ser) { CustomClass cc; const uint8_t* data = ser.data(); @@ -405,7 +406,7 @@ struct CustomSerialiser struct CustomJsonSerialiser { - using Bytes = kv::serialisers::SerialisedEntry; + using Bytes = ccf::kv::serialisers::SerialisedEntry; static Bytes to_serialised(const CustomClass& c) { @@ -439,7 +440,7 @@ struct VPrefix template struct CustomVerboseDumbSerialiser { - using Bytes = kv::serialisers::SerialisedEntry; + using Bytes = ccf::kv::serialisers::SerialisedEntry; static Bytes to_serialised(const CustomClass& c) { @@ -471,25 +472,26 @@ struct CustomVerboseDumbSerialiser } }; -using JsonSerialisedMap = kv::JsonSerialisedMap; -using RawCopySerialisedMap = kv::RawCopySerialisedMap; -using MixSerialisedMapB = kv::TypedMap< +using JsonSerialisedMap = ccf::kv::JsonSerialisedMap; +using RawCopySerialisedMap = + ccf::kv::RawCopySerialisedMap; +using MixSerialisedMapB = ccf::kv::TypedMap< CustomClass, CustomClass, - kv::serialisers::JsonSerialiser, - kv::serialisers::BlitSerialiser>; + ccf::kv::serialisers::JsonSerialiser, + ccf::kv::serialisers::BlitSerialiser>; // SNIPPET_START: CustomSerialisedMap definition -using CustomSerialisedMap = - kv::TypedMap; +using CustomSerialisedMap = ccf::kv:: + TypedMap; // SNIPPET_END: CustomSerialisedMap definition -using CustomJsonMap = kv::TypedMap< +using CustomJsonMap = ccf::kv::TypedMap< CustomClass, CustomClass, CustomJsonSerialiser, CustomJsonSerialiser>; -using VerboseSerialisedMap = kv::TypedMap< +using VerboseSerialisedMap = ccf::kv::TypedMap< CustomClass, CustomClass, CustomVerboseDumbSerialiser, @@ -505,8 +507,8 @@ TEST_CASE_TEMPLATE( CustomJsonMap, VerboseSerialisedMap) { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapType map("public:map"); @@ -519,7 +521,7 @@ TEST_CASE_TEMPLATE( INFO("Serialise/Deserialise 2 kv stores"); { - kv::Store kv_store2; + ccf::kv::Store kv_store2; kv_store2.set_encryptor(encryptor); MapType map2("public:map"); @@ -533,10 +535,10 @@ TEST_CASE_TEMPLATE( tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); - REQUIRE(kv_store2.deserialize(data)->apply() == kv::ApplyResult::PASS); + REQUIRE(kv_store2.deserialize(data)->apply() == ccf::kv::ApplyResult::PASS); auto tx2 = kv_store2.create_tx(); auto handle2 = tx2.rw(map2); @@ -564,11 +566,11 @@ TEST_CASE("nlohmann (de)serialisation" * doctest::test_suite("serialisation")) SUBCASE("baseline") { - auto consensus = std::make_shared(); - using Table = kv::Map, std::string>; - kv::Store s0, s1; + auto consensus = std::make_shared(); + using Table = ccf::kv::Map, std::string>; + ccf::kv::Store s0, s1; s0.set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); s0.set_encryptor(encryptor); s1.set_encryptor(encryptor); @@ -576,21 +578,22 @@ TEST_CASE("nlohmann (de)serialisation" * doctest::test_suite("serialisation")) auto tx = s0.create_tx(); tx.rw(t)->put(k1, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto latest_data = consensus->get_latest_data(); REQUIRE(latest_data.has_value()); REQUIRE( - s1.deserialize(latest_data.value())->apply() != kv::ApplyResult::FAIL); + s1.deserialize(latest_data.value())->apply() != + ccf::kv::ApplyResult::FAIL); } SUBCASE("nlohmann") { - auto consensus = std::make_shared(); - using Table = kv::Map; - kv::Store s0, s1; + auto consensus = std::make_shared(); + using Table = ccf::kv::Map; + ccf::kv::Store s0, s1; s0.set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); s0.set_encryptor(encryptor); s1.set_encryptor(encryptor); @@ -599,12 +602,13 @@ TEST_CASE("nlohmann (de)serialisation" * doctest::test_suite("serialisation")) auto tx = s0.create_tx(); tx.rw(t)->put(k0, v0); tx.rw(t)->put(k1, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto latest_data = consensus->get_latest_data(); REQUIRE(latest_data.has_value()); REQUIRE( - s1.deserialize(latest_data.value())->apply() != kv::ApplyResult::FAIL); + s1.deserialize(latest_data.value())->apply() != + ccf::kv::ApplyResult::FAIL); } } @@ -613,7 +617,7 @@ struct NonSerialisable struct NonSerialiser { - using Bytes = kv::serialisers::SerialisedEntry; + using Bytes = ccf::kv::serialisers::SerialisedEntry; static Bytes to_serialised(const NonSerialisable& ns) { @@ -628,23 +632,23 @@ struct NonSerialiser TEST_CASE("Exceptional serdes" * doctest::test_suite("serialisation")) { - auto encryptor = std::make_shared(); - auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); - kv::Store store; + ccf::kv::Store store; store.set_consensus(consensus); store.set_encryptor(encryptor); - kv::TypedMap< + ccf::kv::TypedMap< NonSerialisable, size_t, NonSerialiser, - kv::serialisers::JsonSerialiser> + ccf::kv::serialisers::JsonSerialiser> bad_map_k("bad_map_k"); - kv::TypedMap< + ccf::kv::TypedMap< size_t, NonSerialisable, - kv::serialisers::JsonSerialiser, + ccf::kv::serialisers::JsonSerialiser, NonSerialiser> bad_map_v("bad_map_v"); @@ -665,21 +669,21 @@ TEST_CASE( "Serialise/deserialise maps with claims" * doctest::test_suite("serialisation")) { - auto consensus = std::make_shared(); - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + auto encryptor = std::make_shared(); - kv::Store kv_store; + ccf::kv::Store kv_store; kv_store.set_consensus(consensus); kv_store.set_encryptor(encryptor); constexpr auto priv_map = "priv_map"; constexpr auto pub_map = "public:pub_map"; - kv::Store kv_store_target; + ccf::kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); ccf::ClaimsDigest claims_digest; - claims_digest.set(crypto::Sha256Hash("claim text")); + claims_digest.set(ccf::crypto::Sha256Hash("claim text")); INFO("Commit to source store, including claims"); { @@ -690,7 +694,7 @@ TEST_CASE( handle_priv->put("privk1", "privv1"); handle_pub->put("pubk1", "pubv1"); - REQUIRE(tx.commit(claims_digest) == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit(claims_digest) == ccf::kv::CommitResult::SUCCESS); } INFO("Deserialise transaction in target store and extract claims"); @@ -698,7 +702,7 @@ TEST_CASE( const auto latest_data = consensus->get_latest_data(); REQUIRE(latest_data.has_value()); auto wrapper = kv_store_target.deserialize(latest_data.value()); - REQUIRE(wrapper->apply() != kv::ApplyResult::FAIL); + REQUIRE(wrapper->apply() != ccf::kv::ApplyResult::FAIL); auto deserialised_claims = wrapper->consume_claims_digest(); REQUIRE(claims_digest == deserialised_claims); diff --git a/src/kv/test/kv_snapshot.cpp b/src/kv/test/kv_snapshot.cpp index 0b139d6feef7..03f59e32fde2 100644 --- a/src/kv/test/kv_snapshot.cpp +++ b/src/kv/test/kv_snapshot.cpp @@ -10,10 +10,10 @@ struct MapTypes { - using StringString = kv::Map; - using NumNum = kv::Map; - using StringValue = kv::Value; - using StringSet = kv::Set; + using StringString = ccf::kv::Map; + using NumNum = ccf::kv::Map; + using StringValue = ccf::kv::Value; + using StringSet = ccf::kv::Set; }; MapTypes::StringString string_map("public:string_map"); @@ -21,12 +21,12 @@ MapTypes::NumNum num_map("public:num_map"); TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); - kv::Version first_snapshot_version = kv::NoVersion; - kv::Version second_snapshot_version = kv::NoVersion; + ccf::kv::Version first_snapshot_version = ccf::kv::NoVersion; + ccf::kv::Version second_snapshot_version = ccf::kv::NoVersion; INFO("Apply transactions to original store"); { @@ -36,7 +36,7 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) handle_1s->put("baz", "hello"); auto handle_1n = tx1.rw(num_map); handle_1n->put(42, 100); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); first_snapshot_version = tx1.commit_version(); auto tx2 = store.create_tx(); @@ -44,7 +44,7 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) handle_2s->remove("baz"); auto handle_2n = tx2.rw(num_map); handle_2n->put(42, 123); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); second_snapshot_version = tx2.commit_version(); auto tx3 = store.create_tx(); @@ -53,9 +53,10 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) // Do not commit tx3 } - std::unique_ptr first_snapshot = nullptr; + std::unique_ptr first_snapshot = + nullptr; { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); first_snapshot = store.snapshot_unsafe_maps(first_snapshot_version); } auto first_serialised_snapshot = @@ -63,16 +64,16 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) INFO("Apply snapshot at 1 to new store"); { - kv::Store new_store; + ccf::kv::Store new_store; new_store.set_encryptor(encryptor); - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; REQUIRE_EQ( new_store.deserialise_snapshot( first_serialised_snapshot.data(), first_serialised_snapshot.size(), hooks), - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); REQUIRE_EQ(new_store.current_version(), first_snapshot_version); auto tx1 = new_store.create_tx(); @@ -113,10 +114,10 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) } } - std::unique_ptr second_snapshot = + std::unique_ptr second_snapshot = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); second_snapshot = store.snapshot_unsafe_maps(second_snapshot_version); } auto second_serialised_snapshot = @@ -124,11 +125,11 @@ TEST_CASE("Simple snapshot" * doctest::test_suite("snapshot")) INFO("Apply snapshot at 2 to new store"); { - kv::Store new_store; + ccf::kv::Store new_store; new_store.set_encryptor(encryptor); - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; new_store.deserialise_snapshot( second_serialised_snapshot.data(), second_serialised_snapshot.size(), @@ -198,14 +199,14 @@ TEST_CASE("Old snapshots" * doctest::test_suite("snapshot")) "AAAAEQAAAAAAAABwdWJsaWM6c3RyaW5nX21hcAIAAAAAAAAAKAAAAAAAAAAFAAAAAAAAACJm" "b28iAAAADQAAAAAAAAABAAAAAAAAACJiYXIiAAAA"; } - const auto raw_snapshot = crypto::raw_from_b64(raw_snapshot_b64); + const auto raw_snapshot = ccf::crypto::raw_from_b64(raw_snapshot_b64); - kv::Store new_store; + ccf::kv::Store new_store; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); new_store.set_encryptor(encryptor); - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; new_store.deserialise_snapshot( raw_snapshot.data(), raw_snapshot.size(), hooks); @@ -255,37 +256,37 @@ TEST_CASE( "Commit transaction while applying snapshot" * doctest::test_suite("snapshot")) { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); MapTypes::StringString string_map("public:string_map"); - kv::Version snapshot_version = kv::NoVersion; + ccf::kv::Version snapshot_version = ccf::kv::NoVersion; INFO("Apply transactions to original store"); { auto tx1 = store.create_tx(); auto handle_1 = tx1.rw("public:string_map"); handle_1->put("foo", "foo"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); // Committed at 1 + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); // Committed at 1 auto tx2 = store.create_tx(); auto handle_2 = tx2.rw("public:string_map"); handle_2->put("bar", "bar"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); // Committed at 2 + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); // Committed at 2 snapshot_version = tx2.commit_version(); } - std::unique_ptr snapshot = nullptr; + std::unique_ptr snapshot = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); snapshot = store.snapshot_unsafe_maps(snapshot_version); } auto serialised_snapshot = store.serialise_snapshot(std::move(snapshot)); INFO("Apply snapshot while committing a transaction"); { - kv::Store new_store; + ccf::kv::Store new_store; new_store.set_encryptor(encryptor); auto tx = new_store.create_tx(); @@ -293,33 +294,33 @@ TEST_CASE( handle->put("in", "flight"); // tx is not committed until the snapshot is deserialised - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; new_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks); // Transaction conflicts as snapshot was applied while transaction was in // flight - REQUIRE(tx.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(tx.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); // Try again auto tx2 = new_store.create_tx(); auto handle2 = tx2.rw("public:string_map"); handle2->put("baz", "baz"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); } } TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); constexpr auto string_map = "public:string_map"; constexpr auto string_value = "public:string_value"; constexpr auto string_set = "public:string_set"; - kv::Version snapshot_version = kv::NoVersion; + ccf::kv::Version snapshot_version = ccf::kv::NoVersion; using MapWrite = MapTypes::StringString::Write; using ValueWrite = MapTypes::StringValue::Write; @@ -332,27 +333,27 @@ TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) std::vector global_set_writes; auto map_hook = - [&](kv::Version v, const MapWrite& w) -> kv::ConsensusHookPtr { + [&](ccf::kv::Version v, const MapWrite& w) -> ccf::kv::ConsensusHookPtr { local_map_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_map_hook = [&](kv::Version v, const MapWrite& w) { + auto global_map_hook = [&](ccf::kv::Version v, const MapWrite& w) { global_map_writes.push_back(w); }; auto value_hook = - [&](kv::Version v, const ValueWrite& w) -> kv::ConsensusHookPtr { + [&](ccf::kv::Version v, const ValueWrite& w) -> ccf::kv::ConsensusHookPtr { local_value_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_value_hook = [&](kv::Version v, const ValueWrite& w) { + auto global_value_hook = [&](ccf::kv::Version v, const ValueWrite& w) { global_value_writes.push_back(w); }; auto set_hook = - [&](kv::Version v, const SetWrite& w) -> kv::ConsensusHookPtr { + [&](ccf::kv::Version v, const SetWrite& w) -> ccf::kv::ConsensusHookPtr { local_set_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_set_hook = [&](kv::Version v, const SetWrite& w) { + auto global_set_hook = [&](ccf::kv::Version v, const SetWrite& w) { global_set_writes.push_back(w); }; @@ -368,7 +369,7 @@ TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) auto set_handle = tx.rw(string_set); set_handle->insert("foo"); set_handle->insert("bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); // Committed at 1 + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Committed at 1 } { @@ -382,19 +383,19 @@ TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) auto set_handle = tx.rw(string_set); set_handle->insert("baz"); set_handle->remove("bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); // Committed at 2 + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Committed at 2 snapshot_version = tx.commit_version(); } } - std::unique_ptr snapshot = nullptr; + std::unique_ptr snapshot = nullptr; { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); snapshot = store.snapshot_unsafe_maps(snapshot_version); } auto serialised_snapshot = store.serialise_snapshot(std::move(snapshot)); - kv::Store new_store; + ccf::kv::Store new_store; new_store.set_encryptor(encryptor); MapTypes::StringString new_string_map(string_map); @@ -419,7 +420,7 @@ TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) { INFO("Deserialise snapshot"); { - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; new_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks); } @@ -518,15 +519,15 @@ TEST_CASE("Commit hooks with snapshot" * doctest::test_suite("snapshot")) value_handle->clear(); auto set_handle = tx.rw(string_set); set_handle->clear(); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); snapshot_version = tx.commit_version(); { - kv::ScopedStoreMapsLock maps_lock(&store); + ccf::kv::ScopedStoreMapsLock maps_lock(&store); snapshot = store.snapshot_unsafe_maps(snapshot_version); } serialised_snapshot = store.serialise_snapshot(std::move(snapshot)); - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; new_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks); diff --git a/src/kv/test/kv_test.cpp b/src/kv/test/kv_test.cpp index 4c2a7f3783fe..ad35888def7e 100644 --- a/src/kv/test/kv_test.cpp +++ b/src/kv/test/kv_test.cpp @@ -23,19 +23,19 @@ struct MapTypes { - using StringString = kv::Map; - using NumNum = kv::Map; - using NumString = kv::Map; - using StringNum = kv::Map; - using UntypedMap = kv::untyped::Map; + using StringString = ccf::kv::Map; + using NumNum = ccf::kv::Map; + using NumString = ccf::kv::Map; + using StringNum = ccf::kv::Map; + using UntypedMap = ccf::kv::untyped::Map; }; TEST_CASE("Map name parsing") { - using SD = kv::SecurityDomain; - using AC = kv::AccessCategory; + using SD = ccf::kv::SecurityDomain; + using AC = ccf::kv::AccessCategory; - auto parse = kv::parse_map_name; + auto parse = ccf::kv::parse_map_name; auto mp = std::make_pair; REQUIRE(parse("foo") == mp(SD::PRIVATE, AC::APPLICATION)); @@ -58,8 +58,8 @@ TEST_CASE("Map name parsing") TEST_CASE("Reads/writes and deletions") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -71,7 +71,7 @@ TEST_CASE("Reads/writes and deletions") INFO("Start empty transaction"); { auto tx = kv_store.create_tx(); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE_THROWS_AS(tx.commit(), std::logic_error); } @@ -89,7 +89,7 @@ TEST_CASE("Reads/writes and deletions") REQUIRE(va.has_value()); REQUIRE(va.value() == v1); REQUIRE(!handle->get_version_of_previous_write(k).has_value()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto commit_v = kv_store.current_version(); @@ -105,7 +105,7 @@ TEST_CASE("Reads/writes and deletions") const auto ver = handle->get_version_of_previous_write(k); REQUIRE(ver.has_value()); REQUIRE(ver.value() == commit_v); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Remove keys"); @@ -128,7 +128,7 @@ TEST_CASE("Reads/writes and deletions") REQUIRE(!va.has_value()); handle->put(k, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -147,7 +147,7 @@ TEST_CASE("Reads/writes and deletions") handle->put(k, v1); auto va = handle->get_globally_committed(k); REQUIRE(!va.has_value()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -156,7 +156,7 @@ TEST_CASE("Reads/writes and deletions") REQUIRE(handle2->has(k)); handle2->remove(k); REQUIRE(!handle2->has(k)); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -171,13 +171,13 @@ TEST_CASE("Reads/writes and deletions") TEST_CASE("sets and values") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); { - INFO("kv::Set"); - using Set = kv::Set; + INFO("ccf::kv::Set"); + using Set = ccf::kv::Set; Set set("public:set"); constexpr auto k1 = "key1"; constexpr auto k2 = "key2"; @@ -209,7 +209,7 @@ TEST_CASE("sets and values") REQUIRE(!set_handle->contains_globally_committed(k1)); REQUIRE(!set_handle->contains_globally_committed(k2)); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -232,7 +232,7 @@ TEST_CASE("sets and values") REQUIRE(std_set.size() == 1); REQUIRE(std_set.find(k1) != std_set.end()); REQUIRE(std_set.find(k2) == std_set.end()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -275,7 +275,7 @@ TEST_CASE("sets and values") REQUIRE(set_handle->contains_globally_committed(k1)); REQUIRE(!set_handle->contains_globally_committed(k2)); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -305,12 +305,13 @@ TEST_CASE("sets and values") std::vector local_writes; std::vector global_writes; - auto map_hook = - [&](kv::Version v, const Set::Write& w) -> kv::ConsensusHookPtr { + auto map_hook = [&]( + ccf::kv::Version v, + const Set::Write& w) -> ccf::kv::ConsensusHookPtr { local_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_hook = [&](kv::Version v, const Set::Write& w) { + auto global_hook = [&](ccf::kv::Version v, const Set::Write& w) { global_writes.push_back(w); }; @@ -326,7 +327,7 @@ TEST_CASE("sets and values") set_handle->insert(k1); set_handle->insert(k2); set_handle->remove(k2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(global_writes.size() == 0); REQUIRE(local_writes.size() == 1); @@ -345,7 +346,7 @@ TEST_CASE("sets and values") auto set_handle = tx.rw(set); set_handle->remove(k1); set_handle->insert(k2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(local_writes.size() == 1); const auto& latest_writes = local_writes.front(); @@ -362,7 +363,7 @@ TEST_CASE("sets and values") auto tx = kv_store.create_tx(); auto set_handle = tx.rw(set); set_handle->insert(k1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); @@ -383,8 +384,8 @@ TEST_CASE("sets and values") } { - INFO("kv::Value"); - using Value = kv::Value; + INFO("ccf::kv::Value"); + using Value = ccf::kv::Value; Value val1("public:value1"); Value val2("public:value2"); @@ -415,7 +416,7 @@ TEST_CASE("sets and values") h2->clear(); REQUIRE(!h2->has()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -429,7 +430,7 @@ TEST_CASE("sets and values") auto h2 = tx.rw(val2); REQUIRE(!h2->has()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -441,7 +442,7 @@ TEST_CASE("sets and values") h1->clear(); REQUIRE(!h1->has()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -452,12 +453,13 @@ TEST_CASE("sets and values") std::vector local_writes; std::vector global_writes; - auto map_hook = - [&](kv::Version v, const Value::Write& w) -> kv::ConsensusHookPtr { + auto map_hook = [&]( + ccf::kv::Version v, + const Value::Write& w) -> ccf::kv::ConsensusHookPtr { local_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_hook = [&](kv::Version v, const Value::Write& w) { + auto global_hook = [&](ccf::kv::Version v, const Value::Write& w) { global_writes.push_back(w); }; @@ -470,7 +472,7 @@ TEST_CASE("sets and values") { auto tx = kv_store.create_tx(); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(local_writes.size() == 0); // Commit without puts } @@ -479,7 +481,7 @@ TEST_CASE("sets and values") auto h1 = tx.rw(val1); h1->put(v1); h1->put(v2); // Override previous value - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(global_writes.size() == 0); REQUIRE(local_writes.size() == 1); @@ -492,7 +494,7 @@ TEST_CASE("sets and values") auto tx = kv_store.create_tx(); auto h1 = tx.rw(val1); h1->clear(); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(local_writes.size() == 1); auto latest_writes = local_writes.front(); @@ -507,7 +509,7 @@ TEST_CASE("sets and values") auto tx = kv_store.create_tx(); auto h1 = tx.rw(val1); h1->put(v3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); @@ -524,10 +526,10 @@ TEST_CASE("sets and values") // Sanity check that transactions can handle a mix of maps, sets, and values INFO("Mixed"); - using TMap = kv::Map; - using TSet = kv::Set; - kv::Value map_name_val("public:map_name"); - kv::Value set_name_val("public:set_name"); + using TMap = ccf::kv::Map; + using TSet = ccf::kv::Set; + ccf::kv::Value map_name_val("public:map_name"); + ccf::kv::Value set_name_val("public:set_name"); constexpr auto n_entries = 10; @@ -554,7 +556,7 @@ TEST_CASE("sets and values") REQUIRE(map_handle->size() == n_entries); REQUIRE(set_handle->size() == n_entries); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -594,16 +596,16 @@ TEST_CASE("sets and values") REQUIRE(map_handle->size() == n_entries); REQUIRE(set_handle->size() == n_entries); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } } struct CustomUnitCreator { - static kv::serialisers::SerialisedEntry get() + static ccf::kv::serialisers::SerialisedEntry get() { - kv::serialisers::SerialisedEntry e; + ccf::kv::serialisers::SerialisedEntry e; for (size_t i = 0; i < 42; ++i) { @@ -624,16 +626,16 @@ TEST_CASE("serialisation of Unit type") const auto v3 = "saluton"; { - INFO("The default unit type allows migration to/from a kv::Map"); + INFO("The default unit type allows migration to/from a ccf::kv::Map"); - using TValue = kv::RawCopySerialisedValue; - using TValueEquivalent = kv::RawCopySerialisedMap; + using TValue = ccf::kv::RawCopySerialisedValue; + using TValueEquivalent = ccf::kv::RawCopySerialisedMap; - using TSet = kv::RawCopySerialisedSet; - using TSetEquivalent = kv::RawCopySerialisedMap; + using TSet = ccf::kv::RawCopySerialisedSet; + using TSetEquivalent = ccf::kv::RawCopySerialisedMap; - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); { @@ -648,7 +650,7 @@ TEST_CASE("serialisation of Unit type") set_handle->put(v2, 1); // Will be visible in TSet handle, but would be // written with different value - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -665,7 +667,7 @@ TEST_CASE("serialisation of Unit type") set_handle->insert(v2); set_handle->insert(v3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -685,68 +687,68 @@ TEST_CASE("serialisation of Unit type") REQUIRE(set_handle->has(v3)); REQUIRE(*set_handle->get(v3) == 0); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } { INFO("Custom UnitCreators produce distinct ledger entries"); - using ValueA = kv::TypedValue< + using ValueA = ccf::kv::TypedValue< std::string, - kv::serialisers::BlitSerialiser, - kv::serialisers::ZeroBlitUnitCreator>; + ccf::kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::ZeroBlitUnitCreator>; std::vector entry_a; { - auto consensus = std::make_shared(); - kv::Store kv_store; - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); kv_store.set_consensus(consensus); auto tx = kv_store.create_tx(); auto val_handle = tx.rw(value_name); val_handle->put(v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto e = consensus->get_latest_data(); REQUIRE(e.has_value()); entry_a = e.value(); } - using ValueB = kv::TypedValue< + using ValueB = ccf::kv::TypedValue< std::string, - kv::serialisers::BlitSerialiser, - kv::serialisers::EmptyUnitCreator>; + ccf::kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::EmptyUnitCreator>; std::vector entry_b; { - auto consensus = std::make_shared(); - kv::Store kv_store; - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); kv_store.set_consensus(consensus); auto tx = kv_store.create_tx(); auto val_handle = tx.rw(value_name); val_handle->put(v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto e = consensus->get_latest_data(); REQUIRE(e.has_value()); entry_b = e.value(); } - using ValueC = kv::TypedValue< + using ValueC = ccf::kv::TypedValue< std::string, - kv::serialisers::BlitSerialiser, + ccf::kv::serialisers::BlitSerialiser, CustomUnitCreator>; std::vector entry_c; { - auto consensus = std::make_shared(); - kv::Store kv_store; - auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); kv_store.set_consensus(consensus); auto tx = kv_store.create_tx(); auto val_handle = tx.rw(value_name); val_handle->put(v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); const auto e = consensus->get_latest_data(); REQUIRE(e.has_value()); entry_c = e.value(); @@ -760,8 +762,8 @@ TEST_CASE("serialisation of Unit type") TEST_CASE("multiple handles") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::NumString map("public:map"); @@ -796,8 +798,8 @@ TEST_CASE("multiple handles") TEST_CASE("clear") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -812,7 +814,7 @@ TEST_CASE("clear") auto handle = tx.rw(map); handle->put(k1, v); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } SUBCASE("Basic") @@ -831,7 +833,7 @@ TEST_CASE("clear") REQUIRE(!handle->has(k1)); REQUIRE(!handle->has(k2)); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -841,7 +843,7 @@ TEST_CASE("clear") REQUIRE(!handle->has(k1)); REQUIRE(!handle->has(k2)); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } @@ -855,17 +857,17 @@ TEST_CASE("clear") auto tx2 = kv_store.create_tx(); auto handle2 = tx2.rw(map); handle2->put(k2, v); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); INFO("clear() conflicts and must be retried"); - REQUIRE(tx1.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); } } TEST_CASE("get_version_of_previous_write") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -883,7 +885,7 @@ TEST_CASE("get_version_of_previous_write") handle->put(k1, v1); handle->put(k2, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto first_version = kv_store.current_version(); @@ -894,7 +896,7 @@ TEST_CASE("get_version_of_previous_write") handle->put(k1, v2); handle->remove(k2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto second_version = kv_store.current_version(); @@ -966,7 +968,7 @@ TEST_CASE("get_version_of_previous_write") } // This conflicts with tx_other so is not committed - REQUIRE(tx.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(tx.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); } } @@ -990,8 +992,8 @@ TEST_CASE("get_version_of_previous_write") TEST_CASE("size") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -1021,7 +1023,7 @@ TEST_CASE("size") handle->put(k1, v); REQUIRE(handle->size() == 2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1052,7 +1054,7 @@ TEST_CASE("size") } REQUIRE(handle->size() == 0); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1082,15 +1084,15 @@ TEST_CASE("size") }); REQUIRE(claimed_size == manual_size); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } } TEST_CASE("foreach") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -1138,7 +1140,7 @@ TEST_CASE("foreach") auto handle = tx.rw(map); handle->put("key1", "value1"); handle->put("key2", "value2"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx2 = kv_store.create_tx(); auto handle2 = tx2.rw(map); @@ -1154,7 +1156,7 @@ TEST_CASE("foreach") auto handle = tx.rw(map); handle->put("key1", "value1"); handle->put("key2", "value2"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx2 = kv_store.create_tx(); auto handle2 = tx2.rw(map); @@ -1175,14 +1177,14 @@ TEST_CASE("foreach") handle->put("key1", "value1"); handle->put("key2", "value2"); handle->put("key3", "value3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->remove("key1"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1227,7 +1229,7 @@ TEST_CASE("foreach") // never see the third) }); REQUIRE(ctr == 2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1273,12 +1275,12 @@ TEST_CASE("foreach") template struct NoDeserialise { - static kv::serialisers::SerialisedEntry to_serialised(const T& t) + static ccf::kv::serialisers::SerialisedEntry to_serialised(const T& t) { - return kv::serialisers::JsonSerialiser::to_serialised(t); + return ccf::kv::serialisers::JsonSerialiser::to_serialised(t); } - static T from_serialised(const kv::serialisers::SerialisedEntry& s) + static T from_serialised(const ccf::kv::serialisers::SerialisedEntry& s) { throw std::logic_error("This deserialiser should not be called"); } @@ -1286,14 +1288,14 @@ struct NoDeserialise TEST_CASE("foreach_key") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); - kv::MapSerialisedWith< + ccf::kv::MapSerialisedWith< std::string, std::string, - kv::serialisers::JsonSerialiser, + ccf::kv::serialisers::JsonSerialiser, NoDeserialise> map("public:map"); @@ -1303,7 +1305,7 @@ TEST_CASE("foreach_key") handle->put("k1", "v1"); handle->put("k2", "v2"); handle->put("k3", "v3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1323,15 +1325,15 @@ TEST_CASE("foreach_key") TEST_CASE("foreach_value") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); - kv::MapSerialisedWith< + ccf::kv::MapSerialisedWith< std::string, std::string, NoDeserialise, - kv::serialisers::JsonSerialiser> + ccf::kv::serialisers::JsonSerialiser> map("public:map"); { @@ -1340,7 +1342,7 @@ TEST_CASE("foreach_value") handle->put("k1", "v1"); handle->put("k2", "v2"); handle->put("k3", "v3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -1359,8 +1361,8 @@ TEST_CASE("foreach_value") TEST_CASE("Modifications during foreach iteration") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::NumString map("public:map"); @@ -1379,7 +1381,7 @@ TEST_CASE("Modifications during foreach iteration") handle->put(i, value1); } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } auto tx = kv_store.create_tx(); @@ -1622,8 +1624,8 @@ TEST_CASE("Modifications during foreach iteration") TEST_CASE("Read-only tx") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -1641,7 +1643,7 @@ TEST_CASE("Read-only tx") auto va = handle->get(k); REQUIRE(va.has_value()); REQUIRE(va.value() == v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Do only reads with an overpowered Tx"); @@ -1697,8 +1699,8 @@ TEST_CASE("Read-only tx") TEST_CASE("Rollback and compact") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -1711,13 +1713,13 @@ TEST_CASE("Rollback and compact") auto tx2 = kv_store.create_tx(); auto handle = tx.rw(map); handle->put(k, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.rollback({kv_store.commit_view(), 0}, kv_store.commit_view()); auto handle2 = tx2.rw(map); auto v = handle2->get(k); REQUIRE(!v.has_value()); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Read committed key"); @@ -1726,7 +1728,7 @@ TEST_CASE("Rollback and compact") auto tx2 = kv_store.create_tx(); auto handle = tx.rw(map); handle->put(k, v1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); auto handle2 = tx2.rw(map); @@ -1741,7 +1743,7 @@ TEST_CASE("Rollback and compact") auto tx2 = kv_store.create_tx(); auto handle = tx.rw(map); handle->remove(k); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); auto handle2 = tx2.rw(map); @@ -1756,16 +1758,17 @@ TEST_CASE("Local commit hooks") std::vector local_writes; std::vector global_writes; - auto map_hook = [&](kv::Version v, const Write& w) -> kv::ConsensusHookPtr { + auto map_hook = + [&](ccf::kv::Version v, const Write& w) -> ccf::kv::ConsensusHookPtr { local_writes.push_back(w); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }; - auto global_hook = [&](kv::Version v, const Write& w) { + auto global_hook = [&](ccf::kv::Version v, const Write& w) { global_writes.push_back(w); }; - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); constexpr auto map_name = "public:map"; MapTypes::StringString map(map_name); @@ -1780,7 +1783,7 @@ TEST_CASE("Local commit hooks") handle->put("key1", "value1"); handle->put("key2", "value2"); handle->remove("key2"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(global_writes.size() == 0); REQUIRE(local_writes.size() == 1); @@ -1797,7 +1800,7 @@ TEST_CASE("Local commit hooks") auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->remove("key1"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(global_writes.size() == 0); REQUIRE(local_writes.size() == 1); @@ -1818,7 +1821,7 @@ TEST_CASE("Local commit hooks") auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->put("key2", "value2"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(local_writes.size() == 0); REQUIRE(global_writes.size() == 0); @@ -1833,7 +1836,7 @@ TEST_CASE("Local commit hooks") auto handle = tx.rw(map); handle->remove("key2"); handle->put("key3", "value3"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(global_writes.size() == 0); REQUIRE(local_writes.size() == 1); @@ -1860,20 +1863,20 @@ TEST_CASE("Global commit hooks") struct GlobalHookInput { - kv::Version version; + ccf::kv::Version version; Write writes; }; std::vector global_writes; - auto global_hook = [&](kv::Version v, const Write& w) { + auto global_hook = [&](ccf::kv::Version v, const Write& w) { global_writes.emplace_back(GlobalHookInput({v, w})); }; - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); - using MapT = kv::Map; + using MapT = ccf::kv::Map; MapT map_with_hook("public:map_with_hook"); kv_store.set_global_hook( map_with_hook.get_name(), map_with_hook.wrap_commit_hook(global_hook)); @@ -1892,7 +1895,7 @@ TEST_CASE("Global commit hooks") auto tx1 = kv_store.create_tx(); auto handle_hook = tx1.rw(map_with_hook); handle_hook->put("key1", "value1"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(1); @@ -1912,11 +1915,11 @@ TEST_CASE("Global commit hooks") auto tx3 = kv_store.create_tx(); auto handle_hook = tx1.rw(map_with_hook); handle_hook->put("key1", "value1"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); handle_hook = tx2.rw(map_with_hook); handle_hook->put("key2", "value2"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); const auto compact_version = kv_store.current_version(); @@ -1924,7 +1927,7 @@ TEST_CASE("Global commit hooks") // version of the store auto handle_no_hook = tx3.rw(map_no_hook); handle_no_hook->put("key3", "value3"); - REQUIRE(tx3.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx3.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(compact_version); @@ -1949,19 +1952,19 @@ TEST_CASE("Global commit hooks") auto tx3 = kv_store.create_tx(); auto handle_hook = tx1.rw(map_with_hook); handle_hook->put("key1", "value1"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); // This does not affect map_with_hook but still increments the current // version of the store auto handle_no_hook = tx2.rw(map_no_hook); handle_no_hook->put("key2", "value2"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); const auto compact_version = kv_store.current_version(); handle_hook = tx3.rw(map_with_hook); handle_hook->put("key3", "value3"); - REQUIRE(tx3.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx3.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(compact_version); @@ -1981,14 +1984,14 @@ TEST_CASE("Global commit hooks") auto tx2 = kv_store.create_tx(); auto handle_hook = tx1.rw(map_with_hook); handle_hook->put("key1", "value1"); - REQUIRE(tx1.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx1.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); global_writes.clear(); handle_hook = tx2.rw(map_with_hook); handle_hook->put("key2", "value2"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); kv_store.compact(kv_store.current_version()); @@ -2004,8 +2007,8 @@ TEST_CASE("Global commit hooks") TEST_CASE("Deserialising from other Store") { - auto encryptor = std::make_shared(); - kv::Store store; + auto encryptor = std::make_shared(); + ccf::kv::Store store; store.set_encryptor(encryptor); MapTypes::NumString public_map("public:public"); @@ -2019,18 +2022,18 @@ TEST_CASE("Deserialising from other Store") tx1.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); - kv::Store clone; + ccf::kv::Store clone; clone.set_encryptor(encryptor); - REQUIRE(clone.deserialize(data)->apply() == kv::ApplyResult::PASS); + REQUIRE(clone.deserialize(data)->apply() == ccf::kv::ApplyResult::PASS); } TEST_CASE("Deserialise return status") { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); ccf::Signatures signatures(ccf::Tables::SIGNATURES); @@ -2040,11 +2043,11 @@ TEST_CASE("Deserialise return status") ccf::Nodes nodes(ccf::Tables::NODES); MapTypes::NumNum data("public:data"); - constexpr auto default_curve = crypto::CurveID::SECP384R1; - auto kp = crypto::make_key_pair(default_curve); + constexpr auto default_curve = ccf::crypto::CurveID::SECP384R1; + auto kp = ccf::crypto::make_key_pair(default_curve); - auto history = - std::make_shared(store, kv::test::PrimaryNodeId, *kp); + auto history = std::make_shared( + store, ccf::kv::test::PrimaryNodeId, *kp); store.set_history(history); { @@ -2055,26 +2058,26 @@ TEST_CASE("Deserialise return status") tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); - REQUIRE(store.deserialize(data)->apply() == kv::ApplyResult::PASS); + REQUIRE(store.deserialize(data)->apply() == ccf::kv::ApplyResult::PASS); } { auto tx = store.create_reserved_tx(store.next_txid()); auto sig_handle = tx.rw(signatures); auto tree_handle = tx.rw(serialised_tree); - ccf::PrimarySignature sigv(kv::test::PrimaryNodeId, 2); + ccf::PrimarySignature sigv(ccf::kv::test::PrimaryNodeId, 2); sig_handle->put(sigv); tree_handle->put({}); auto [success_, data_, claims_digest, commit_evidence_digest, hooks] = tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); REQUIRE( - store.deserialize(data)->apply() == kv::ApplyResult::PASS_SIGNATURE); + store.deserialize(data)->apply() == ccf::kv::ApplyResult::PASS_SIGNATURE); } INFO("Signature transactions with additional contents should fail"); @@ -2082,26 +2085,26 @@ TEST_CASE("Deserialise return status") auto tx = store.create_reserved_tx(store.next_txid()); auto sig_handle = tx.rw(signatures); auto data_handle = tx.rw(data); - ccf::PrimarySignature sigv(kv::test::PrimaryNodeId, 2); + ccf::PrimarySignature sigv(ccf::kv::test::PrimaryNodeId, 2); sig_handle->put(sigv); data_handle->put(43, 43); auto [success_, data_, claims_digest, commit_evidence_digest, hooks] = tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); - REQUIRE(store.deserialize(data)->apply() == kv::ApplyResult::FAIL); + REQUIRE(store.deserialize(data)->apply() == ccf::kv::ApplyResult::FAIL); } } TEST_CASE("Map swap between stores") { - auto encryptor = std::make_shared(); - kv::Store s1; + auto encryptor = std::make_shared(); + ccf::kv::Store s1; s1.set_encryptor(encryptor); - kv::Store s2; + ccf::kv::Store s2; s2.set_encryptor(encryptor); MapTypes::NumNum d("data"); @@ -2111,14 +2114,14 @@ TEST_CASE("Map swap between stores") auto tx = s1.create_tx(); auto v = tx.rw(d); v->put(42, 42); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { auto tx = s1.create_tx(); auto v = tx.rw(pd); v->put(14, 14); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } const auto target_version = s1.current_version(); @@ -2127,7 +2130,7 @@ TEST_CASE("Map swap between stores") auto tx = s2.create_tx(); auto v = tx.rw(d); v->put(41, 41); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } s2.swap_private_maps(s1); @@ -2167,13 +2170,13 @@ TEST_CASE("Map swap between stores") TEST_CASE("Private recovery map swap") { - auto encryptor = std::make_shared(); - kv::Store s1; + auto encryptor = std::make_shared(); + ccf::kv::Store s1; s1.set_encryptor(encryptor); MapTypes::NumNum priv1("private"); MapTypes::NumString pub1("public:data"); - kv::Store s2; + ccf::kv::Store s2; s2.set_encryptor(encryptor); MapTypes::NumNum priv2("private"); MapTypes::NumString pub2("public:data"); @@ -2309,8 +2312,8 @@ TEST_CASE("Private recovery map swap") TEST_CASE("Conflict resolution") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -2319,10 +2322,10 @@ TEST_CASE("Conflict resolution") auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->put("foo", "initial"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } - auto try_write = [&](kv::Tx& tx, const std::string& s) { + auto try_write = [&](ccf::kv::Tx& tx, const std::string& s) { auto handle = tx.rw(map); // Introduce read-dependency @@ -2365,14 +2368,14 @@ TEST_CASE("Conflict resolution") // A second transaction is committed, conflicting with the first try_write(tx2, "baz"); const auto res2 = tx2.commit(); - REQUIRE(res2 == kv::CommitResult::SUCCESS); + REQUIRE(res2 == ccf::kv::CommitResult::SUCCESS); confirm_state({"baz"}, {"bar"}); } // Trying to commit first transaction produces a conflict auto res1 = tx1.commit(); - REQUIRE(res1 == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(res1 == ccf::kv::CommitResult::FAIL_CONFLICT); confirm_state({"baz"}, {"bar"}); // A third transaction just wants to read the value @@ -2386,13 +2389,13 @@ TEST_CASE("Conflict resolution") // Expected results are committed res1 = tx1.commit(); - REQUIRE(res1 == kv::CommitResult::SUCCESS); + REQUIRE(res1 == ccf::kv::CommitResult::SUCCESS); confirm_state({"baz", "buzz"}, {"bar"}); // Third transaction completes later, has no conflicts but reports the earlier // version it read auto res3 = tx3.commit(); - REQUIRE(res3 == kv::CommitResult::SUCCESS); + REQUIRE(res3 == ccf::kv::CommitResult::SUCCESS); REQUIRE(tx1.commit_version() > tx2.commit_version()); REQUIRE(tx3.get_txid()->version >= tx2.get_txid()->version); @@ -2419,8 +2422,8 @@ TEST_CASE("Conflict resolution - removals") auto c = Cases(i); INFO("Considering case " << c); - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString map("public:map"); @@ -2428,7 +2431,7 @@ TEST_CASE("Conflict resolution - removals") // Ensure maps already exist, by making prior writes auto tx = kv_store.create_tx(); tx.rw(map)->put("", ""); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } // Simulate parallel execution by interleaving tx steps @@ -2477,12 +2480,12 @@ TEST_CASE("Conflict resolution - removals") { auto handle = tx2.rw(map); handle->put(k, "hello"); - REQUIRE(tx2.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx2.commit() == ccf::kv::CommitResult::SUCCESS); } const auto expected = c == Cases::ReadSameKey ? - kv::CommitResult::FAIL_CONFLICT : - kv::CommitResult::SUCCESS; + ccf::kv::CommitResult::FAIL_CONFLICT : + ccf::kv::CommitResult::SUCCESS; CHECK_EQ(tx1.commit(), expected); @@ -2508,8 +2511,8 @@ TEST_CASE("Conflict resolution - removals") TEST_CASE("Cross-map conflicts") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringString source("public:source"); MapTypes::StringString dest("public:dest"); @@ -2520,7 +2523,7 @@ TEST_CASE("Cross-map conflicts") auto tx = kv_store.create_tx(); auto source_handle = tx.wo(source); source_handle->put("hello", "world"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } { @@ -2541,11 +2544,11 @@ TEST_CASE("Cross-map conflicts") auto interfere_tx = kv_store.create_tx(); auto src_handle = interfere_tx.wo(source); src_handle->put("hello", "alice"); - REQUIRE(interfere_tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(interfere_tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Copying operation should conflict on commit"); - REQUIRE(copy_tx.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(copy_tx.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); } { @@ -2570,11 +2573,11 @@ TEST_CASE("Cross-map conflicts") auto interfere_tx = kv_store.create_tx(); auto src_handle = interfere_tx.wo(source); src_handle->put("hello", "bob"); - REQUIRE(interfere_tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(interfere_tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Moving operation should conflict on commit"); - REQUIRE(move_tx.commit() == kv::CommitResult::FAIL_CONFLICT); + REQUIRE(move_tx.commit() == ccf::kv::CommitResult::FAIL_CONFLICT); } } @@ -2585,8 +2588,8 @@ std::string rand_string(size_t i) TEST_CASE("Mid-tx compaction") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); MapTypes::StringNum map_a("public:A"); MapTypes::StringNum map_b("public:B"); @@ -2610,7 +2613,7 @@ TEST_CASE("Mid-tx compaction") handle_b->put(key_b, new_val); const auto result = tx.commit(); - REQUIRE(result == kv::CommitResult::SUCCESS); + REQUIRE(result == ccf::kv::CommitResult::SUCCESS); }; increment_vals(); @@ -2631,7 +2634,7 @@ TEST_CASE("Mid-tx compaction") REQUIRE(a_opt == b_opt); const auto result = tx.commit(); - REQUIRE(result == kv::CommitResult::SUCCESS); + REQUIRE(result == ccf::kv::CommitResult::SUCCESS); } { @@ -2649,7 +2652,7 @@ TEST_CASE("Mid-tx compaction") REQUIRE(a_opt == b_opt); const auto result = tx.commit(); - REQUIRE(result == kv::CommitResult::SUCCESS); + REQUIRE(result == ccf::kv::CommitResult::SUCCESS); } { @@ -2679,9 +2682,9 @@ TEST_CASE("Mid-tx compaction") REQUIRE(a_opt == b_opt); const auto result = tx.commit(); - REQUIRE(result == kv::CommitResult::SUCCESS); + REQUIRE(result == ccf::kv::CommitResult::SUCCESS); } - catch (const kv::CompactedVersionConflict& e) + catch (const ccf::kv::CompactedVersionConflict& e) { threw = true; } @@ -2694,8 +2697,8 @@ TEST_CASE("Mid-tx compaction") TEST_CASE("Store clear") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); kv_store.initialise_term(42); @@ -2715,7 +2718,7 @@ TEST_CASE("Store clear") handle_a->put("key" + std::to_string(i), 42); handle_b->put("key" + std::to_string(i), 42); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } auto current_version = kv_store.current_version(); @@ -2749,18 +2752,18 @@ TEST_CASE("Store clear") TEST_CASE("Reported TxID after commit") { - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); kv_store.set_consensus(consensus); const auto map_name = "public:map"; MapTypes::StringString map(map_name); auto store_last_seqno = kv_store.current_version(); - kv::Term initial_term = 2; - kv::Term store_commit_term = initial_term; - kv::Term store_read_term = 0; + ccf::kv::Term initial_term = 2; + ccf::kv::Term store_commit_term = initial_term; + ccf::kv::Term store_read_term = 0; INFO("Initialise store"); { @@ -2772,13 +2775,14 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); store_read_term = store_commit_term; } REQUIRE(kv_store.current_version() == store_last_seqno); REQUIRE_EQ( - kv_store.current_txid(), kv::TxID(store_read_term, store_last_seqno)); + kv_store.current_txid(), + ccf::kv::TxID(store_read_term, store_last_seqno)); } INFO("Empty committed tx"); @@ -2790,7 +2794,7 @@ TEST_CASE("Reported TxID after commit") // Tx is not yet committed REQUIRE_THROWS_AS(tx.get_txid(), std::logic_error); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Committed transaction was not assigned a TxID because it was empty REQUIRE_FALSE(tx.get_txid().has_value()); @@ -2807,7 +2811,7 @@ TEST_CASE("Reported TxID after commit") // No need to read a key, acquiring a map handle is sufficient to acquire a // valid TxID - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Reported TxID includes store read term and last seqno auto tx_id = tx.get_txid(); @@ -2827,7 +2831,7 @@ TEST_CASE("Reported TxID after commit") // Rollback at the current TxID, in the next term kv_store.rollback(kv_store.current_txid(), ++store_commit_term); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2845,7 +2849,7 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.ro(map); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2862,7 +2866,7 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.ro(map); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2877,7 +2881,7 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.rw(map); handle->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); store_last_seqno = kv_store.current_version(); store_read_term = store_commit_term; @@ -2893,7 +2897,7 @@ TEST_CASE("Reported TxID after commit") // Read-only tx should report the new term auto tx = kv_store.create_tx(); auto handle = tx.ro(map); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2907,7 +2911,7 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.ro(map); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2926,7 +2930,7 @@ TEST_CASE("Reported TxID after commit") auto tx = kv_store.create_tx(); auto handle = tx.ro(map); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_id = tx.get_txid(); REQUIRE(tx_id.has_value()); @@ -2986,19 +2990,19 @@ std::map kv_map_range(H& h, std::optional from, std::optional to) TEST_CASE("Range") { - using KVMap = kv::untyped::Map; + using KVMap = ccf::kv::untyped::Map; using KeyType = KVMap::K; using ValueType = KVMap::V; using RefMap = std::map; - using Serialiser = kv::serialisers::JsonSerialiser; + using Serialiser = ccf::kv::serialisers::JsonSerialiser; size_t size = 100; size_t entries_space_size = size * 10; const auto map_name = "public:map"; const ValueType empty_value = {}; - kv::Store kv_store; - auto encryptor = std::make_shared(); + ccf::kv::Store kv_store; + auto encryptor = std::make_shared(); kv_store.set_encryptor(encryptor); RefMap ref; @@ -3020,7 +3024,7 @@ TEST_CASE("Range") h->put(serialised_key, empty_value); ref[serialised_key] = empty_value; } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Compare ranges between KV map and reference"); @@ -3076,7 +3080,7 @@ TEST_CASE("Range") h->remove(key_to_remove); REQUIRE(ref.erase(key_to_remove) == 1); } - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Range does not include key"); @@ -3127,9 +3131,9 @@ TEST_CASE("Range") TEST_CASE("Ledger entry chunk request") { - kv::Store store; - auto encryptor = std::make_shared(); - auto consensus = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); + auto consensus = std::make_shared(); store.set_encryptor(encryptor); store.set_consensus(consensus); @@ -3140,18 +3144,18 @@ TEST_CASE("Ledger entry chunk request") ccf::Nodes nodes(ccf::Tables::NODES); MapTypes::NumNum data("public:data"); - constexpr auto default_curve = crypto::CurveID::SECP384R1; - auto kp = crypto::make_key_pair(default_curve); + constexpr auto default_curve = ccf::crypto::CurveID::SECP384R1; + auto kp = ccf::crypto::make_key_pair(default_curve); - auto history = - std::make_shared(store, kv::test::PrimaryNodeId, *kp); + auto history = std::make_shared( + store, ccf::kv::test::PrimaryNodeId, *kp); store.set_history(history); SUBCASE("Chunk at next signature") { // Ledger chunk flag is not set in the store REQUIRE(!store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); INFO("Add a transaction with the chunking flag enabled"); { @@ -3159,16 +3163,16 @@ TEST_CASE("Ledger entry chunk request") auto tx = store.create_tx(); // Request a ledger chunk at the next signature - tx.set_flag(kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); + tx.set_flag(ccf::kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); auto h1 = tx.rw(map); h1->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } // Flag is now set in the store REQUIRE(store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); INFO("Roll back the last transaction"); { @@ -3178,7 +3182,7 @@ TEST_CASE("Ledger entry chunk request") // Ledger chunk flag is still set in the store REQUIRE(store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); // Roll the last transaction back to clear the flag in the store store.rollback( @@ -3187,7 +3191,7 @@ TEST_CASE("Ledger entry chunk request") // Ledger chunk flag is not set in the store anymore REQUIRE(!store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); } INFO("Add another transaction with the chunking flag enabled"); @@ -3196,16 +3200,16 @@ TEST_CASE("Ledger entry chunk request") auto tx = store.create_tx(); // Request a ledger chunk at the next signature again - tx.set_flag(kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); + tx.set_flag(ccf::kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); auto h1 = tx.rw(map); h1->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } // Ledger chunk flag is now set in the store REQUIRE(store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); INFO( "Add a signature transaction which triggers chunk via entry header flag"); @@ -3214,29 +3218,31 @@ TEST_CASE("Ledger entry chunk request") auto tx = store.create_reserved_tx(txid); auto sig_handle = tx.rw(signatures); auto tree_handle = tx.rw(serialised_tree); - ccf::PrimarySignature sigv(kv::test::PrimaryNodeId, txid.version); + ccf::PrimarySignature sigv(ccf::kv::test::PrimaryNodeId, txid.version); sig_handle->put(sigv); tree_handle->put({}); auto [success_, data_, claims_digest, commit_evidence_digest, hooks] = tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); REQUIRE( - store.deserialize(data)->apply() == kv::ApplyResult::PASS_SIGNATURE); + store.deserialize(data)->apply() == + ccf::kv::ApplyResult::PASS_SIGNATURE); // Header flag is set in the last entry const uint8_t* entry_data = data.data(); size_t entry_data_size = data.size(); - auto header = serialized::peek( + auto header = serialized::peek( entry_data, entry_data_size); - REQUIRE((header.flags & kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER) != 0); + REQUIRE( + (header.flags & ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER) != 0); } // Ledger chunk flag is not set in the store anymore REQUIRE(!store.flag_enabled( - kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE)); } SUBCASE("Chunk before this transaction") @@ -3249,11 +3255,11 @@ TEST_CASE("Ledger entry chunk request") auto tx = store.create_tx(); // Request a ledger chunk before tx - tx.set_flag(kv::CommittableTx::Flag::LEDGER_CHUNK_BEFORE_THIS_TX); + tx.set_flag(ccf::kv::CommittableTx::Flag::LEDGER_CHUNK_BEFORE_THIS_TX); auto h1 = tx.rw(map); h1->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Verify that flag is included in serialised entry"); @@ -3264,15 +3270,16 @@ TEST_CASE("Ledger entry chunk request") const uint8_t* entry_data = data.data(); size_t entry_data_size = data.size(); - auto header = serialized::peek( + auto header = serialized::peek( entry_data, entry_data_size); - REQUIRE((header.flags & kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE) != 0); + REQUIRE( + (header.flags & ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_BEFORE) != 0); } } SUBCASE("Chunk when the snapshotter requires one") { - store.set_flag(kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); + store.set_flag(ccf::kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); INFO("Add a signature that triggers a snapshot"); { @@ -3285,36 +3292,38 @@ TEST_CASE("Ledger entry chunk request") // Add the signature auto sig_handle = tx.rw(signatures); auto tree_handle = tx.rw(serialised_tree); - ccf::PrimarySignature sigv(kv::test::PrimaryNodeId, txid.version); + ccf::PrimarySignature sigv(ccf::kv::test::PrimaryNodeId, txid.version); sig_handle->put(sigv); tree_handle->put({}); auto [success_, data_, claims_digest, commit_evidence_digest, hooks] = tx.commit_reserved(); auto& success = success_; auto& data = data_; - REQUIRE(success == kv::CommitResult::SUCCESS); + REQUIRE(success == ccf::kv::CommitResult::SUCCESS); REQUIRE( - store.deserialize(data)->apply() == kv::ApplyResult::PASS_SIGNATURE); + store.deserialize(data)->apply() == + ccf::kv::ApplyResult::PASS_SIGNATURE); // Check that the ledger chunk header flag is set in the last entry const uint8_t* entry_data = data.data(); size_t entry_data_size = data.size(); - auto header = serialized::peek( + auto header = serialized::peek( entry_data, entry_data_size); - REQUIRE((header.flags & kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER) != 0); + REQUIRE( + (header.flags & ccf::kv::EntryFlags::FORCE_LEDGER_CHUNK_AFTER) != 0); } } } int main(int argc, char** argv) { - logger::config::default_init(); - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/kv/test/null_encryptor.h b/src/kv/test/null_encryptor.h index afffa0a31928..c845fa66a3d2 100644 --- a/src/kv/test/null_encryptor.h +++ b/src/kv/test/null_encryptor.h @@ -4,7 +4,7 @@ #include "kv/kv_types.h" -namespace kv +namespace ccf::kv { // NullTxEncryptor does not decrypt or verify integrity class NullTxEncryptor : public AbstractTxEncryptor @@ -37,7 +37,7 @@ namespace kv return true; } - crypto::HashBytes get_commit_nonce( + ccf::crypto::HashBytes get_commit_nonce( const TxID& tx_id, bool historical_hint = false) override { return {}; diff --git a/src/kv/test/stub_consensus.h b/src/kv/test/stub_consensus.h index 413861f8f68f..b6469ca56597 100644 --- a/src/kv/test/stub_consensus.h +++ b/src/kv/test/stub_consensus.h @@ -9,7 +9,7 @@ #include #include -namespace kv::test +namespace ccf::kv::test { static NodeId PrimaryNodeId = std::string("PrimaryNodeId"); static NodeId FirstBackupNodeId = std::string("FirstBackupNodeId"); diff --git a/src/kv/tx.cpp b/src/kv/tx.cpp index 5f628de656cc..8501efa53954 100644 --- a/src/kv/tx.cpp +++ b/src/kv/tx.cpp @@ -9,7 +9,7 @@ #include "kv/tx_pimpl.h" #include "kv/untyped_map.h" -namespace kv +namespace ccf::kv { MapChanges::MapChanges( const std::shared_ptr& m, @@ -75,15 +75,15 @@ namespace kv // NB: The created maps are always untyped. Only the handles over them // are typed - auto new_map = std::make_shared( - pimpl->store, map_name, kv::get_security_domain(map_name)); + auto new_map = std::make_shared( + pimpl->store, map_name, ccf::kv::get_security_domain(map_name)); pimpl->created_maps[map_name] = new_map; abstract_map = new_map; } auto untyped_map = - std::dynamic_pointer_cast(abstract_map); + std::dynamic_pointer_cast(abstract_map); if (untyped_map == nullptr) { throw std::logic_error( diff --git a/src/kv/tx_pimpl.h b/src/kv/tx_pimpl.h index 7944c9fb7315..db9ee90373e9 100644 --- a/src/kv/tx_pimpl.h +++ b/src/kv/tx_pimpl.h @@ -4,7 +4,7 @@ #include "ccf/tx.h" -namespace kv +namespace ccf::kv { struct BaseTx::PrivateImpl { diff --git a/src/kv/untyped_change_set.h b/src/kv/untyped_change_set.h index a20aa7f2b29f..6385ec84b2a3 100644 --- a/src/kv/untyped_change_set.h +++ b/src/kv/untyped_change_set.h @@ -14,7 +14,7 @@ # include "ds/rb_map.h" #endif -namespace kv::untyped +namespace ccf::kv::untyped { using SerialisedEntry = ccf::ByteVector; using SerialisedKeyHasher = std::hash; @@ -23,7 +23,7 @@ namespace kv::untyped using V = SerialisedEntry; using H = SerialisedKeyHasher; - using VersionV = kv::VersionV; + using VersionV = ccf::kv::VersionV; #ifndef KV_STATE_RB using State = champ::Map; @@ -46,19 +46,19 @@ namespace kv::untyped public: const size_t rollback_counter = {}; - const kv::untyped::State state = {}; - const kv::untyped::State committed = {}; + const ccf::kv::untyped::State state = {}; + const ccf::kv::untyped::State committed = {}; const Version start_version = {}; Version read_version = NoVersion; - kv::untyped::Read reads = {}; - kv::untyped::Write writes = {}; + ccf::kv::untyped::Read reads = {}; + ccf::kv::untyped::Write writes = {}; ChangeSet( size_t rollbacks, - kv::untyped::State& current_state, - kv::untyped::State& committed_state, - kv::untyped::Write changed_writes, + ccf::kv::untyped::State& current_state, + ccf::kv::untyped::State& committed_state, + ccf::kv::untyped::Write changed_writes, Version current_version) : rollback_counter(rollbacks), state(current_state), @@ -81,10 +81,11 @@ namespace kv::untyped // obliterates the current state. struct SnapshotChangeSet : public ChangeSet { - const kv::untyped::State state; + const ccf::kv::untyped::State state; const Version version; - SnapshotChangeSet(kv::untyped::State&& snapshot_state, Version version_) : + SnapshotChangeSet( + ccf::kv::untyped::State&& snapshot_state, Version version_) : state(std::move(snapshot_state)), version(version_) {} @@ -101,15 +102,15 @@ namespace kv::untyped namespace map { template <> - inline size_t get_size( - const kv::untyped::VersionV& data) + inline size_t get_size( + const ccf::kv::untyped::VersionV& data) { return sizeof(uint64_t) + sizeof(data.version) + data.value.size(); } template <> - inline size_t serialize( - const kv::untyped::VersionV& t, uint8_t*& data, size_t& size) + inline size_t serialize( + const ccf::kv::untyped::VersionV& t, uint8_t*& data, size_t& size) { uint64_t data_size = sizeof(t.version) + t.value.size(); serialized::write( @@ -131,29 +132,29 @@ namespace map } template <> - inline kv::untyped::VersionV deserialize( + inline ccf::kv::untyped::VersionV deserialize( const uint8_t*& data, size_t& size) { - kv::untyped::VersionV ret; + ccf::kv::untyped::VersionV ret; uint64_t data_size = serialized::read(data, size); - kv::Version version = serialized::read(data, size); + ccf::kv::Version version = serialized::read(data, size); ret.version = version; - data_size -= sizeof(kv::Version); + data_size -= sizeof(ccf::kv::Version); ret.value.append(data, data + data_size); serialized::skip(data, size, data_size); return ret; } template <> - inline size_t get_size( - const kv::untyped::SerialisedEntry& data) + inline size_t get_size( + const ccf::kv::untyped::SerialisedEntry& data) { return sizeof(uint64_t) + data.size(); } template <> - inline size_t serialize( - const kv::untyped::SerialisedEntry& t, uint8_t*& data, size_t& size) + inline size_t serialize( + const ccf::kv::untyped::SerialisedEntry& t, uint8_t*& data, size_t& size) { uint64_t data_size = t.size(); serialized::write( @@ -167,11 +168,11 @@ namespace map } template <> - inline kv::untyped::SerialisedEntry deserialize( - const uint8_t*& data, size_t& size) + inline ccf::kv::untyped::SerialisedEntry deserialize< + ccf::kv::untyped::SerialisedEntry>(const uint8_t*& data, size_t& size) { uint64_t data_size = serialized::read(data, size); - kv::untyped::SerialisedEntry ret; + ccf::kv::untyped::SerialisedEntry ret; ret.append(data, data + data_size); serialized::skip(data, size, data_size); return ret; diff --git a/src/kv/untyped_map.h b/src/kv/untyped_map.h index 587c50b24223..ccfab022f5c7 100644 --- a/src/kv/untyped_map.h +++ b/src/kv/untyped_map.h @@ -16,7 +16,7 @@ #include #include -namespace kv::untyped +namespace ccf::kv::untyped { struct LocalCommit { @@ -33,7 +33,7 @@ namespace kv::untyped LocalCommit* next = nullptr; LocalCommit* prev = nullptr; }; - using LocalCommits = ds::DLList; + using LocalCommits = ::ds::DLList; struct Roll { @@ -68,14 +68,14 @@ namespace kv::untyped class Map : public AbstractMap { public: - using K = kv::untyped::SerialisedEntry; - using V = kv::untyped::SerialisedEntry; - using H = kv::untyped::SerialisedKeyHasher; + using K = ccf::kv::untyped::SerialisedEntry; + using V = ccf::kv::untyped::SerialisedEntry; + using H = ccf::kv::untyped::SerialisedKeyHasher; - using StateSnapshot = kv::untyped::State::Snapshot; + using StateSnapshot = ccf::kv::untyped::State::Snapshot; - using CommitHook = kv::untyped::CommitHook; - using MapHook = kv::untyped::MapHook; + using CommitHook = ccf::kv::untyped::CommitHook; + using MapHook = ccf::kv::untyped::MapHook; private: AbstractStore* store; @@ -293,7 +293,7 @@ namespace kv::untyped private: const std::string name; const SecurityDomain security_domain; - const kv::Version version; + const ccf::kv::Version version; std::unique_ptr map_snapshot; @@ -301,7 +301,7 @@ namespace kv::untyped Snapshot( const std::string& name_, SecurityDomain security_domain_, - kv::Version version_, + ccf::kv::Version version_, std::unique_ptr&& map_snapshot_) : name(name_), security_domain(security_domain_), @@ -327,10 +327,10 @@ namespace kv::untyped }; // Public typedefs for external consumption - using ReadOnlyHandle = kv::untyped::MapHandle; - using WriteOnlyHandle = kv::untyped::MapHandle; - using Handle = kv::untyped::MapHandle; - using Diff = kv::untyped::MapDiff; + using ReadOnlyHandle = ccf::kv::untyped::MapHandle; + using WriteOnlyHandle = ccf::kv::untyped::MapHandle; + using Handle = ccf::kv::untyped::MapHandle; + using Diff = ccf::kv::untyped::MapDiff; Map( AbstractStore* store_, @@ -357,7 +357,7 @@ namespace kv::untyped bool include_reads) override { const auto non_abstract = - dynamic_cast(changes); + dynamic_cast(changes); if (non_abstract == nullptr) { LOG_FAIL_FMT("Unable to serialise map due to type mismatch"); @@ -560,7 +560,7 @@ namespace kv::untyped /** Get store that the map belongs to * - * @return Pointer to `kv::AbstractStore` + * @return Pointer to `ccf::kv::AbstractStore` */ AbstractStore* get_store() override { @@ -802,7 +802,7 @@ namespace kv::untyped { if (current->version <= version) { - kv::untyped::Write writes; + ccf::kv::untyped::Write writes; if (track_deletes_on_missing_keys) { writes = current->writes; diff --git a/src/kv/untyped_map_diff.cpp b/src/kv/untyped_map_diff.cpp index 4361bb8b020c..e544347bce31 100644 --- a/src/kv/untyped_map_diff.cpp +++ b/src/kv/untyped_map_diff.cpp @@ -5,7 +5,7 @@ #include "kv/untyped_change_set.h" -namespace kv::untyped +namespace ccf::kv::untyped { void MapDiff::foreach_(const MapDiff::ElementVisitorWithEarlyOut& f) { @@ -20,7 +20,8 @@ namespace kv::untyped } } - MapDiff::MapDiff(kv::untyped::ChangeSet& cs, const std::string& map_name) : + MapDiff::MapDiff( + ccf::kv::untyped::ChangeSet& cs, const std::string& map_name) : writes(cs.writes), map_name(map_name) {} diff --git a/src/kv/untyped_map_handle.cpp b/src/kv/untyped_map_handle.cpp index edcbb601501c..be4a1fafd384 100644 --- a/src/kv/untyped_map_handle.cpp +++ b/src/kv/untyped_map_handle.cpp @@ -5,7 +5,7 @@ #include "kv/untyped_change_set.h" -namespace kv::untyped +namespace ccf::kv::untyped { const MapHandle::ValueType* MapHandle::read_key(const KeyType& key) { @@ -85,7 +85,7 @@ namespace kv::untyped } MapHandle::MapHandle( - kv::untyped::ChangeSet& cs, const std::string& map_name) : + ccf::kv::untyped::ChangeSet& cs, const std::string& map_name) : tx_changes(cs), map_name(map_name) {} diff --git a/src/kv/version_v.h b/src/kv/version_v.h index 71ed5725def0..576cbe19b808 100644 --- a/src/kv/version_v.h +++ b/src/kv/version_v.h @@ -2,7 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once -namespace kv +namespace ccf::kv { using Version = uint64_t; diff --git a/src/node/acme_challenge_frontend.h b/src/node/acme_challenge_frontend.h index 260c50fb97eb..f82e5792bbfa 100644 --- a/src/node/acme_challenge_frontend.h +++ b/src/node/acme_challenge_frontend.h @@ -14,8 +14,7 @@ namespace ccf class ACMERpcEndpoints : public CommonEndpointRegistry { public: - ACMERpcEndpoints( - NetworkState& network, ccfapp::AbstractNodeContext& context) : + ACMERpcEndpoints(NetworkState& network, ccf::AbstractNodeContext& context) : CommonEndpointRegistry( get_actor_prefix(ActorsType::acme_challenge), context) { @@ -94,8 +93,7 @@ namespace ccf ACMERpcEndpoints endpoints; public: - ACMERpcFrontend( - NetworkState& network, ccfapp::AbstractNodeContext& context) : + ACMERpcFrontend(NetworkState& network, ccf::AbstractNodeContext& context) : RpcFrontend(*network.tables, endpoints, context), endpoints(network, context) {} diff --git a/src/node/acme_client.h b/src/node/acme_client.h index c2b05cfc8988..56564731178e 100644 --- a/src/node/acme_client.h +++ b/src/node/acme_client.h @@ -65,7 +65,7 @@ namespace ACME public: Client( const ClientConfig& config, - std::shared_ptr account_key_pair = nullptr) : + std::shared_ptr account_key_pair = nullptr) : config(config) { set_account_key(account_key_pair); @@ -74,7 +74,8 @@ namespace ACME virtual ~Client() {} void get_certificate( - std::shared_ptr service_key_, bool override_time = false) + std::shared_ptr service_key_, + bool override_time = false) { using namespace std::chrono_literals; using namespace std::chrono; @@ -137,7 +138,7 @@ namespace ACME order.account_url, cit->second.challenge_url, [this, order_url = order.order_url, &challenge = cit->second]( - const http::HeaderMap& headers, const nlohmann::json& j) { + const ccf::http::HeaderMap& headers, const nlohmann::json& j) { threading::ThreadMessaging::instance().add_task_after( schedule_check_challenge(order_url, challenge), std::chrono::milliseconds(0)); @@ -148,11 +149,11 @@ namespace ACME } virtual void set_account_key( - std::shared_ptr new_account_key_pair) + std::shared_ptr new_account_key_pair) { account_key_pair = new_account_key_pair != nullptr ? new_account_key_pair : - crypto::make_key_pair(); + ccf::crypto::make_key_pair(); LOG_DEBUG_FMT( "ACME: new account public key: {}", ccf::ds::to_hex(account_key_pair->public_key_der())); @@ -171,8 +172,8 @@ namespace ACME virtual void on_http_request( const http::URL& url, http::Request&& req, - std::function< - bool(http_status status, http::HeaderMap&&, std::vector&&)> + std::function&&)> callback) = 0; void make_request( @@ -180,8 +181,8 @@ namespace ACME const http::URL& url, const std::vector& body, http_status expected_status, - std::function&)> - ok_callback) + std::function&)> ok_callback) { std::unique_lock guard(req_lock); @@ -192,12 +193,13 @@ namespace ACME "ACME: Requesting https://{}:{}{}", url.host, port, url.path); http::Request r(url.path, method); - r.set_header(http::headers::ACCEPT, "*/*"); + r.set_header(ccf::http::headers::ACCEPT, "*/*"); r.set_header( - http::headers::HOST, fmt::format("{}:{}", url.host, url.port)); + ccf::http::headers::HOST, fmt::format("{}:{}", url.host, url.port)); if (!body.empty()) { - r.set_header(http::headers::CONTENT_TYPE, "application/jose+json"); + r.set_header( + ccf::http::headers::CONTENT_TYPE, "application/jose+json"); r.set_body(&body); } auto req = r.build_request(); @@ -209,7 +211,7 @@ namespace ACME std::move(r), [this, expected_status, ok_callback]( http_status status, - http::HeaderMap&& headers, + ccf::http::HeaderMap&& headers, std::vector&& data) { for (auto& [k, v] : headers) { @@ -259,7 +261,8 @@ namespace ACME const http::URL& url, const std::vector& body, http_status expected_status, - std::function + std::function< + void(const ccf::http::HeaderMap& headers, const nlohmann::json&)> ok_callback) { make_request( @@ -268,7 +271,8 @@ namespace ACME body, expected_status, [ok_callback]( - const http::HeaderMap& headers, const std::vector& data) { + const ccf::http::HeaderMap& headers, + const std::vector& data) { nlohmann::json jr; if (!data.empty()) @@ -293,8 +297,8 @@ namespace ACME void post_as_get( const std::string& account_url, const std::string& resource_url, - std::function&)> - ok_callback) + std::function&)> ok_callback) { if (nonces.empty()) { @@ -316,7 +320,7 @@ namespace ACME void post_as_get_json( const std::string& account_url, const std::string& resource_url, - std::function + std::function ok_callback, bool empty_payload = false) { @@ -342,7 +346,8 @@ namespace ACME json_to_bytes(jws), HTTP_STATUS_OK, [ok_callback]( - const http::HeaderMap& headers, const std::vector& data) { + const ccf::http::HeaderMap& headers, + const std::vector& data) { try { ok_callback(headers, nlohmann::json::parse(data)); @@ -358,8 +363,8 @@ namespace ACME } ClientConfig config; - std::shared_ptr service_key; - std::shared_ptr account_key_pair; + std::shared_ptr service_key; + std::shared_ptr account_key_pair; nlohmann::json directory; nlohmann::json account; @@ -423,11 +428,11 @@ namespace ACME static std::string json_to_b64url( const nlohmann::json& j, bool with_padding = true) { - return crypto::b64url_from_raw(json_to_bytes(j), with_padding); + return ccf::crypto::b64url_from_raw(json_to_bytes(j), with_padding); } static void convert_signature_to_ieee_p1363( - std::vector& sig, const crypto::KeyPair& signer) + std::vector& sig, const ccf::crypto::KeyPair& signer) { // Convert signature from ASN.1 format to IEEE P1363 const unsigned char* pp = sig.data(); @@ -447,7 +452,7 @@ namespace ACME JWS( const nlohmann::json& header_, const nlohmann::json& payload_, - crypto::KeyPair& signer_, + ccf::crypto::KeyPair& signer_, bool empty_payload = false) { LOG_TRACE_FMT("ACME: JWS header: {}", header_.dump()); @@ -457,7 +462,7 @@ namespace ACME set(header_b64, payload_b64, signer_); } - JWS(const nlohmann::json& header_, crypto::KeyPair& signer_) : + JWS(const nlohmann::json& header_, ccf::crypto::KeyPair& signer_) : JWS(header_, nlohmann::json::object_t(), signer_, true) {} @@ -467,12 +472,12 @@ namespace ACME void set( const std::string& header_b64, const std::string& payload_b64, - crypto::KeyPair& signer) + ccf::crypto::KeyPair& signer) { auto msg = header_b64 + "." + payload_b64; auto sig = signer.sign(s2v(msg)); convert_signature_to_ieee_p1363(sig, signer); - auto sig_b64 = crypto::b64url_from_raw(sig); + auto sig_b64 = ccf::crypto::b64url_from_raw(sig); (*this)["protected"] = header_b64; (*this)["payload"] = payload_b64; @@ -507,7 +512,7 @@ namespace ACME }; static std::optional get_header_value( - const http::HeaderMap& headers, const std::string& name) + const ccf::http::HeaderMap& headers, const std::string& name) { for (const auto& [k, v] : headers) { @@ -545,15 +550,15 @@ namespace ACME } static std::pair get_crv_alg( - const std::shared_ptr& key_pair) + const std::shared_ptr& key_pair) { std::string crv, alg; - if (key_pair->get_curve_id() == crypto::CurveID::SECP256R1) + if (key_pair->get_curve_id() == ccf::crypto::CurveID::SECP256R1) { crv = "P-256"; alg = "ES256"; } - else if (key_pair->get_curve_id() == crypto::CurveID::SECP384R1) + else if (key_pair->get_curve_id() == ccf::crypto::CurveID::SECP384R1) { crv = "P-384"; alg = "ES384"; @@ -634,7 +639,7 @@ namespace ACME url, {}, HTTP_STATUS_OK, - [this](const http::HeaderMap&, const nlohmann::json& j) { + [this](const ccf::http::HeaderMap&, const nlohmann::json& j) { directory = j; request_new_account(); }); @@ -648,8 +653,8 @@ namespace ACME url, {}, HTTP_STATUS_NO_CONTENT, - [this, - ok_callback](const http::HeaderMap& headers, const nlohmann::json& j) { + [this, ok_callback]( + const ccf::http::HeaderMap& headers, const nlohmann::json& j) { ok_callback(); return true; }); @@ -675,8 +680,8 @@ namespace ACME JWK jwk( "EC", crv_alg.first, - crypto::b64url_from_raw(key_coords.x, false), - crypto::b64url_from_raw(key_coords.y, false)); + ccf::crypto::b64url_from_raw(key_coords.x, false), + ccf::crypto::b64url_from_raw(key_coords.y, false)); nlohmann::json header = { {"alg", crv_alg.second}, @@ -696,7 +701,7 @@ namespace ACME url, json_to_bytes(jws), HTTP_STATUS_CREATED, - [this](const http::HeaderMap& headers, const nlohmann::json& j) { + [this](const ccf::http::HeaderMap& headers, const nlohmann::json& j) { expect_string(j, "status", "valid"); account = j; auto loc_opt = get_header_value(headers, "location"); @@ -762,7 +767,7 @@ namespace ACME json_to_bytes(jws), HTTP_STATUS_CREATED, [this, account_url]( - const http::HeaderMap& headers, const nlohmann::json& j) { + const ccf::http::HeaderMap& headers, const nlohmann::json& j) { expect(j, "status"); expect(j, "finalize"); @@ -817,7 +822,7 @@ namespace ACME order.account_url, authz_url, [this, order_url = order.order_url, authz_url]( - const http::HeaderMap& headers, const nlohmann::json& j) { + const ccf::http::HeaderMap& headers, const nlohmann::json& j) { LOG_TRACE_FMT("ACME: authorization reply: {}", j.dump()); expect_string(j, "status", "pending"); expect(j, "challenges"); @@ -875,11 +880,11 @@ namespace ACME JWK jwk( "EC", crv_alg.first, - crypto::b64url_from_raw(key_coords.x, false), - crypto::b64url_from_raw(key_coords.y, false)); + ccf::crypto::b64url_from_raw(key_coords.x, false), + ccf::crypto::b64url_from_raw(key_coords.y, false)); - auto thumbprint = crypto::sha256(s2v(nlohmann::json(jwk).dump())); - return crypto::b64url_from_raw(thumbprint, false); + auto thumbprint = ccf::crypto::sha256(s2v(nlohmann::json(jwk).dump())); + return ccf::crypto::b64url_from_raw(thumbprint, false); } void add_challenge( @@ -952,7 +957,8 @@ namespace ACME order->account_url, challenge.authorization_url, [this, order_url, challenge_token = challenge.token]( - const http::HeaderMap& headers, const std::vector& body) { + const ccf::http::HeaderMap& headers, + const std::vector& body) { auto j = nlohmann::json::parse(body); LOG_TRACE_FMT("ACME: authorization status: {}", j.dump()); expect(j, "status"); @@ -1042,7 +1048,8 @@ namespace ACME order->account_url, order->order_url, [this, order_url]( - const http::HeaderMap& headers, const std::vector& body) { + const ccf::http::HeaderMap& headers, + const std::vector& body) { auto j = nlohmann::json::parse(body); LOG_TRACE_FMT("ACME: finalization status: {}", j.dump()); expect(j, "status"); @@ -1108,7 +1115,7 @@ namespace ACME virtual std::vector get_service_csr() { - std::vector alt_names; + std::vector alt_names; alt_names.push_back({config.service_dns_name, false}); for (const auto& an : config.alternative_names) alt_names.push_back({an, false}); @@ -1141,7 +1148,8 @@ namespace ACME auto csr = get_service_csr(); - nlohmann::json payload = {{"csr", crypto::b64url_from_raw(csr, false)}}; + nlohmann::json payload = { + {"csr", ccf::crypto::b64url_from_raw(csr, false)}}; JWS jws(header, payload, *account_key_pair); @@ -1152,7 +1160,7 @@ namespace ACME json_to_bytes(jws), HTTP_STATUS_OK, [this, order_url = order->order_url]( - const http::HeaderMap& headers, const nlohmann::json& j) { + const ccf::http::HeaderMap& headers, const nlohmann::json& j) { LOG_TRACE_FMT("ACME: finalization status: {}", j.dump()); expect(j, "status"); const auto status = j["status"].get(); @@ -1203,7 +1211,8 @@ namespace ACME order->account_url, order->certificate_url, [this, order_url]( - const http::HeaderMap& headers, const std::vector& data) { + const ccf::http::HeaderMap& headers, + const std::vector& data) { std::string c(data.data(), data.data() + data.size()); LOG_TRACE_FMT("ACME: obtained certificate (chain): {}", c); diff --git a/src/node/ccf_acme_client.h b/src/node/ccf_acme_client.h index d0ce9c0bee96..467c4d2923de 100644 --- a/src/node/ccf_acme_client.h +++ b/src/node/ccf_acme_client.h @@ -77,8 +77,8 @@ namespace ccf std::shared_ptr rpc_map, std::shared_ptr rpc_sessions, std::shared_ptr challenge_frontend, - std::shared_ptr store, - std::shared_ptr account_key_pair = nullptr, + std::shared_ptr store, + std::shared_ptr account_key_pair = nullptr, std::shared_ptr challenge_handler_ = nullptr) : ACME::Client(get_client_config(config), account_key_pair), config_name(config_name), @@ -98,14 +98,14 @@ namespace ccf virtual ~ACMEClient() {} virtual void set_account_key( - std::shared_ptr new_account_key_pair) override + std::shared_ptr new_account_key_pair) override { ACME::Client::set_account_key(new_account_key_pair); install_wildcard_response(); } virtual void check_expiry( - std::shared_ptr tables, + std::shared_ptr tables, std::unique_ptr& identity) { auto now = std::chrono::system_clock::now(); @@ -115,7 +115,7 @@ namespace ccf auto cert = certs->get(config_name); if (cert) { - auto v = crypto::make_verifier(*cert); + auto v = ccf::crypto::make_verifier(*cert); double rem_pct = v->remaining_percentage(now); LOG_TRACE_FMT( "ACME: remaining certificate validity for '{}': {}%, {} seconds", @@ -143,7 +143,7 @@ namespace ccf std::shared_ptr rpc_map; std::shared_ptr rpc_sessions; std::shared_ptr challenge_frontend; - std::shared_ptr store; + std::shared_ptr store; std::shared_ptr challenge_handler; void install_wildcard_response() @@ -158,8 +158,8 @@ namespace ccf } virtual void on_http_request( - const http::URL& url, - http::Request&& req, + const ::http::URL& url, + ::http::Request&& req, std::function< bool(http_status status, http::HeaderMap&&, std::vector&&)> callback) override @@ -244,7 +244,7 @@ namespace ccf // will install it later, in the global hook on that table. auto tx = store->create_tx(); auto certs = tx.rw(Tables::ACME_CERTIFICATES); - certs->put(config_name, crypto::Pem(certificate)); + certs->put(config_name, ccf::crypto::Pem(certificate)); tx.commit(); } }; diff --git a/src/node/channels.h b/src/node/channels.h index e570b7481126..909eda4efc91 100644 --- a/src/node/channels.h +++ b/src/node/channels.h @@ -94,7 +94,7 @@ FMT_END_NAMESPACE namespace ccf { using MsgNonce = uint64_t; - using GcmHdr = crypto::FixedSizeGcmHeader; + using GcmHdr = ccf::crypto::FixedSizeGcmHeader; // Receive nonces were previously stored per-thread. For backwards // compatibility (live communication with nodes still using this format), we @@ -184,11 +184,11 @@ namespace ccf ccf::pal::Mutex lock; NodeId self; - const crypto::Pem& service_cert; - crypto::KeyPairPtr node_kp; - const crypto::Pem& node_cert; - crypto::VerifierPtr peer_cv; - crypto::Pem peer_cert; + const ccf::crypto::Pem& service_cert; + ccf::crypto::KeyPairPtr node_kp; + const ccf::crypto::Pem& node_cert; + ccf::crypto::VerifierPtr peer_cv; + ccf::crypto::Pem peer_cert; ringbuffer::WriterPtr to_host; NodeId peer_id; @@ -203,8 +203,8 @@ namespace ccf size_t message_limit; // Used for AES GCM authentication/encryption - std::unique_ptr recv_key = nullptr; - std::unique_ptr send_key = nullptr; + std::unique_ptr recv_key = nullptr; + std::unique_ptr send_key = nullptr; // Incremented for each tagged/encrypted message std::atomic send_nonce{1}; @@ -485,8 +485,8 @@ namespace ccf } // Validate cert and signature in message - crypto::Pem cert; - crypto::VerifierPtr verifier; + ccf::crypto::Pem cert; + ccf::crypto::VerifierPtr verifier; if (!verify_peer_certificate(pc, cert, verifier)) { CHANNEL_RECV_FAIL( @@ -594,8 +594,8 @@ namespace ccf } // Validate cert and signature in message - crypto::Pem cert; - crypto::VerifierPtr verifier; + ccf::crypto::Pem cert; + ccf::crypto::VerifierPtr verifier; if (!verify_peer_certificate(pc, cert, verifier)) { CHANNEL_RECV_FAIL( @@ -712,13 +712,13 @@ namespace ccf bool verify_peer_certificate( std::span pc, - crypto::Pem& cert, - crypto::VerifierPtr& verifier) + ccf::crypto::Pem& cert, + ccf::crypto::VerifierPtr& verifier) { if (!pc.empty()) { - cert = crypto::Pem(pc); - verifier = crypto::make_verifier(cert); + cert = ccf::crypto::Pem(pc); + verifier = ccf::crypto::make_verifier(cert); // 'true' is `ignore_time` => These node-to-node channels do not care // about certificate times, and should still pass even when given @@ -745,7 +745,7 @@ namespace ccf bool verify_peer_signature( std::span msg, std::span sig, - crypto::VerifierPtr verifier) + ccf::crypto::VerifierPtr verifier) { CHANNEL_RECV_TRACE( "Verifying peer signature with peer certificate serial {}", @@ -764,13 +764,13 @@ namespace ccf const std::string label_to = self.value() + peer_id.value(); const std::span label( reinterpret_cast(label_to.c_str()), label_to.size()); - const auto key_bytes = crypto::hkdf( - crypto::MDType::SHA256, + const auto key_bytes = ccf::crypto::hkdf( + ccf::crypto::MDType::SHA256, shared_key_size, kex_ctx.get_shared_secret(), hkdf_salt, label); - send_key = crypto::make_key_aes_gcm(key_bytes); + send_key = ccf::crypto::make_key_aes_gcm(key_bytes); send_nonce = 1; } @@ -781,13 +781,13 @@ namespace ccf const std::span label( reinterpret_cast(label_from.c_str()), label_from.size()); - const auto key_bytes = crypto::hkdf( - crypto::MDType::SHA256, + const auto key_bytes = ccf::crypto::hkdf( + ccf::crypto::MDType::SHA256, shared_key_size, kex_ctx.get_shared_secret(), hkdf_salt, label); - recv_key = crypto::make_key_aes_gcm(key_bytes); + recv_key = ccf::crypto::make_key_aes_gcm(key_bytes); local_recv_nonce = 0; } @@ -832,7 +832,7 @@ namespace ccf peer_cert = {}; peer_cv.reset(); - auto e = crypto::get_entropy(); + auto e = ccf::crypto::get_entropy(); hkdf_salt = e->random(salt_len); // As a future simplification, we would like this to always be true @@ -855,7 +855,7 @@ namespace ccf peer_cert = {}; peer_cv.reset(); - auto e = crypto::get_entropy(); + auto e = ccf::crypto::get_entropy(); hkdf_salt = e->random(salt_len); } @@ -958,9 +958,9 @@ namespace ccf Channel( ringbuffer::AbstractWriterFactory& writer_factory, - const crypto::Pem& service_cert_, - crypto::KeyPairPtr node_kp_, - const crypto::Pem& node_cert_, + const ccf::crypto::Pem& service_cert_, + ccf::crypto::KeyPairPtr node_kp_, + const ccf::crypto::Pem& node_cert_, const NodeId& self_, const NodeId& peer_id_, size_t message_limit_) : @@ -973,7 +973,7 @@ namespace ccf status(fmt::format("Channel to {}", peer_id_), INACTIVE), message_limit(message_limit_) { - auto e = crypto::get_entropy(); + auto e = ccf::crypto::get_entropy(); hkdf_salt = e->random(salt_len); } diff --git a/src/node/encryptor.h b/src/node/encryptor.h index 81ec71f6da63..dfb50c901291 100644 --- a/src/node/encryptor.h +++ b/src/node/encryptor.h @@ -13,9 +13,9 @@ namespace ccf // - 8 LSB are unique sequence number // - 4 MSB (except final bit) are the 4 LSB of term // - Final bit indicates a snapshot - struct TxGcmHeader : public crypto::StandardGcmHeader + struct TxGcmHeader : public ccf::crypto::StandardGcmHeader { - using crypto::StandardGcmHeader::StandardGcmHeader; + using ccf::crypto::StandardGcmHeader::StandardGcmHeader; constexpr static uint8_t IV_DELIMITER = 8; void set_iv_seq(uint64_t seq) @@ -47,5 +47,5 @@ namespace ccf } }; - using NodeEncryptor = kv::TxEncryptor; + using NodeEncryptor = ccf::kv::TxEncryptor; } \ No newline at end of file diff --git a/src/node/endpoint_context_impl.h b/src/node/endpoint_context_impl.h index b8d1393726d4..faae3ec335fc 100644 --- a/src/node/endpoint_context_impl.h +++ b/src/node/endpoint_context_impl.h @@ -11,11 +11,11 @@ namespace ccf // to user apps) struct EndpointContextImpl : public ccf::endpoints::EndpointContext { - std::unique_ptr owned_tx = nullptr; + std::unique_ptr owned_tx = nullptr; EndpointContextImpl( const std::shared_ptr& r, - std::unique_ptr t) : + std::unique_ptr t) : ccf::endpoints::EndpointContext(r, *t), owned_tx(std::move(t)) {} diff --git a/src/node/gov/gov_endpoint_registry.h b/src/node/gov/gov_endpoint_registry.h index 4a67504e6265..e01fef55b722 100644 --- a/src/node/gov/gov_endpoint_registry.h +++ b/src/node/gov/gov_endpoint_registry.h @@ -25,7 +25,7 @@ namespace ccf public: GovEndpointRegistry( - NetworkState& network_, ccfapp::AbstractNodeContext& context_) : + NetworkState& network_, ccf::AbstractNodeContext& context_) : CommonEndpointRegistry(get_actor_prefix(ActorsType::members), context_), network(network_), share_manager(network_.ledger_secrets) diff --git a/src/node/gov/handlers/proposals.h b/src/node/gov/handlers/proposals.h index 3434f2b45bbe..50d8ae689ebc 100644 --- a/src/node/gov/handlers/proposals.h +++ b/src/node/gov/handlers/proposals.h @@ -29,7 +29,7 @@ namespace ccf::gov::endpoints }; ProposalSubmissionResult validate_proposal_submission_time( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::string& created_at, const std::vector& request_digest, const ccf::ProposalId& proposal_id) @@ -59,8 +59,8 @@ namespace ccf::gov::endpoints if (!replay_keys.empty()) { const auto [min_created_at, _] = - nonstd::split_1(replay_keys[replay_keys.size() / 2], ":"); - auto [key_ts, __] = nonstd::split_1(key, ":"); + ccf::nonstd::split_1(replay_keys[replay_keys.size() / 2], ":"); + auto [key_ts, __] = ccf::nonstd::split_1(key, ":"); if (key_ts < min_created_at) { return { @@ -93,7 +93,7 @@ namespace ccf::gov::endpoints } void record_cose_governance_history( - kv::Tx& tx, + ccf::kv::Tx& tx, const MemberId& caller_id, const std::span& cose_sign1) { @@ -104,7 +104,7 @@ namespace ccf::gov::endpoints } void remove_all_other_non_open_proposals( - kv::Tx& tx, const ProposalId& proposal_id) + ccf::kv::Tx& tx, const ProposalId& proposal_id) { auto p = tx.rw(jsgov::Tables::PROPOSALS); auto pi = @@ -129,9 +129,9 @@ namespace ccf::gov::endpoints // Evaluate JS functions on this proposal. Result is presented in modified // proposal_info argument, which is written back to the KV by this function void resolve_proposal( - ccfapp::AbstractNodeContext& context, + ccf::AbstractNodeContext& context, ccf::NetworkState& network, - kv::Tx& tx, + ccf::kv::Tx& tx, const ProposalId& proposal_id, const std::span& proposal_bytes, ccf::jsgov::ProposalInfo& proposal_info, @@ -387,7 +387,7 @@ namespace ccf::gov::endpoints void init_proposals_handlers( ccf::BaseEndpointRegistry& registry, NetworkState& network, - ccfapp::AbstractNodeContext& node_context) + ccf::AbstractNodeContext& node_context) { //// implementation of TSP interface Proposals auto create_proposal = [&](auto& ctx, ApiVersion api_version) { @@ -418,15 +418,15 @@ namespace ccf::gov::endpoints return; } - auto hasher = crypto::make_incremental_sha256(); + auto hasher = ccf::crypto::make_incremental_sha256(); hasher->update_hash(root_at_read.value().h); - request_digest = crypto::sha256( + request_digest = ccf::crypto::sha256( cose_ident.signature.data(), cose_ident.signature.size()); hasher->update_hash(request_digest); - const crypto::Sha256Hash proposal_hash = hasher->finalise(); + const ccf::crypto::Sha256Hash proposal_hash = hasher->finalise(); proposal_id = proposal_hash.hex_str(); } diff --git a/src/node/gov/handlers/recovery.h b/src/node/gov/handlers/recovery.h index 3e88ecf94b12..71bdc3559c66 100644 --- a/src/node/gov/handlers/recovery.h +++ b/src/node/gov/handlers/recovery.h @@ -12,7 +12,7 @@ namespace ccf::gov::endpoints void init_recovery_handlers( ccf::BaseEndpointRegistry& registry, ShareManager& share_manager, - ccfapp::AbstractNodeContext& node_context) + ccf::AbstractNodeContext& node_context) { auto get_encrypted_share_for_member = [&](auto& ctx, ApiVersion api_version) { @@ -44,7 +44,7 @@ namespace ccf::gov::endpoints auto response_body = nlohmann::json::object(); response_body["memberId"] = member_id; response_body["encryptedShare"] = - crypto::b64_from_raw(encrypted_share.value()); + ccf::crypto::b64_from_raw(encrypted_share.value()); ctx.rpc_ctx->set_response_json(response_body, HTTP_STATUS_OK); return; @@ -124,8 +124,8 @@ namespace ccf::gov::endpoints return; } - auto raw_recovery_share = - crypto::raw_from_b64(params["share"].template get()); + auto raw_recovery_share = ccf::crypto::raw_from_b64( + params["share"].template get()); size_t submitted_shares_count = 0; try diff --git a/src/node/gov/handlers/service_state.h b/src/node/gov/handlers/service_state.h index 7778eb03618b..e2c440a1d080 100644 --- a/src/node/gov/handlers/service_state.h +++ b/src/node/gov/handlers/service_state.h @@ -40,7 +40,7 @@ namespace ccf::gov::endpoints nlohmann::json produce_user_description( const ccf::UserId& user_id, - const crypto::Pem& user_cert, + const ccf::crypto::Pem& user_cert, ccf::UserInfo::ReadOnlyHandle* user_info_handle) { auto user = nlohmann::json::object(); @@ -85,9 +85,10 @@ namespace ccf::gov::endpoints case ccf::QuoteFormat::oe_sgx_v1: { quote_info["format"] = "OE_SGX_v1"; - quote_info["quote"] = crypto::b64_from_raw(node_info.quote_info.quote); + quote_info["quote"] = + ccf::crypto::b64_from_raw(node_info.quote_info.quote); quote_info["endorsements"] = - crypto::b64_from_raw(node_info.quote_info.endorsements); + ccf::crypto::b64_from_raw(node_info.quote_info.endorsements); break; } case ccf::QuoteFormat::insecure_virtual: @@ -100,12 +101,12 @@ namespace ccf::gov::endpoints quote_info["format"] = "AMD_SEV_SNP_v1"; if (node_info.quote_info.uvm_endorsements.has_value()) { - quote_info["uvmEndorsements"] = - crypto::b64_from_raw(node_info.quote_info.uvm_endorsements.value()); + quote_info["uvmEndorsements"] = ccf::crypto::b64_from_raw( + node_info.quote_info.uvm_endorsements.value()); } if (node_info.quote_info.endorsed_tcb.has_value()) { - quote_info["endorsedTcb"] = crypto::b64_from_raw( + quote_info["endorsedTcb"] = ccf::crypto::b64_from_raw( ds::from_hex(node_info.quote_info.endorsed_tcb.value())); } break; @@ -161,7 +162,7 @@ namespace ccf::gov::endpoints ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_body(std::move(constitution.value())); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, + ccf::http::headers::CONTENT_TYPE, http::headervalues::contenttype::JAVASCRIPT); return; } @@ -478,7 +479,8 @@ namespace ccf::gov::endpoints auto info = nlohmann::json::object(); // cert is stored as DER - convert to PEM for API - const auto cert_pem = crypto::cert_der_to_pem(metadata.cert); + const auto cert_pem = + ccf::crypto::cert_der_to_pem(metadata.cert); info["certificate"] = cert_pem.str(); info["issuer"] = metadata.issuer; @@ -642,7 +644,7 @@ namespace ccf::gov::endpoints user_certs_handle->foreach([&user_list, user_info_handle]( const ccf::UserId& user_id, - const crypto::Pem& user_cert) { + const ccf::crypto::Pem& user_cert) { user_list.push_back( produce_user_description(user_id, user_cert, user_info_handle)); return true; diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h index 3dcbcb84018f..46599b5123fe 100644 --- a/src/node/historical_queries.h +++ b/src/node/historical_queries.h @@ -64,9 +64,10 @@ FMT_END_NAMESPACE namespace ccf::historical { static constexpr auto slow_fetch_threshold = std::chrono::milliseconds(1000); + static constexpr size_t soft_to_raw_ratio{5}; static std::optional get_signature( - const kv::StorePtr& sig_store) + const ccf::kv::StorePtr& sig_store) { auto tx = sig_store->create_read_only_tx(); auto signatures = tx.ro(ccf::Tables::SIGNATURES); @@ -74,7 +75,7 @@ namespace ccf::historical } static std::optional> get_tree( - const kv::StorePtr& sig_store) + const ccf::kv::StorePtr& sig_store) { auto tx = sig_store->create_read_only_tx(); auto tree = @@ -85,7 +86,7 @@ namespace ccf::historical class StateCacheImpl { protected: - kv::Store& source_store; + ccf::kv::Store& source_store; std::shared_ptr source_ledger_secrets; ringbuffer::WriterPtr to_host; @@ -95,10 +96,9 @@ namespace ccf::historical // whether to keep all the writes so that we can build a diff later bool track_deletes_on_missing_keys_v = false; - enum class RequestStage + enum class StoreStage { Fetching, - Untrusted, Trusted, }; @@ -132,16 +132,16 @@ namespace ccf::historical struct StoreDetails { std::chrono::milliseconds time_until_fetch = {}; - RequestStage current_stage = RequestStage::Fetching; - crypto::Sha256Hash entry_digest = {}; + StoreStage current_stage = StoreStage::Fetching; + ccf::crypto::Sha256Hash entry_digest = {}; ccf::ClaimsDigest claims_digest = {}; - kv::StorePtr store = nullptr; + ccf::kv::StorePtr store = nullptr; bool is_signature = false; TxReceiptImplPtr receipt = nullptr; ccf::TxID transaction_id; bool has_commit_evidence = false; - crypto::HashBytes get_commit_nonce() + ccf::crypto::HashBytes get_commit_nonce() { if (store != nullptr) { @@ -234,11 +234,13 @@ namespace ccf::historical return {}; } - void adjust_ranges( + std::pair, std::vector> adjust_ranges( const SeqNoCollection& new_seqnos, bool should_include_receipts, SeqNo earliest_ledger_secret_seqno) { + std::vector removed{}, added{}; + bool any_diff = false; // If a seqno is earlier than the earliest known ledger secret, we will @@ -266,6 +268,7 @@ namespace ccf::historical { // No longer looking for a seqno which was previously requested. // Remove it from my_stores + removed.push_back(prev_it->first); prev_it = my_stores.erase(prev_it); any_diff |= true; } @@ -279,6 +282,7 @@ namespace ccf::historical { // If this is too early for known secrets, just record that it // was requested but don't add it to all_stores yet + added.push_back(*new_it); prev_it = my_stores.insert_or_assign(prev_it, *new_it, nullptr); any_too_early = true; } @@ -293,6 +297,7 @@ namespace ccf::historical details = std::make_shared(); all_stores.insert_or_assign(all_it, *new_it, details); } + added.push_back(*new_it); prev_it = my_stores.insert_or_assign(prev_it, *new_it, details); } any_diff |= true; @@ -311,7 +316,7 @@ namespace ccf::historical if (!any_diff && (should_include_receipts == include_receipts)) { HISTORICAL_LOG("Identical to previous request"); - return; + return {removed, added}; } include_receipts = should_include_receipts; @@ -331,6 +336,7 @@ namespace ccf::historical populate_receipts(seqno); } } + return {removed, added}; } void populate_receipts(ccf::SeqNo new_seqno) @@ -493,6 +499,136 @@ namespace ccf::historical ExpiryDuration default_expiry_duration = std::chrono::seconds(1800); + // These two combine into an effective O(log(N)) lookup/add/remove by + // handle. + std::list lru_requests; + std::map::iterator> lru_lookup; + + // To maintain the estimated size consumed by all requests. Gets updated + // when ledger entries are fetched, and when requests are dropped. + std::unordered_map> store_to_requests; + std::unordered_map raw_store_sizes{}; + + CacheSize soft_store_cache_limit{1ll * 1024 * 1024 * 512 /*512 MB*/}; + CacheSize soft_store_cache_limit_raw = + soft_store_cache_limit / soft_to_raw_ratio; + CacheSize estimated_store_cache_size{0}; + + void add_request_ref(SeqNo seq, CompoundHandle handle) + { + auto it = store_to_requests.find(seq); + + if (it == store_to_requests.end()) + { + store_to_requests.insert({seq, {handle}}); + auto size = raw_store_sizes.find(seq); + if (size != raw_store_sizes.end()) + { + estimated_store_cache_size += size->second; + } + } + else + { + it->second.insert(handle); + } + } + + void add_request_refs(CompoundHandle handle) + { + for (const auto& [seq, _] : requests.at(handle).my_stores) + { + add_request_ref(seq, handle); + } + } + + void remove_request_ref(SeqNo seq, CompoundHandle handle) + { + auto it = store_to_requests.find(seq); + assert(it != store_to_requests.end()); + + it->second.erase(handle); + if (it->second.empty()) + { + store_to_requests.erase(it); + auto size = raw_store_sizes.find(seq); + if (size != raw_store_sizes.end()) + { + estimated_store_cache_size -= size->second; + raw_store_sizes.erase(size); + } + } + } + + void remove_request_refs(CompoundHandle handle) + { + for (const auto& [seq, _] : requests.at(handle).my_stores) + { + remove_request_ref(seq, handle); + } + } + + void lru_promote(CompoundHandle handle) + { + auto it = lru_lookup.find(handle); + if (it != lru_lookup.end()) + { + lru_requests.erase(it->second); + it->second = lru_requests.insert(lru_requests.begin(), handle); + } + else + { + lru_lookup[handle] = lru_requests.insert(lru_requests.begin(), handle); + add_request_refs(handle); + } + } + + void lru_shrink_to_fit(size_t threshold) + { + while (estimated_store_cache_size > threshold) + { + if (lru_requests.empty()) + { + LOG_FAIL_FMT( + "LRU shrink to {} requested but cache is already empty", threshold); + return; + } + + const auto handle = lru_requests.back(); + LOG_DEBUG_FMT( + "Cache size shrinking (reached {} / {}). Dropping {}", + estimated_store_cache_size, + threshold, + handle); + + remove_request_refs(handle); + lru_lookup.erase(handle); + + requests.erase(handle); + lru_requests.pop_back(); + } + } + + void lru_evict(CompoundHandle handle) + { + auto it = lru_lookup.find(handle); + if (it != lru_lookup.end()) + { + remove_request_refs(handle); + lru_requests.erase(it->second); + lru_lookup.erase(it); + } + } + + void update_store_raw_size(SeqNo seq, size_t new_size) + { + auto& stored_size = raw_store_sizes[seq]; + assert(!stored_size || stored_size == new_size); + + estimated_store_cache_size -= stored_size; + estimated_store_cache_size += new_size; + stored_size = new_size; + } + void fetch_entry_at(ccf::SeqNo seqno) { fetch_entries_range(seqno, seqno); @@ -568,8 +704,8 @@ namespace ccf::historical void process_deserialised_store( const StoreDetailsPtr& details, - const kv::StorePtr& store, - const crypto::Sha256Hash& entry_digest, + const ccf::kv::StorePtr& store, + const ccf::crypto::Sha256Hash& entry_digest, ccf::SeqNo seqno, bool is_signature, ccf::ClaimsDigest&& claims_digest, @@ -577,7 +713,7 @@ namespace ccf::historical { // Deserialisation includes a GCM integrity check, so all entries // have been verified by the time we get here. - details->current_stage = RequestStage::Trusted; + details->current_stage = StoreStage::Trusted; details->has_commit_evidence = has_commit_evidence; details->entry_digest = entry_digest; @@ -667,7 +803,7 @@ namespace ccf::historical } bool handle_encrypted_past_ledger_secret( - const kv::StorePtr& store, LedgerSecretPtr encrypting_secret) + const ccf::kv::StorePtr& store, LedgerSecretPtr encrypting_secret) { // Read encrypted secrets from store auto tx = store->create_read_only_tx(); @@ -760,6 +896,8 @@ namespace ccf::historical HISTORICAL_LOG("First time I've seen handle {}", handle); } + lru_promote(handle); + Request& request = it->second; update_earliest_known_ledger_secret(); @@ -772,9 +910,18 @@ namespace ccf::historical seqnos.size(), *seqnos.begin(), include_receipts); - request.adjust_ranges( + auto [removed, added] = request.adjust_ranges( seqnos, include_receipts, earliest_secret_.valid_from); + for (auto seq : removed) + { + remove_request_ref(seq, handle); + } + for (auto seq : added) + { + add_request_ref(seq, handle); + } + // If the earliest target entry cannot be deserialised with the earliest // known ledger secret, record the target seqno and begin fetching the // previous historical ledger secret. @@ -789,10 +936,9 @@ namespace ccf::historical for (auto seqno : seqnos) { auto target_details = request.get_store_details(seqno); - if ( target_details != nullptr && - target_details->current_stage == RequestStage::Trusted && + target_details->current_stage == StoreStage::Trusted && (!request.include_receipts || target_details->receipt != nullptr)) { // Have this store, associated txid and receipt and trust it - add @@ -823,6 +969,7 @@ namespace ccf::historical { if (request_it->second.get_store_details(seqno) != nullptr) { + lru_evict(request_it->first); request_it = requests.erase(request_it); } else @@ -832,10 +979,10 @@ namespace ccf::historical } } - std::vector states_to_stores( + std::vector states_to_stores( const std::vector& states) { - std::vector stores; + std::vector stores; for (size_t i = 0; i < states.size(); i++) { stores.push_back(states[i]->store); @@ -845,7 +992,7 @@ namespace ccf::historical public: StateCacheImpl( - kv::Store& store, + ccf::kv::Store& store, const std::shared_ptr& secrets, const ringbuffer::WriterPtr& host_writer) : source_store(store), @@ -856,7 +1003,7 @@ namespace ccf::historical std::make_shared(historical_ledger_secrets)) {} - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( const CompoundHandle& handle, ccf::SeqNo seqno, ExpiryDuration seconds_until_expiry) @@ -870,7 +1017,7 @@ namespace ccf::historical return range[0]; } - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( const CompoundHandle& handle, ccf::SeqNo seqno) { return get_store_at(handle, seqno, default_expiry_duration); @@ -895,7 +1042,7 @@ namespace ccf::historical return get_state_at(handle, seqno, default_expiry_duration); } - std::vector get_store_range( + std::vector get_store_range( const CompoundHandle& handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno, @@ -908,7 +1055,7 @@ namespace ccf::historical false)); } - std::vector get_store_range( + std::vector get_store_range( const CompoundHandle& handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno) @@ -939,7 +1086,7 @@ namespace ccf::historical handle, start_seqno, end_seqno, default_expiry_duration); } - std::vector get_stores_for( + std::vector get_stores_for( const CompoundHandle& handle, const SeqNoCollection& seqnos, ExpiryDuration seconds_until_expiry) @@ -948,7 +1095,7 @@ namespace ccf::historical get_states_internal(handle, seqnos, seconds_until_expiry, false)); } - std::vector get_stores_for( + std::vector get_stores_for( const CompoundHandle& handle, const SeqNoCollection& seqnos) { return get_stores_for(handle, seqnos, default_expiry_duration); @@ -977,6 +1124,12 @@ namespace ccf::historical default_expiry_duration = duration; } + void set_soft_cache_limit(CacheSize cache_limit) + { + soft_store_cache_limit = cache_limit; + soft_store_cache_limit_raw = soft_store_cache_limit / soft_to_raw_ratio; + } + void track_deletes_on_missing_keys(bool track) { track_deletes_on_missing_keys_v = track; @@ -985,8 +1138,8 @@ namespace ccf::historical bool drop_cached_states(const CompoundHandle& handle) { std::lock_guard guard(requests_lock); + lru_evict(handle); const auto erased_count = requests.erase(handle); - HISTORICAL_LOG("Dropping historical request {}", handle); return erased_count > 0; } @@ -1000,15 +1153,14 @@ namespace ccf::historical std::lock_guard guard(requests_lock); const auto it = all_stores.find(seqno); auto details = it == all_stores.end() ? nullptr : it->second.lock(); - if ( - details == nullptr || details->current_stage != RequestStage::Fetching) + if (details == nullptr || details->current_stage != StoreStage::Fetching) { // Unexpected entry, we already have it or weren't asking for it - // ignore this resubmission return false; } - kv::ApplyResult deserialise_result; + ccf::kv::ApplyResult deserialise_result; ccf::ClaimsDigest claims_digest; bool has_commit_evidence; auto store = deserialise_ledger_entry( @@ -1019,7 +1171,7 @@ namespace ccf::historical claims_digest, has_commit_evidence); - if (deserialise_result == kv::ApplyResult::FAIL) + if (deserialise_result == ccf::kv::ApplyResult::FAIL) { return false; } @@ -1061,7 +1213,7 @@ namespace ccf::historical } const auto is_signature = - deserialise_result == kv::ApplyResult::PASS_SIGNATURE; + deserialise_result == ccf::kv::ApplyResult::PASS_SIGNATURE; update_earliest_known_ledger_secret(); @@ -1084,7 +1236,7 @@ namespace ccf::historical "Processing historical store at {} ({})", seqno, (size_t)deserialise_result); - const auto entry_digest = crypto::Sha256Hash({data, size}); + const auto entry_digest = ccf::crypto::Sha256Hash({data, size}); process_deserialised_store( details, store, @@ -1094,6 +1246,7 @@ namespace ccf::historical std::move(claims_digest), has_commit_evidence); + update_store_raw_size(seqno, size); return true; } @@ -1117,8 +1270,9 @@ namespace ccf::historical while (size > 0) { const auto header = - serialized::peek(data, size); - const auto whole_size = header.size + kv::serialised_entry_header_size; + serialized::peek(data, size); + const auto whole_size = + header.size + ccf::kv::serialised_entry_header_size; all_accepted &= handle_ledger_entry(seqno, data, whole_size); data += whole_size; size -= whole_size; @@ -1165,16 +1319,16 @@ namespace ccf::historical } } - kv::StorePtr deserialise_ledger_entry( + ccf::kv::StorePtr deserialise_ledger_entry( ccf::SeqNo seqno, const uint8_t* data, size_t size, - kv::ApplyResult& result, + ccf::kv::ApplyResult& result, ccf::ClaimsDigest& claims_digest, bool& has_commit_evidence) { // Create a new store and try to deserialise this entry into it - kv::StorePtr store = std::make_shared( + ccf::kv::StorePtr store = std::make_shared( false /* Do not start from very first seqno */, true /* Make use of historical secrets */); @@ -1209,7 +1363,7 @@ namespace ccf::historical auto exec = store->deserialize({data, data + size}, public_only); if (exec == nullptr) { - result = kv::ApplyResult::FAIL; + result = ccf::kv::ApplyResult::FAIL; return nullptr; } @@ -1226,7 +1380,7 @@ namespace ccf::historical "Exception while attempting to deserialise entry {}: {}", seqno, e.what()); - result = kv::ApplyResult::FAIL; + result = ccf::kv::ApplyResult::FAIL; } return store; @@ -1245,6 +1399,7 @@ namespace ccf::historical { LOG_DEBUG_FMT( "Dropping expired historical query with handle {}", it->first); + lru_evict(it->first); it = requests.erase(it); } else @@ -1255,6 +1410,8 @@ namespace ccf::historical } } + lru_shrink_to_fit(soft_store_cache_limit_raw); + { auto it = all_stores.begin(); std::optional> range_to_request = @@ -1268,7 +1425,7 @@ namespace ccf::historical } else { - if (details->current_stage == RequestStage::Fetching) + if (details->current_stage == StoreStage::Fetching) { details->time_until_fetch -= elapsed_ms; if (details->time_until_fetch.count() <= 0) @@ -1324,7 +1481,7 @@ namespace ccf::historical StateCache(Ts&&... ts) : StateCacheImpl(std::forward(ts)...) {} - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( RequestHandle handle, ccf::SeqNo seqno, ExpiryDuration seconds_until_expiry) override @@ -1333,7 +1490,7 @@ namespace ccf::historical make_compound_handle(handle), seqno, seconds_until_expiry); } - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( RequestHandle handle, ccf::SeqNo seqno) override { return StateCacheImpl::get_store_at(make_compound_handle(handle), seqno); @@ -1353,7 +1510,7 @@ namespace ccf::historical return StateCacheImpl::get_state_at(make_compound_handle(handle), seqno); } - std::vector get_store_range( + std::vector get_store_range( RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno, @@ -1366,7 +1523,7 @@ namespace ccf::historical seconds_until_expiry); } - std::vector get_store_range( + std::vector get_store_range( RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno) override @@ -1397,7 +1554,7 @@ namespace ccf::historical make_compound_handle(handle), start_seqno, end_seqno); } - std::vector get_stores_for( + std::vector get_stores_for( RequestHandle handle, const SeqNoCollection& seqnos, ExpiryDuration seconds_until_expiry) override @@ -1406,7 +1563,7 @@ namespace ccf::historical make_compound_handle(handle), seqnos, seconds_until_expiry); } - std::vector get_stores_for( + std::vector get_stores_for( RequestHandle handle, const SeqNoCollection& seqnos) override { return StateCacheImpl::get_stores_for( @@ -1434,6 +1591,11 @@ namespace ccf::historical StateCacheImpl::set_default_expiry_duration(duration); } + void set_soft_cache_limit(CacheSize cache_limit) override + { + StateCacheImpl::set_soft_cache_limit(cache_limit); + } + void track_deletes_on_missing_keys(bool track) override { StateCacheImpl::track_deletes_on_missing_keys(track); diff --git a/src/node/historical_queries_adapter.cpp b/src/node/historical_queries_adapter.cpp index 1d95056c6f1a..3ca36ca16376 100644 --- a/src/node/historical_queries_adapter.cpp +++ b/src/node/historical_queries_adapter.cpp @@ -17,7 +17,7 @@ namespace ccf // Legacy JSON format, retained for compatibility nlohmann::json out = nlohmann::json::object(); - out["signature"] = crypto::b64_from_raw(receipt.signature); + out["signature"] = ccf::crypto::b64_from_raw(receipt.signature); auto proof = nlohmann::json::array(); if (receipt.path != nullptr) @@ -99,7 +99,7 @@ namespace ccf node.direction == ccf::HistoryTree::Path::Direction::PATH_LEFT ? ccf::ProofReceipt::ProofStep::Left : ccf::ProofReceipt::ProofStep::Right; - const auto hash = crypto::Sha256Hash::from_span( + const auto hash = ccf::crypto::Sha256Hash::from_span( std::span( node.hash.bytes, sizeof(node.hash.bytes))); proof_receipt->proof.push_back({direction, hash}); @@ -128,7 +128,7 @@ namespace ccf { // Signature transaction auto sig_receipt = std::make_shared(); - sig_receipt->signed_root = crypto::Sha256Hash::from_span( + sig_receipt->signed_root = ccf::crypto::Sha256Hash::from_span( std::span( in.root.bytes, sizeof(in.root.bytes))); @@ -191,7 +191,7 @@ namespace ccf::historical } HistoricalTxStatus is_tx_committed_v2( - kv::Consensus* consensus, + ccf::kv::Consensus* consensus, ccf::View view, ccf::SeqNo seqno, std::string& error_reason) @@ -242,7 +242,7 @@ namespace ccf::historical class TTxIDExtractor> TEndpointFunction _adapter_v3( const TQueryHandler& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const TTxIDExtractor& extractor) { @@ -340,7 +340,7 @@ namespace ccf::historical ccf::endpoints::EndpointFunction adapter_v3( const HandleHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const TxIDExtractor& extractor) { @@ -352,7 +352,7 @@ namespace ccf::historical ccf::endpoints::ReadOnlyEndpointFunction read_only_adapter_v3( const HandleReadOnlyHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const ReadOnlyTxIDExtractor& extractor) { @@ -365,7 +365,7 @@ namespace ccf::historical ccf::endpoints::EndpointFunction read_write_adapter_v3( const HandleReadWriteHistoricalQuery& f, - ccfapp::AbstractNodeContext& node_context, + ccf::AbstractNodeContext& node_context, const CheckHistoricalTxStatus& available, const TxIDExtractor& extractor) { diff --git a/src/node/historical_queries_utils.cpp b/src/node/historical_queries_utils.cpp index 6daf3801837d..eaa0281e1537 100644 --- a/src/node/historical_queries_utils.cpp +++ b/src/node/historical_queries_utils.cpp @@ -11,13 +11,13 @@ namespace ccf { - static std::map> + static std::map> service_endorsement_cache; namespace historical { std::optional find_previous_service_identity( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, ccf::historical::StatePtr& state, AbstractStateCache& state_cache) { @@ -61,7 +61,7 @@ namespace ccf } bool populate_service_endorsements( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, ccf::historical::StatePtr& state, AbstractStateCache& state_cache, std::shared_ptr @@ -91,7 +91,7 @@ namespace ccf "transaction is in a pre-2.0 part of the ledger."); } - auto v = crypto::make_unique_verifier(*receipt.node_cert); + auto v = ccf::crypto::make_unique_verifier(*receipt.node_cert); if (!v->verify_certificate( {&network_identity->cert}, {}, /* ignore_time */ true)) { @@ -107,8 +107,8 @@ namespace ccf return false; } - auto hpubkey = crypto::public_key_pem_from_cert( - crypto::cert_pem_to_der(opt_psi->cert)); + auto hpubkey = ccf::crypto::public_key_pem_from_cert( + ccf::crypto::cert_pem_to_der(opt_psi->cert)); auto eit = service_endorsement_cache.find(hpubkey); if (eit != service_endorsement_cache.end()) @@ -119,10 +119,11 @@ namespace ccf } else { - auto ncv = crypto::make_unique_verifier(network_identity->cert); + auto ncv = + ccf::crypto::make_unique_verifier(network_identity->cert); auto endorsement = create_endorsed_cert( hpubkey, - crypto::get_subject_name(opt_psi->cert), + ccf::crypto::get_subject_name(opt_psi->cert), {}, ncv->validity_period(), network_identity->priv_key, diff --git a/src/node/history.h b/src/node/history.h index 1201aec46be2..41e2f62c282a 100644 --- a/src/node/history.h +++ b/src/node/history.h @@ -24,7 +24,7 @@ FMT_BEGIN_NAMESPACE template <> -struct formatter +struct formatter { template constexpr auto parse(ParseContext& ctx) @@ -33,7 +33,7 @@ struct formatter } template - auto format(const kv::TxHistory::RequestID& p, FormatContext& ctx) const + auto format(const ccf::kv::TxHistory::RequestID& p, FormatContext& ctx) const { return format_to( ctx.out(), "", std::get<0>(p), std::get<1>(p)); @@ -81,26 +81,26 @@ namespace ccf return os; } - static inline void log_hash(const crypto::Sha256Hash& h, HashOp flag) + static inline void log_hash(const ccf::crypto::Sha256Hash& h, HashOp flag) { LOG_TRACE_FMT("History [{}] {}", flag, h); } - class NullTxHistoryPendingTx : public kv::PendingTx + class NullTxHistoryPendingTx : public ccf::kv::PendingTx { - kv::TxID txid; - kv::Store& store; + ccf::kv::TxID txid; + ccf::kv::Store& store; NodeId id; public: NullTxHistoryPendingTx( - kv::TxID txid_, kv::Store& store_, const NodeId& id_) : + ccf::kv::TxID txid_, ccf::kv::Store& store_, const NodeId& id_) : txid(txid_), store(store_), id(id_) {} - kv::PendingTxInfo call() override + ccf::kv::PendingTxInfo call() override { auto sig = store.create_reserved_tx(txid); auto signatures = @@ -114,18 +114,19 @@ namespace ccf } }; - class NullTxHistory : public kv::TxHistory + class NullTxHistory : public ccf::kv::TxHistory { - kv::Store& store; + ccf::kv::Store& store; NodeId id; protected: - kv::Version version = 0; - kv::Term term_of_last_version = 0; - kv::Term term_of_next_version = 0; + ccf::kv::Version version = 0; + ccf::kv::Term term_of_last_version = 0; + ccf::kv::Term term_of_next_version = 0; public: - NullTxHistory(kv::Store& store_, const NodeId& id_, crypto::KeyPair&) : + NullTxHistory( + ccf::kv::Store& store_, const NodeId& id_, ccf::crypto::KeyPair&) : store(store_), id(id_) {} @@ -136,37 +137,41 @@ namespace ccf } void append_entry( - const crypto::Sha256Hash& digest, - std::optional term_of_next_version_ = std::nullopt) override + const ccf::crypto::Sha256Hash& digest, + std::optional term_of_next_version_ = + std::nullopt) override { version++; } - kv::TxHistory::Result verify_and_sign( - PrimarySignature&, kv::Term*, kv::Configuration::Nodes&) override + ccf::kv::TxHistory::Result verify_and_sign( + PrimarySignature&, + ccf::kv::Term*, + ccf::kv::Configuration::Nodes&) override { - return kv::TxHistory::Result::OK; + return ccf::kv::TxHistory::Result::OK; } - bool verify(kv::Term*, ccf::PrimarySignature*) override + bool verify(ccf::kv::Term*, ccf::PrimarySignature*) override { return true; } - void set_term(kv::Term t) override + void set_term(ccf::kv::Term t) override { term_of_last_version = t; term_of_next_version = t; } - void rollback(const kv::TxID& tx_id, kv::Term commit_term_) override + void rollback( + const ccf::kv::TxID& tx_id, ccf::kv::Term commit_term_) override { version = tx_id.version; term_of_last_version = tx_id.term; term_of_next_version = commit_term_; } - void compact(kv::Version) override {} + void compact(ccf::kv::Version) override {} bool init_from_snapshot(const std::vector&) override { @@ -190,21 +195,21 @@ namespace ccf void start_signature_emit_timer() override {} - crypto::Sha256Hash get_replicated_state_root() override + ccf::crypto::Sha256Hash get_replicated_state_root() override { - return crypto::Sha256Hash(std::to_string(version)); + return ccf::crypto::Sha256Hash(std::to_string(version)); } - std::tuple + std::tuple get_replicated_state_txid_and_root() override { return { {term_of_last_version, version}, - crypto::Sha256Hash(std::to_string(version)), + ccf::crypto::Sha256Hash(std::to_string(version)), term_of_next_version}; } - std::vector get_proof(kv::Version) override + std::vector get_proof(ccf::kv::Version) override { return {}; } @@ -219,7 +224,7 @@ namespace ccf return {}; } - void set_endorsed_certificate(const crypto::Pem& cert) override {} + void set_endorsed_certificate(const ccf::crypto::Pem& cert) override {} }; // Use optimised CCF openssl_sha256 function to avoid performance regression @@ -235,7 +240,7 @@ namespace ccf memcpy(&block[0], l.bytes, sha256_byte_size); memcpy(&block[sha256_byte_size], r.bytes, sha256_byte_size); - crypto::openssl_sha256(block, out.bytes); + ccf::crypto::openssl_sha256(block, out.bytes); } using HistoryTree = merkle::TreeT; @@ -301,23 +306,23 @@ namespace ccf }; template - class MerkleTreeHistoryPendingTx : public kv::PendingTx + class MerkleTreeHistoryPendingTx : public ccf::kv::PendingTx { - kv::TxID txid; - kv::Store& store; - kv::TxHistory& history; + ccf::kv::TxID txid; + ccf::kv::Store& store; + ccf::kv::TxHistory& history; NodeId id; - crypto::KeyPair& kp; - crypto::Pem& endorsed_cert; + ccf::crypto::KeyPair& kp; + ccf::crypto::Pem& endorsed_cert; public: MerkleTreeHistoryPendingTx( - kv::TxID txid_, - kv::Store& store_, - kv::TxHistory& history_, + ccf::kv::TxID txid_, + ccf::kv::Store& store_, + ccf::kv::TxHistory& history_, const NodeId& id_, - crypto::KeyPair& kp_, - crypto::Pem& endorsed_cert_) : + ccf::crypto::KeyPair& kp_, + ccf::crypto::Pem& endorsed_cert_) : txid(txid_), store(store_), history(history_), @@ -326,14 +331,14 @@ namespace ccf endorsed_cert(endorsed_cert_) {} - kv::PendingTxInfo call() override + ccf::kv::PendingTxInfo call() override { auto sig = store.create_reserved_tx(txid); auto signatures = sig.template wo(ccf::Tables::SIGNATURES); auto serialised_tree = sig.template wo( ccf::Tables::SERIALISED_MERKLE_TREE); - crypto::Sha256Hash root = history.get_replicated_state_root(); + ccf::crypto::Sha256Hash root = history.get_replicated_state_root(); std::vector primary_sig; @@ -366,7 +371,7 @@ namespace ccf tree = new HistoryTree(serialised); } - MerkleTreeHistory(crypto::Sha256Hash first_hash = {}) + MerkleTreeHistory(ccf::crypto::Sha256Hash first_hash = {}) { tree = new HistoryTree(merkle::Hash(first_hash.h)); } @@ -383,15 +388,15 @@ namespace ccf tree = new HistoryTree(serialised); } - void append(const crypto::Sha256Hash& hash) + void append(const ccf::crypto::Sha256Hash& hash) { tree->insert(merkle::Hash(hash.h)); } - crypto::Sha256Hash get_root() const + ccf::crypto::Sha256Hash get_root() const { const merkle::Hash& root = tree->root(); - crypto::Sha256Hash result; + ccf::crypto::Sha256Hash result; std::copy(root.bytes, root.bytes + root.size(), result.h.begin()); return result; } @@ -399,7 +404,7 @@ namespace ccf void operator=(const MerkleTreeHistory& rhs) { delete (tree); - crypto::Sha256Hash root(rhs.get_root()); + ccf::crypto::Sha256Hash root(rhs.get_root()); tree = new HistoryTree(merkle::Hash(root.h)); } @@ -475,23 +480,23 @@ namespace ccf return index >= begin_index() && index <= end_index(); } - crypto::Sha256Hash get_leaf(uint64_t index) + ccf::crypto::Sha256Hash get_leaf(uint64_t index) { const merkle::Hash& leaf = tree->leaf(index); - crypto::Sha256Hash result; + ccf::crypto::Sha256Hash result; std::copy(leaf.bytes, leaf.bytes + leaf.size(), result.h.begin()); return result; } }; template - class HashedTxHistory : public kv::TxHistory + class HashedTxHistory : public ccf::kv::TxHistory { - kv::Store& store; + ccf::kv::Store& store; NodeId id; T replicated_state_tree; - crypto::KeyPair& kp; + ccf::crypto::KeyPair& kp; std::optional<::threading::TaskQueue::TimerEntry> emit_signature_timer_entry = std::nullopt; @@ -499,16 +504,16 @@ namespace ccf size_t sig_ms_interval; ccf::pal::Mutex state_lock; - kv::Term term_of_last_version = 0; - kv::Term term_of_next_version; + ccf::kv::Term term_of_last_version = 0; + ccf::kv::Term term_of_next_version; - std::optional endorsed_cert = std::nullopt; + std::optional endorsed_cert = std::nullopt; public: HashedTxHistory( - kv::Store& store_, + ccf::kv::Store& store_, const NodeId& id_, - crypto::KeyPair& kp_, + ccf::crypto::KeyPair& kp_, size_t sig_tx_interval_ = 0, size_t sig_ms_interval_ = 0, bool signature_timer = false) : @@ -549,11 +554,11 @@ namespace ccf auto sig_disp = consensus->get_signature_disposition(); switch (sig_disp) { - case kv::Consensus::SignatureDisposition::CANT_REPLICATE: + case ccf::kv::Consensus::SignatureDisposition::CANT_REPLICATE: { break; } - case kv::Consensus::SignatureDisposition::CAN_SIGN: + case ccf::kv::Consensus::SignatureDisposition::CAN_SIGN: { if (self->store.committable_gap() > 0) { @@ -561,7 +566,7 @@ namespace ccf } break; } - case kv::Consensus::SignatureDisposition::SHOULD_SIGN: + case ccf::kv::Consensus::SignatureDisposition::SHOULD_SIGN: { should_emit_signature = true; break; @@ -623,41 +628,43 @@ namespace ccf replicated_state_tree.deserialise(tree.value()); - crypto::Sha256Hash hash; + ccf::crypto::Sha256Hash hash; std::copy_n( - hash_at_snapshot.begin(), crypto::Sha256Hash::SIZE, hash.h.begin()); + hash_at_snapshot.begin(), + ccf::crypto::Sha256Hash::SIZE, + hash.h.begin()); replicated_state_tree.append(hash); return true; } - crypto::Sha256Hash get_replicated_state_root() override + ccf::crypto::Sha256Hash get_replicated_state_root() override { std::lock_guard guard(state_lock); return replicated_state_tree.get_root(); } - std::tuple + std::tuple get_replicated_state_txid_and_root() override { std::lock_guard guard(state_lock); return { {term_of_last_version, - static_cast(replicated_state_tree.end_index())}, + static_cast(replicated_state_tree.end_index())}, replicated_state_tree.get_root(), term_of_next_version}; } - kv::TxHistory::Result verify_and_sign( + ccf::kv::TxHistory::Result verify_and_sign( PrimarySignature& sig, - kv::Term* term, - kv::Configuration::Nodes& config) override + ccf::kv::Term* term, + ccf::kv::Configuration::Nodes& config) override { if (!verify(term, &sig)) { - return kv::TxHistory::Result::FAIL; + return ccf::kv::TxHistory::Result::FAIL; } - kv::TxHistory::Result result = kv::TxHistory::Result::OK; + ccf::kv::TxHistory::Result result = ccf::kv::TxHistory::Result::OK; sig.node = id; sig.sig = kp.sign_hash(sig.root.h.data(), sig.root.h.size()); @@ -666,7 +673,8 @@ namespace ccf } bool verify( - kv::Term* term = nullptr, PrimarySignature* signature = nullptr) override + ccf::kv::Term* term = nullptr, + PrimarySignature* signature = nullptr) override { auto tx = store.create_read_only_tx(); auto signatures = @@ -707,7 +715,7 @@ namespace ccf } } - void set_term(kv::Term t) override + void set_term(ccf::kv::Term t) override { // This should only be called once, when the store first knows about its // term @@ -717,7 +725,7 @@ namespace ccf } void rollback( - const kv::TxID& tx_id, kv::Term term_of_next_version_) override + const ccf::kv::TxID& tx_id, ccf::kv::Term term_of_next_version_) override { std::lock_guard guard(state_lock); LOG_TRACE_FMT("Rollback to {}.{}", tx_id.term, tx_id.version); @@ -727,7 +735,7 @@ namespace ccf log_hash(replicated_state_tree.get_root(), ROLLBACK); } - void compact(kv::Version v) override + void compact(ccf::kv::Version v) override { std::lock_guard guard(state_lock); // Receipts can only be retrieved to the flushed index. Keep a range of @@ -782,7 +790,7 @@ namespace ccf true); } - std::vector get_proof(kv::Version index) override + std::vector get_proof(ccf::kv::Version index) override { std::lock_guard guard(state_lock); return replicated_state_tree.get_proof(index).to_v(); @@ -804,15 +812,15 @@ namespace ccf void append(const std::vector& data) override { - crypto::Sha256Hash rh(data); + ccf::crypto::Sha256Hash rh(data); log_hash(rh, APPEND); std::lock_guard guard(state_lock); replicated_state_tree.append(rh); } void append_entry( - const crypto::Sha256Hash& digest, - std::optional expected_term_of_next_version = + const ccf::crypto::Sha256Hash& digest, + std::optional expected_term_of_next_version = std::nullopt) override { log_hash(digest, APPEND); @@ -827,7 +835,7 @@ namespace ccf replicated_state_tree.append(digest); } - void set_endorsed_certificate(const crypto::Pem& cert) override + void set_endorsed_certificate(const ccf::crypto::Pem& cert) override { if (cert.empty()) { diff --git a/src/node/hooks.h b/src/node/hooks.h index 8517493957d9..2ca2536a0e3b 100644 --- a/src/node/hooks.h +++ b/src/node/hooks.h @@ -16,15 +16,15 @@ namespace ccf std::string port; }; - class ConfigurationChangeHook : public kv::ConsensusHook + class ConfigurationChangeHook : public ccf::kv::ConsensusHook { - kv::Version version; + ccf::kv::Version version; std::map> cfg_delta; std::unordered_set learners; std::unordered_set retired_nodes; public: - ConfigurationChangeHook(kv::Version version_, const Nodes::Write& w) : + ConfigurationChangeHook(ccf::kv::Version version_, const Nodes::Write& w) : version(version_) { for (const auto& [node_id, opt_ni] : w) @@ -64,7 +64,7 @@ namespace ccf } } - void call(kv::ConfigurableConsensus* consensus) override + void call(ccf::kv::ConfigurableConsensus* consensus) override { auto configuration = consensus->get_latest_configuration_unsafe(); for (const auto& [node_id, opt_ni] : cfg_delta) diff --git a/src/node/http_node_client.h b/src/node/http_node_client.h index 285b10e4284b..42578f36f17f 100644 --- a/src/node/http_node_client.h +++ b/src/node/http_node_client.h @@ -13,16 +13,16 @@ namespace ccf public: HTTPNodeClient( std::shared_ptr rpc_map, - crypto::KeyPairPtr node_sign_kp, - const crypto::Pem& self_signed_node_cert_, - const std::optional& endorsed_node_cert_) : + ccf::crypto::KeyPairPtr node_sign_kp, + const ccf::crypto::Pem& self_signed_node_cert_, + const std::optional& endorsed_node_cert_) : NodeClient( rpc_map, node_sign_kp, self_signed_node_cert_, endorsed_node_cert_) {} virtual ~HTTPNodeClient() {} - virtual bool make_request(http::Request& request) override + virtual bool make_request(::http::Request& request) override { const auto& node_cert = endorsed_node_cert.has_value() ? endorsed_node_cert.value() : @@ -35,7 +35,7 @@ namespace ccf auto ctx = ccf::make_rpc_context(node_session, packed); std::shared_ptr search = - http::fetch_rpc_handler(ctx, rpc_map); + ::http::fetch_rpc_handler(ctx, rpc_map); search->process(ctx); diff --git a/src/node/identity.h b/src/node/identity.h index 28402706e23a..e6754298e39e 100644 --- a/src/node/identity.h +++ b/src/node/identity.h @@ -20,8 +20,8 @@ namespace ccf struct NetworkIdentity { - crypto::Pem priv_key; - crypto::Pem cert; + ccf::crypto::Pem priv_key; + ccf::crypto::Pem cert; std::optional type = IdentityType::REPLICATED; std::string subject_name = "CN=CCF Service"; @@ -37,13 +37,13 @@ namespace ccf {} NetworkIdentity() = default; - virtual crypto::Pem issue_certificate( + virtual ccf::crypto::Pem issue_certificate( const std::string& valid_from, size_t validity_period_days) { return {}; } - virtual void set_certificate(const crypto::Pem& certificate) {} + virtual void set_certificate(const ccf::crypto::Pem& certificate) {} virtual ~NetworkIdentity() {} }; @@ -55,16 +55,16 @@ namespace ccf ReplicatedNetworkIdentity( const std::string& subject_name_, - crypto::CurveID curve_id, + ccf::crypto::CurveID curve_id, const std::string& valid_from, size_t validity_period_days) : NetworkIdentity(subject_name_) { auto identity_key_pair = - std::make_shared(curve_id); + std::make_shared(curve_id); priv_key = identity_key_pair->private_key_pem(); - cert = crypto::create_self_signed_cert( + cert = ccf::crypto::create_self_signed_cert( identity_key_pair, subject_name, {} /* SAN */, @@ -83,13 +83,13 @@ namespace ccf cert = other.cert; } - virtual crypto::Pem issue_certificate( + virtual ccf::crypto::Pem issue_certificate( const std::string& valid_from, size_t validity_period_days) override { auto identity_key_pair = - std::make_shared(priv_key); + std::make_shared(priv_key); - return crypto::create_self_signed_cert( + return ccf::crypto::create_self_signed_cert( identity_key_pair, subject_name, {} /* SAN */, @@ -97,7 +97,7 @@ namespace ccf validity_period_days); } - virtual void set_certificate(const crypto::Pem& new_cert) override + virtual void set_certificate(const ccf::crypto::Pem& new_cert) override { cert = new_cert; } diff --git a/src/node/jwt_key_auto_refresh.h b/src/node/jwt_key_auto_refresh.h index 25ed7721c5aa..b81129f79b6d 100644 --- a/src/node/jwt_key_auto_refresh.h +++ b/src/node/jwt_key_auto_refresh.h @@ -2,7 +2,6 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ccf/serdes.h" #include "ccf/service/tables/jwt.h" #include "http/http_builder.h" #include "http/http_rpc_context.h" @@ -18,22 +17,22 @@ namespace ccf private: size_t refresh_interval_s; NetworkState& network; - std::shared_ptr consensus; + std::shared_ptr consensus; std::shared_ptr rpcsessions; std::shared_ptr rpc_map; - crypto::KeyPairPtr node_sign_kp; - crypto::Pem node_cert; + ccf::crypto::KeyPairPtr node_sign_kp; + ccf::crypto::Pem node_cert; std::atomic_size_t attempts; public: JwtKeyAutoRefresh( size_t refresh_interval_s, NetworkState& network, - const std::shared_ptr& consensus, + const std::shared_ptr& consensus, const std::shared_ptr& rpcsessions, const std::shared_ptr& rpc_map, - const crypto::KeyPairPtr& node_sign_kp, - const crypto::Pem& node_cert) : + const ccf::crypto::KeyPairPtr& node_sign_kp, + const ccf::crypto::Pem& node_cert) : refresh_interval_s(refresh_interval_s), network(network), consensus(consensus), @@ -105,15 +104,15 @@ namespace ccf template void send_refresh_jwt_keys(T msg) { - auto body = serdes::pack(msg, serdes::Pack::Text); - - http::Request request(fmt::format( + ::http::Request request(fmt::format( "/{}/{}", ccf::get_actor_prefix(ccf::ActorsType::nodes), "jwt_keys/refresh")); request.set_header( http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); - request.set_body(&body); + + auto body = nlohmann::json(msg).dump(); + request.set_body(body); auto packed = request.build_request(); @@ -122,7 +121,7 @@ namespace ccf auto ctx = ccf::make_rpc_context(node_session, packed); std::shared_ptr search = - http::fetch_rpc_handler(ctx, this->rpc_map); + ::http::fetch_rpc_handler(ctx, this->rpc_map); search->process(ctx); } @@ -232,10 +231,10 @@ namespace ccf send_refresh_jwt_keys_error(); return; } - http::URL jwks_url; + ::http::URL jwks_url; try { - jwks_url = http::parse_url_full(jwks_url_str); + jwks_url = ::http::parse_url_full(jwks_url_str); } catch (const std::invalid_argument& e) { @@ -275,8 +274,8 @@ namespace ccf issuer, issuer_constraint, status, std::move(data)); return true; }); - http::Request r(jwks_url.path, HTTP_GET); - r.set_header(http::headers::HOST, std::string(jwks_url.host)); + ::http::Request r(jwks_url.path, HTTP_GET); + r.set_header(ccf::http::headers::HOST, std::string(jwks_url.host)); http_client->send_request(std::move(r)); } @@ -317,7 +316,7 @@ namespace ccf } auto metadata_url_str = issuer + "/.well-known/openid-configuration"; - auto metadata_url = http::parse_url_full(metadata_url_str); + auto metadata_url = ::http::parse_url_full(metadata_url_str); auto metadata_url_port = !metadata_url.port.empty() ? metadata_url.port : "443"; @@ -338,13 +337,13 @@ namespace ccf std::string(metadata_url_port), [this, issuer, ca]( http_status status, - http::HeaderMap&&, + ccf::http::HeaderMap&&, std::vector&& data) { handle_jwt_metadata_response(issuer, ca, status, std::move(data)); return true; }); - http::Request r(metadata_url.path, HTTP_GET); - r.set_header(http::headers::HOST, std::string(metadata_url.host)); + ::http::Request r(metadata_url.path, HTTP_GET); + r.set_header(ccf::http::headers::HOST, std::string(metadata_url.host)); http_client->send_request(std::move(r)); return true; }); diff --git a/src/node/ledger_secret.h b/src/node/ledger_secret.h index 9cecd05d21d9..8552e55ac430 100644 --- a/src/node/ledger_secret.h +++ b/src/node/ledger_secret.h @@ -18,16 +18,17 @@ namespace ccf struct LedgerSecret { std::vector raw_key; - std::shared_ptr key; - std::optional previous_secret_stored_version = std::nullopt; - std::optional commit_secret = std::nullopt; + std::shared_ptr key; + std::optional previous_secret_stored_version = + std::nullopt; + std::optional commit_secret = std::nullopt; - const crypto::HashBytes& get_commit_secret() + const ccf::crypto::HashBytes& get_commit_secret() { if (!commit_secret.has_value()) { - commit_secret = crypto::hmac( - crypto::MDType::SHA256, + commit_secret = ccf::crypto::hmac( + ccf::crypto::MDType::SHA256, raw_key, {commit_secret_label_, commit_secret_label_ + sizeof(commit_secret_label_)}); @@ -53,16 +54,16 @@ namespace ccf // seed key. LedgerSecret(const LedgerSecret& other) : raw_key(other.raw_key), - key(crypto::make_key_aes_gcm(other.raw_key)), + key(ccf::crypto::make_key_aes_gcm(other.raw_key)), previous_secret_stored_version(other.previous_secret_stored_version) {} LedgerSecret( std::vector&& raw_key_, - std::optional previous_secret_stored_version_ = + std::optional previous_secret_stored_version_ = std::nullopt) : raw_key(raw_key_), - key(crypto::make_key_aes_gcm(std::move(raw_key_))), + key(ccf::crypto::make_key_aes_gcm(std::move(raw_key_))), previous_secret_stored_version(previous_secret_stored_version_) {} }; @@ -76,14 +77,14 @@ namespace ccf inline LedgerSecretPtr make_ledger_secret() { return std::make_shared( - crypto::get_entropy()->random(crypto::GCM_DEFAULT_KEY_SIZE)); + ccf::crypto::get_entropy()->random(ccf::crypto::GCM_DEFAULT_KEY_SIZE)); } inline std::vector decrypt_previous_ledger_secret_raw( const LedgerSecretPtr& ledger_secret, const std::vector& encrypted_previous_secret_raw) { - crypto::GcmCipher encrypted_ls; + ccf::crypto::GcmCipher encrypted_ls; encrypted_ls.deserialise(encrypted_previous_secret_raw); std::vector decrypted_ls_raw; diff --git a/src/node/ledger_secrets.h b/src/node/ledger_secrets.h index a440e67923b3..c30223d876d9 100644 --- a/src/node/ledger_secrets.h +++ b/src/node/ledger_secrets.h @@ -16,7 +16,7 @@ namespace ccf { - using LedgerSecretsMap = std::map; + using LedgerSecretsMap = std::map; using VersionedLedgerSecret = LedgerSecretsMap::value_type; class LedgerSecrets @@ -30,13 +30,13 @@ namespace ccf // All rollback that would result in the removal of some of these secrets // would imply that the transaction that added the node itself was rolled // back. - kv::Version initial_latest_ledger_secret_version = 0; + ccf::kv::Version initial_latest_ledger_secret_version = 0; std::optional last_used_secret_it = std::nullopt; LedgerSecretPtr get_secret_for_version( - kv::Version version, bool historical_hint = false) + ccf::kv::Version version, bool historical_hint = false) { if (ledger_secrets.empty()) { @@ -91,7 +91,7 @@ namespace ccf return std::prev(search)->second; } - void take_dependency_on_secrets(kv::ReadOnlyTx& tx) + void take_dependency_on_secrets(ccf::kv::ReadOnlyTx& tx) { // Ledger secrets are not stored in the KV. Instead, they are // cached in a unique LedgerSecrets instance that can be accessed @@ -107,7 +107,7 @@ namespace ccf public: LedgerSecrets() = default; - void init(kv::Version initial_version = 1) + void init(ccf::kv::Version initial_version = 1) { std::lock_guard guard(lock); @@ -126,7 +126,7 @@ namespace ccf initial_latest_ledger_secret_version = ledger_secrets.rbegin()->first; } - void adjust_previous_secret_stored_version(kv::Version version) + void adjust_previous_secret_stored_version(ccf::kv::Version version) { // To be able to lookup the last active ledger secret before the service // crashed, the ledger secret created after the public recovery is @@ -166,7 +166,7 @@ namespace ccf return *ledger_secrets.begin(); } - VersionedLedgerSecret get_latest(kv::ReadOnlyTx& tx) + VersionedLedgerSecret get_latest(ccf::kv::ReadOnlyTx& tx) { std::lock_guard guard(lock); @@ -182,7 +182,7 @@ namespace ccf } std::pair> - get_latest_and_penultimate(kv::ReadOnlyTx& tx) + get_latest_and_penultimate(ccf::kv::ReadOnlyTx& tx) { std::lock_guard guard(lock); @@ -204,7 +204,8 @@ namespace ccf } LedgerSecretsMap get( - kv::ReadOnlyTx& tx, std::optional up_to = std::nullopt) + ccf::kv::ReadOnlyTx& tx, + std::optional up_to = std::nullopt) { std::lock_guard guard(lock); @@ -244,8 +245,8 @@ namespace ccf ledger_secrets.merge(restored_ledger_secrets); } - std::shared_ptr get_encryption_key_for( - kv::Version version, bool historical_hint = false) + std::shared_ptr get_encryption_key_for( + ccf::kv::Version version, bool historical_hint = false) { std::lock_guard guard(lock); auto ls = get_secret_for_version(version, historical_hint); @@ -257,13 +258,13 @@ namespace ccf } LedgerSecretPtr get_secret_for( - kv::Version version, bool historical_hint = false) + ccf::kv::Version version, bool historical_hint = false) { std::lock_guard guard(lock); return get_secret_for_version(version, historical_hint); } - void set_secret(kv::Version version, LedgerSecretPtr&& secret) + void set_secret(ccf::kv::Version version, LedgerSecretPtr&& secret) { std::lock_guard guard(lock); @@ -277,7 +278,7 @@ namespace ccf LOG_INFO_FMT("Added new ledger secret at seqno {}", version); } - void rollback(kv::Version version) + void rollback(ccf::kv::Version version) { std::lock_guard guard(lock); if (ledger_secrets.empty()) diff --git a/src/node/node_client.h b/src/node/node_client.h index 8a7c2275f688..0ef4ca117933 100644 --- a/src/node/node_client.h +++ b/src/node/node_client.h @@ -13,16 +13,16 @@ namespace ccf { protected: std::shared_ptr rpc_map; - crypto::KeyPairPtr node_sign_kp; - const crypto::Pem& self_signed_node_cert; - const std::optional& endorsed_node_cert = std::nullopt; + ccf::crypto::KeyPairPtr node_sign_kp; + const ccf::crypto::Pem& self_signed_node_cert; + const std::optional& endorsed_node_cert = std::nullopt; public: NodeClient( std::shared_ptr rpc_map_, - crypto::KeyPairPtr node_sign_kp_, - const crypto::Pem& self_signed_node_cert_, - const std::optional& endorsed_node_cert_) : + ccf::crypto::KeyPairPtr node_sign_kp_, + const ccf::crypto::Pem& self_signed_node_cert_, + const std::optional& endorsed_node_cert_) : rpc_map(rpc_map_), node_sign_kp(node_sign_kp_), self_signed_node_cert(self_signed_node_cert_), @@ -31,6 +31,6 @@ namespace ccf virtual ~NodeClient() {} - virtual bool make_request(http::Request& request) = 0; + virtual bool make_request(::http::Request& request) = 0; }; } diff --git a/src/node/node_signature_verify.h b/src/node/node_signature_verify.h index 2e2de87e714c..b3b7bacf59cd 100644 --- a/src/node/node_signature_verify.h +++ b/src/node/node_signature_verify.h @@ -9,14 +9,14 @@ namespace ccf { static bool verify_node_signature( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const NodeId& node_id, const uint8_t* expected_sig, size_t expected_sig_size, const uint8_t* expected_root, size_t expected_root_size) { - crypto::Pem node_cert; + ccf::crypto::Pem node_cert; auto node_endorsed_certs = tx.template ro( ccf::Tables::NODE_ENDORSED_CERTIFICATES); auto node_endorsed_cert = node_endorsed_certs->get(node_id); @@ -47,20 +47,20 @@ namespace ccf node_cert = node_endorsed_cert.value(); } - crypto::VerifierPtr from_cert = crypto::make_verifier(node_cert); + ccf::crypto::VerifierPtr from_cert = ccf::crypto::make_verifier(node_cert); return from_cert->verify_hash( expected_root, expected_root_size, expected_sig, expected_sig_size, - crypto::MDType::SHA256); + ccf::crypto::MDType::SHA256); } static bool verify_node_signature( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const NodeId& node_id, const std::vector& expected_sig, - const crypto::Sha256Hash& expected_root) + const ccf::crypto::Sha256Hash& expected_root) { return verify_node_signature( tx, diff --git a/src/node/node_state.h b/src/node/node_state.h index cdc09b67188d..485eb21d086e 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -11,7 +11,6 @@ #include "ccf/pal/attestation.h" #include "ccf/pal/locking.h" #include "ccf/pal/platform.h" -#include "ccf/serdes.h" #include "ccf/service/node_info_network.h" #include "ccf/service/tables/acme_certificates.h" #include "ccf/service/tables/service.h" @@ -61,8 +60,8 @@ namespace ccf struct NodeCreateInfo { - crypto::Pem self_signed_node_cert; - crypto::Pem service_cert; + ccf::crypto::Pem self_signed_node_cert; + ccf::crypto::Pem service_cert; }; void reset_data(std::vector& data) @@ -81,14 +80,14 @@ namespace ccf pal::Mutex lock; StartType start_type; - crypto::CurveID curve_id; - std::vector subject_alt_names = {}; + ccf::crypto::CurveID curve_id; + std::vector subject_alt_names = {}; - std::shared_ptr node_sign_kp; + std::shared_ptr node_sign_kp; NodeId self; - std::shared_ptr node_encrypt_kp; - crypto::Pem self_signed_node_cert; - std::optional endorsed_node_cert = std::nullopt; + std::shared_ptr node_encrypt_kp; + ccf::crypto::Pem self_signed_node_cert; + std::optional endorsed_node_cert = std::nullopt; QuoteInfo quote_info; pal::PlatformAttestationMeasurement node_measurement; StartupConfig config; @@ -125,15 +124,15 @@ namespace ccf NetworkState& network; - std::shared_ptr consensus; + std::shared_ptr consensus; std::shared_ptr rpc_map; std::shared_ptr indexer; std::shared_ptr n2n_channels; std::shared_ptr> cmd_forwarder; std::shared_ptr rpcsessions; - std::shared_ptr history; - std::shared_ptr encryptor; + std::shared_ptr history; + std::shared_ptr encryptor; ShareManager share_manager; std::shared_ptr snapshotter; @@ -141,11 +140,11 @@ namespace ccf // // recovery // - std::shared_ptr recovery_store; + std::shared_ptr recovery_store; - kv::Version recovery_v; - crypto::Sha256Hash recovery_root; - std::vector view_history; + ccf::kv::Version recovery_v; + ccf::crypto::Sha256Hash recovery_root; + std::vector view_history; ::consensus::Index last_recovered_signed_idx = 0; RecoveredEncryptedLedgerSecrets recovered_encrypted_ledger_secrets = {}; LedgerSecretsMap recovered_ledger_secrets = {}; @@ -160,7 +159,7 @@ namespace ccf std::unique_ptr startup_snapshot_info = nullptr; // Set to the snapshot seqno when a node starts from one and remembered for // the lifetime of the node - kv::Version startup_seqno = 0; + ccf::kv::Version startup_seqno = 0; // ACME certificate endorsement client std::map> @@ -171,10 +170,10 @@ namespace ccf acme_challenge_handlers; size_t num_acme_interfaces = 0; - std::shared_ptr make_encryptor() + std::shared_ptr make_encryptor() { #ifdef USE_NULL_ENCRYPTOR - return std::make_shared(); + return std::make_shared(); #else return std::make_shared(network.ledger_secrets); #endif @@ -183,7 +182,7 @@ namespace ccf // Returns true if the snapshot is already verified (via embedded receipt) void initialise_startup_snapshot(bool recovery = false) { - std::shared_ptr snapshot_store; + std::shared_ptr snapshot_store; if (!recovery) { // Create a new store to verify the snapshot only @@ -206,7 +205,7 @@ namespace ccf snapshot_store = network.tables; } - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; startup_snapshot_info = initialise_from_snapshot( snapshot_store, std::move(startup_snapshot), @@ -225,12 +224,12 @@ namespace ccf ringbuffer::AbstractWriterFactory& writer_factory, NetworkState& network, std::shared_ptr rpcsessions, - crypto::CurveID curve_id_) : + ccf::crypto::CurveID curve_id_) : sm("NodeState", NodeStartupState::uninitialized), curve_id(curve_id_), - node_sign_kp(std::make_shared(curve_id_)), + node_sign_kp(std::make_shared(curve_id_)), self(compute_node_id_from_kp(node_sign_kp)), - node_encrypt_kp(crypto::make_rsa_key_pair()), + node_encrypt_kp(ccf::crypto::make_rsa_key_pair()), writer_factory(writer_factory), to_host(writer_factory.create_writer_to_outside()), network(network), @@ -239,7 +238,7 @@ namespace ccf {} QuoteVerificationResult verify_quote( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info_, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) override @@ -318,7 +317,7 @@ namespace ccf config.attestation.environment.security_policy.value(); auto security_policy_digest = - crypto::Sha256Hash(crypto::raw_from_b64(security_policy)); + ccf::crypto::Sha256Hash(ccf::crypto::raw_from_b64(security_policy)); if (security_policy_digest != quoted_digest.value()) { throw std::logic_error(fmt::format( @@ -343,7 +342,7 @@ namespace ccf { try { - auto uvm_endorsements_raw = crypto::raw_from_b64( + auto uvm_endorsements_raw = ccf::crypto::raw_from_b64( config.attestation.environment.uvm_endorsements.value()); snp_uvm_endorsements = verify_uvm_endorsements(uvm_endorsements_raw, node_measurement); @@ -452,7 +451,7 @@ namespace ccf }; pal::PlatformAttestationReportData report_data = - crypto::Sha256Hash((node_sign_kp->public_key_der())); + ccf::crypto::Sha256Hash((node_sign_kp->public_key_der())); pal::generate_quote( report_data, @@ -532,11 +531,11 @@ namespace ccf "identity"); } - crypto::Pem previous_service_identity_cert( + ccf::crypto::Pem previous_service_identity_cert( config.recover.previous_service_identity.value()); network.identity = std::make_unique( - crypto::get_subject_name(previous_service_identity_cert), + ccf::crypto::get_subject_name(previous_service_identity_cert), curve_id, config.startup_host_time, config.initial_service_certificate_validity_days); @@ -601,7 +600,7 @@ namespace ccf status == HTTP_STATUS_TEMPORARY_REDIRECT) && location != headers.end()) { - const auto& url = http::parse_url_full(location->second); + const auto& url = ::http::parse_url_full(location->second); config.join.target_rpc_address = make_net_address(url.host, url.port); LOG_INFO_FMT("Target node redirected to {}", location->second); @@ -619,7 +618,7 @@ namespace ccf return; } - auto j = serdes::unpack(data, serdes::Pack::Text); + auto j = nlohmann::json::parse(data); JoinNetworkNodeToNode::Out resp; try @@ -644,7 +643,7 @@ namespace ccf network.ledger_secrets->init_from_map( std::move(resp.network_info->ledger_secrets)); - crypto::Pem n2n_channels_cert; + ccf::crypto::Pem n2n_channels_cert; if (!resp.network_info->endorsed_certificate.has_value()) { // Endorsed certificate was added to join response in 2.x @@ -670,12 +669,12 @@ namespace ccf } View view = VIEW_UNKNOWN; - std::vector view_history_ = {}; + std::vector view_history_ = {}; if (startup_snapshot_info) { // It is only possible to deserialise the entire snapshot then, // once the ledger secrets have been passed in by the network - kv::ConsensusHookPtrs hooks; + ccf::kv::ConsensusHookPtrs hooks; deserialise_snapshot( network.tables, startup_snapshot_info->raw, @@ -773,16 +772,15 @@ namespace ccf LOG_DEBUG_FMT( "Sending join request to {}", config.join.target_rpc_address); - const auto body = serdes::pack(join_params, serdes::Pack::Text); + const auto body = nlohmann::json(join_params).dump(); - LOG_DEBUG_FMT( - "Sending join request body: {}", std::string(body.begin(), body.end())); + LOG_DEBUG_FMT("Sending join request body: {}", body); - http::Request r( + ::http::Request r( fmt::format("/{}/{}", get_actor_prefix(ActorsType::nodes), "join")); r.set_header( http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); - r.set_body(&body); + r.set_body(body); join_client->send_request(std::move(r)); } @@ -837,9 +835,10 @@ namespace ccf network.tables->set_map_hook( network.jwt_issuers.get_name(), - [this](kv::Version, const kv::untyped::Write&) -> kv::ConsensusHookPtr { + [this](ccf::kv::Version, const ccf::kv::untyped::Write&) + -> ccf::kv::ConsensusHookPtr { jwt_key_auto_refresh->schedule_once(); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); }); } @@ -889,12 +888,12 @@ namespace ccf // When reading the private ledger, deserialise in the recovery store - kv::ApplyResult result = kv::ApplyResult::FAIL; + ccf::kv::ApplyResult result = ccf::kv::ApplyResult::FAIL; try { auto r = network.tables->deserialize(entry, true); result = r->apply(); - if (result == kv::ApplyResult::FAIL) + if (result == ccf::kv::ApplyResult::FAIL) { LOG_FAIL_FMT( "Failed to deserialise public ledger entry: {}", result); @@ -918,7 +917,7 @@ namespace ccf } // If the ledger entry is a signature, it is safe to compact the store - if (result == kv::ApplyResult::PASS_SIGNATURE) + if (result == ccf::kv::ApplyResult::PASS_SIGNATURE) { // If the ledger entry is a signature, it is safe to compact the store network.tables->compact(last_recovered_idx); @@ -1009,8 +1008,8 @@ namespace ccf snapshotter->init_after_public_recovery(); snapshotter->set_snapshot_generation(false); - kv::Version index = 0; - kv::Term view = 0; + ccf::kv::Version index = 0; + ccf::kv::Term view = 0; auto ls = tx.ro(network.signatures)->get(); if (ls.has_value()) @@ -1069,11 +1068,11 @@ namespace ccf LOG_INFO_FMT("Deserialising private ledger entry [{}]", entry.size()); // When reading the private ledger, deserialise in the recovery store - kv::ApplyResult result = kv::ApplyResult::FAIL; + ccf::kv::ApplyResult result = ccf::kv::ApplyResult::FAIL; try { result = recovery_store->deserialize(entry)->apply(); - if (result == kv::ApplyResult::FAIL) + if (result == ccf::kv::ApplyResult::FAIL) { LOG_FAIL_FMT( "Failed to deserialise private ledger entry: {}", result); @@ -1093,7 +1092,7 @@ namespace ccf return; } - if (result == kv::ApplyResult::PASS_SIGNATURE) + if (result == ccf::kv::ApplyResult::PASS_SIGNATURE) { recovery_store->compact(last_recovered_idx); } @@ -1169,7 +1168,7 @@ namespace ccf // we need to recover soon again. trigger_snapshot(tx); - if (tx.commit() != kv::CommitResult::SUCCESS) + if (tx.commit() != ccf::kv::CommitResult::SUCCESS) { throw std::logic_error( "Could not commit transaction when finishing network recovery"); @@ -1191,8 +1190,9 @@ namespace ccf network.encrypted_ledger_secrets.get_name(), network.encrypted_ledger_secrets.wrap_map_hook( [this]( - kv::Version version, const EncryptedLedgerSecretsInfo::Write& w) - -> kv::ConsensusHookPtr { + ccf::kv::Version version, + const EncryptedLedgerSecretsInfo::Write& w) + -> ccf::kv::ConsensusHookPtr { if (!w.has_value()) { throw std::logic_error(fmt::format( @@ -1206,7 +1206,7 @@ namespace ccf network.tables->unset_map_hook( network.encrypted_ledger_secrets.get_name()); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); } @@ -1238,7 +1238,7 @@ namespace ccf // void setup_private_recovery_store() { - recovery_store = std::make_shared( + recovery_store = std::make_shared( true /* Check transactions in order */, true /* Make use of historical secrets */); auto recovery_history = std::make_shared( @@ -1261,8 +1261,8 @@ namespace ccf if (startup_snapshot_info) { - std::vector view_history_; - kv::ConsensusHookPtrs hooks; + std::vector view_history_; + ccf::kv::ConsensusHookPtrs hooks; deserialise_snapshot( recovery_store, startup_snapshot_info->raw, @@ -1279,34 +1279,35 @@ namespace ccf recovery_v); } - void trigger_recovery_shares_refresh(kv::Tx& tx) override + void trigger_recovery_shares_refresh(ccf::kv::Tx& tx) override { share_manager.shuffle_recovery_shares(tx); } - void trigger_ledger_chunk(kv::Tx& tx) override + void trigger_ledger_chunk(ccf::kv::Tx& tx) override { - auto tx_ = static_cast(&tx); + auto tx_ = static_cast(&tx); if (tx_ == nullptr) { throw std::logic_error("Could not cast tx to CommittableTx"); } - tx_->set_flag(kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); + tx_->set_flag( + ccf::kv::CommittableTx::Flag::LEDGER_CHUNK_AT_NEXT_SIGNATURE); } - void trigger_snapshot(kv::Tx& tx) override + void trigger_snapshot(ccf::kv::Tx& tx) override { - auto committable_tx = static_cast(&tx); + auto committable_tx = static_cast(&tx); if (committable_tx == nullptr) { throw std::logic_error("Could not cast tx to CommittableTx"); } committable_tx->set_flag( - kv::CommittableTx::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); + ccf::kv::CommittableTx::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); } void trigger_acme_refresh( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::optional>& interfaces = std::nullopt) override { @@ -1393,7 +1394,7 @@ namespace ccf } void transition_service_to_open( - kv::Tx& tx, + ccf::kv::Tx& tx, AbstractGovernanceEffects::ServiceIdentities identities) override { std::lock_guard guard(lock); @@ -1432,7 +1433,7 @@ namespace ccf "next service certificates"); } - const crypto::Pem from_proposal( + const ccf::crypto::Pem from_proposal( identities.previous->data(), identities.previous->size()); if (prev_ident.value() != from_proposal) { @@ -1492,7 +1493,7 @@ namespace ccf } } - void initiate_private_recovery(kv::Tx& tx) override + void initiate_private_recovery(ccf::kv::Tx& tx) override { std::lock_guard guard(lock); sm.expect(NodeStartupState::partOfPublicNetwork); @@ -1671,7 +1672,7 @@ namespace ccf } } - bool rekey_ledger(kv::Tx& tx) override + bool rekey_ledger(ccf::kv::Tx& tx) override { std::lock_guard guard(lock); sm.expect(NodeStartupState::partOfNetwork); @@ -1708,7 +1709,7 @@ namespace ccf return self; } - kv::Version get_startup_snapshot_seqno() override + ccf::kv::Version get_startup_snapshot_seqno() override { std::lock_guard guard(lock); return startup_seqno; @@ -1719,7 +1720,7 @@ namespace ccf return rpcsessions->get_session_metrics(); } - crypto::Pem get_self_signed_certificate() override + ccf::crypto::Pem get_self_signed_certificate() override { std::lock_guard guard(lock); return self_signed_node_cert; @@ -1735,7 +1736,8 @@ namespace ccf // throw. Attempts to handle IPv6 by also splitting on ':', but this is // untested. const auto final_component = - nonstd::split(nonstd::split(hostname, ".").back(), ":").back(); + ccf::nonstd::split(ccf::nonstd::split(hostname, ".").back(), ":") + .back(); if (final_component.empty()) { throw std::runtime_error(fmt::format( @@ -1752,21 +1754,21 @@ namespace ccf return true; } - std::vector get_subject_alternative_names() + std::vector get_subject_alternative_names() { // If no Subject Alternative Name (SAN) is passed in at node creation, // default to using node's RPC address as single SAN. Otherwise, use // specified SANs. if (!config.node_certificate.subject_alt_names.empty()) { - return crypto::sans_from_string_list( + return ccf::crypto::sans_from_string_list( config.node_certificate.subject_alt_names); } else { // Construct SANs from RPC interfaces, manually detecting whether each // is a domain name or IP - std::vector sans; + std::vector sans; for (const auto& [_, interface] : config.network.rpc_interfaces) { auto host = split_net_address(interface.published_address).first; @@ -1874,12 +1876,13 @@ namespace ccf create_params.node_id = self; create_params.certificate_signing_request = node_sign_kp->create_csr( config.node_certificate.subject_name, subject_alt_names); - create_params.node_endorsed_certificate = crypto::create_endorsed_cert( - create_params.certificate_signing_request, - config.startup_host_time, - config.node_certificate.initial_validity_days, - network.identity->priv_key, - network.identity->cert); + create_params.node_endorsed_certificate = + ccf::crypto::create_endorsed_cert( + create_params.certificate_signing_request, + config.startup_host_time, + config.node_certificate.initial_validity_days, + network.identity->priv_key, + network.identity->cert); // Even though endorsed certificate is updated on (global) hook, history // requires it to generate signatures @@ -1900,14 +1903,15 @@ namespace ccf create_params.service_data = config.service_data; create_params.create_txid = {create_view, last_recovered_signed_idx + 1}; - const auto body = serdes::pack(create_params, serdes::Pack::Text); + const auto body = nlohmann::json(create_params).dump(); - http::Request request( + ::http::Request request( fmt::format("/{}/{}", get_actor_prefix(ActorsType::nodes), "create")); request.set_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); - request.set_body(&body); + request.set_body(body); return request.build_request(); } @@ -1930,8 +1934,7 @@ namespace ccf return false; } - const auto body = - serdes::unpack(ctx->get_response_body(), serdes::Pack::Text); + const auto body = nlohmann::json::parse(ctx->get_response_body()); if (!body.is_boolean()) { LOG_FAIL_FMT("Expected boolean body in create response"); @@ -1950,7 +1953,7 @@ namespace ccf auto ctx = make_rpc_context(node_session, packed); std::shared_ptr search = - http::fetch_rpc_handler(ctx, this->rpc_map); + ::http::fetch_rpc_handler(ctx, this->rpc_map); search->process(ctx); @@ -2015,13 +2018,13 @@ namespace ccf network.tables->set_map_hook( network.secrets.get_name(), network.secrets.wrap_map_hook( - [this](kv::Version hook_version, const Secrets::Write& w) - -> kv::ConsensusHookPtr { + [this](ccf::kv::Version hook_version, const Secrets::Write& w) + -> ccf::kv::ConsensusHookPtr { // Used to rekey the ledger on a live service if (!is_part_of_network()) { // Ledger rekey is not allowed during recovery - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); } const auto& ledger_secrets_for_nodes = w; @@ -2057,13 +2060,13 @@ namespace ccf } } - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); network.tables->set_global_hook( network.secrets.get_name(), network.secrets.wrap_commit_hook([this]( - kv::Version hook_version, + ccf::kv::Version hook_version, const Secrets::Write& w) { // Used on recovery to initiate private recovery on backup nodes. if (!is_part_of_public_network()) @@ -2128,7 +2131,7 @@ namespace ccf network.encrypted_submitted_shares.get_name(), network.encrypted_submitted_shares.wrap_commit_hook( [this]( - kv::Version hook_version, + ccf::kv::Version hook_version, const EncryptedSubmittedShares::Write& w) { // Initiate recovery procedure from global hook, once all recovery // shares have been submitted (i.e. recovered_ledger_secrets is @@ -2158,7 +2161,7 @@ namespace ccf network.tables->set_global_hook( network.nodes.get_name(), network.nodes.wrap_commit_hook( - [this](kv::Version hook_version, const Nodes::Write& w) { + [this](ccf::kv::Version hook_version, const Nodes::Write& w) { std::vector retired_committed_nodes; for (const auto& [node_id, node_info] : w) { @@ -2181,8 +2184,9 @@ namespace ccf network.node_endorsed_certificates.get_name(), network.node_endorsed_certificates.wrap_map_hook( [this]( - kv::Version hook_version, - const NodeEndorsedCertificates::Write& w) -> kv::ConsensusHookPtr { + ccf::kv::Version hook_version, + const NodeEndorsedCertificates::Write& w) + -> ccf::kv::ConsensusHookPtr { for (auto const& [node_id, endorsed_certificate] : w) { if (node_id != self) @@ -2203,14 +2207,14 @@ namespace ccf n2n_channels->set_endorsed_node_cert(endorsed_node_cert.value()); } - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); network.tables->set_global_hook( network.node_endorsed_certificates.get_name(), network.node_endorsed_certificates.wrap_commit_hook( [this]( - kv::Version hook_version, + ccf::kv::Version hook_version, const NodeEndorsedCertificates::Write& w) { for (auto const& [node_id, endorsed_certificate] : w) { @@ -2238,7 +2242,7 @@ namespace ccf // for the initial addition of the node (the self-signed // certificate is output to disk then). auto [valid_from, valid_to] = - crypto::make_verifier(endorsed_node_cert.value()) + ccf::crypto::make_verifier(endorsed_node_cert.value()) ->validity_period(); self_signed_node_cert = create_self_signed_cert( node_sign_kp, @@ -2255,42 +2259,43 @@ namespace ccf network.tables->set_global_hook( network.service.get_name(), - network.service.wrap_commit_hook([this]( - kv::Version hook_version, - const Service::Write& w) { - if (!w.has_value()) - { - throw std::logic_error("Unexpected deletion in service value"); - } + network.service.wrap_commit_hook( + [this](ccf::kv::Version hook_version, const Service::Write& w) { + if (!w.has_value()) + { + throw std::logic_error("Unexpected deletion in service value"); + } - // Service open on historical service has no effect - auto hook_pubk_pem = - crypto::public_key_pem_from_cert(crypto::cert_pem_to_der(w->cert)); - auto current_pubk_pem = - crypto::make_key_pair(network.identity->priv_key)->public_key_pem(); - if (hook_pubk_pem != current_pubk_pem) - { - LOG_TRACE_FMT( - "Ignoring historical service open at seqno {} for {}", - hook_version, - w->cert.str()); - return; - } + // Service open on historical service has no effect + auto hook_pubk_pem = ccf::crypto::public_key_pem_from_cert( + ccf::crypto::cert_pem_to_der(w->cert)); + auto current_pubk_pem = + ccf::crypto::make_key_pair(network.identity->priv_key) + ->public_key_pem(); + if (hook_pubk_pem != current_pubk_pem) + { + LOG_TRACE_FMT( + "Ignoring historical service open at seqno {} for {}", + hook_version, + w->cert.str()); + return; + } - network.identity->set_certificate(w->cert); - if (w->status == ServiceStatus::OPEN) - { - open_user_frontend(); + network.identity->set_certificate(w->cert); + if (w->status == ServiceStatus::OPEN) + { + open_user_frontend(); - RINGBUFFER_WRITE_MESSAGE(::consensus::ledger_open, to_host); - LOG_INFO_FMT("Service open at seqno {}", hook_version); - } - })); + RINGBUFFER_WRITE_MESSAGE(::consensus::ledger_open, to_host); + LOG_INFO_FMT("Service open at seqno {}", hook_version); + } + })); network.tables->set_global_hook( network.acme_certificates.get_name(), network.acme_certificates.wrap_commit_hook( - [this](kv::Version hook_version, const ACMECertificates::Write& w) { + [this]( + ccf::kv::Version hook_version, const ACMECertificates::Write& w) { for (auto const& [interface_id, interface] : config.network.rpc_interfaces) { @@ -2315,7 +2320,7 @@ namespace ccf })); } - kv::Version get_last_recovered_signed_idx() override + ccf::kv::Version get_last_recovered_signed_idx() override { // On recovery, only one node recovers the public ledger and is thus // aware of the version at which the new ledger secret is applicable @@ -2332,8 +2337,9 @@ namespace ccf network.encrypted_ledger_secrets.get_name(), network.encrypted_ledger_secrets.wrap_map_hook( [this]( - kv::Version version, const EncryptedLedgerSecretsInfo::Write& w) - -> kv::ConsensusHookPtr { + ccf::kv::Version version, + const EncryptedLedgerSecretsInfo::Write& w) + -> ccf::kv::ConsensusHookPtr { auto encrypted_ledger_secret_info = w; if (!encrypted_ledger_secret_info.has_value()) { @@ -2360,7 +2366,7 @@ namespace ccf recovered_encrypted_ledger_secrets.emplace_back( std::move(encrypted_ledger_secret_info.value())); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); } @@ -2371,7 +2377,7 @@ namespace ccf } void setup_n2n_channels( - const std::optional& endorsed_node_certificate_ = + const std::optional& endorsed_node_certificate_ = std::nullopt) { // If the endorsed node certificate is available at the time the @@ -2419,7 +2425,7 @@ namespace ccf ServiceStatus service_status, ReconfigurationType reconfiguration_type, bool public_only = false, - const std::optional& endorsed_node_certificate_ = + const std::optional& endorsed_node_certificate_ = std::nullopt) { setup_n2n_channels(endorsed_node_certificate_); @@ -2430,11 +2436,12 @@ namespace ccf auto node_client = std::make_shared( rpc_map, node_sign_kp, self_signed_node_cert, endorsed_node_cert); - kv::MembershipState membership_state = kv::MembershipState::Active; + ccf::kv::MembershipState membership_state = + ccf::kv::MembershipState::Active; consensus = std::make_shared( consensus_config, - std::make_unique>(network.tables), + std::make_unique>(network.tables), std::make_unique<::consensus::LedgerEnclave>(writer_factory), n2n_channels, shared_state, @@ -2451,8 +2458,8 @@ namespace ccf network.tables->set_map_hook( network.nodes.get_name(), network.nodes.wrap_map_hook( - [](kv::Version version, const Nodes::Write& w) - -> kv::ConsensusHookPtr { + [](ccf::kv::Version version, const Nodes::Write& w) + -> ccf::kv::ConsensusHookPtr { return std::make_unique(version, w); })); @@ -2466,33 +2473,33 @@ namespace ccf network.signatures.get_name(), network.signatures.wrap_map_hook( [s = this->snapshotter]( - kv::Version version, const Signatures::Write& w) { + ccf::kv::Version version, const Signatures::Write& w) { assert(w.has_value()); auto sig = w.value(); s->record_signature(version, sig.sig, sig.node, sig.cert); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); network.tables->set_map_hook( network.serialise_tree.get_name(), network.serialise_tree.wrap_map_hook( [s = this->snapshotter]( - kv::Version version, const SerialisedMerkleTree::Write& w) { + ccf::kv::Version version, const SerialisedMerkleTree::Write& w) { assert(w.has_value()); auto tree = w.value(); s->record_serialised_tree(version, tree); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); network.tables->set_map_hook( network.snapshot_evidence.get_name(), network.snapshot_evidence.wrap_map_hook( [s = this->snapshotter]( - kv::Version version, const SnapshotEvidence::Write& w) { + ccf::kv::Version version, const SnapshotEvidence::Write& w) { assert(w.has_value()); auto snapshot_evidence = w.value(); s->record_snapshot_evidence_idx(version, snapshot_evidence); - return kv::ConsensusHookPtr(nullptr); + return ccf::kv::ConsensusHookPtr(nullptr); })); setup_basic_hooks(); @@ -2597,7 +2604,7 @@ namespace ccf return config; } - virtual crypto::Pem get_network_cert() override + virtual ccf::crypto::Pem get_network_cert() override { return network.identity->cert; } @@ -2611,8 +2618,8 @@ namespace ccf // Stop-gap until it becomes easier to use other HTTP clients virtual void make_http_request( - const http::URL& url, - http::Request&& req, + const ::http::URL& url, + ::http::Request&& req, std::function< bool(http_status status, http::HeaderMap&&, std::vector&&)> callback, @@ -2620,8 +2627,8 @@ namespace ccf const std::string& app_protocol = "HTTP1", bool authenticate_as_node_client_certificate = false) override { - std::optional client_cert = std::nullopt; - std::optional client_cert_key = std::nullopt; + std::optional client_cert = std::nullopt; + std::optional client_cert_key = std::nullopt; if (authenticate_as_node_client_certificate) { client_cert = @@ -2650,7 +2657,7 @@ namespace ccf snapshotter->write_snapshot(snapshot_buf, request_id); } - virtual std::shared_ptr get_store() override + virtual std::shared_ptr get_store() override { return network.tables; } diff --git a/src/node/node_to_node.h b/src/node/node_to_node.h index 6e2af37cbc4b..19fabe592231 100644 --- a/src/node/node_to_node.h +++ b/src/node/node_to_node.h @@ -98,12 +98,12 @@ namespace ccf virtual void initialize( const NodeId& self_id, - const crypto::Pem& service_cert, - crypto::KeyPairPtr node_kp, - const std::optional& node_cert = std::nullopt) = 0; + const ccf::crypto::Pem& service_cert, + ccf::crypto::KeyPairPtr node_kp, + const std::optional& node_cert = std::nullopt) = 0; virtual void set_endorsed_node_cert( - const crypto::Pem& endorsed_node_cert) = 0; + const ccf::crypto::Pem& endorsed_node_cert) = 0; virtual bool send_encrypted( const NodeId& to, diff --git a/src/node/node_to_node_channel_manager.h b/src/node/node_to_node_channel_manager.h index 414f0b09b5f2..19d8610d927e 100644 --- a/src/node/node_to_node_channel_manager.h +++ b/src/node/node_to_node_channel_manager.h @@ -26,9 +26,9 @@ namespace ccf struct ThisNode { NodeId node_id; - crypto::Pem service_cert; - crypto::KeyPairPtr node_kp; - std::optional endorsed_node_cert = std::nullopt; + ccf::crypto::Pem service_cert; + ccf::crypto::KeyPairPtr node_kp; + std::optional endorsed_node_cert = std::nullopt; }; std::unique_ptr this_node; //< Not available at construction, only // after calling initialize() @@ -93,9 +93,9 @@ namespace ccf void initialize( const NodeId& self_id, - const crypto::Pem& service_cert, - crypto::KeyPairPtr node_kp, - const std::optional& node_cert) override + const ccf::crypto::Pem& service_cert, + ccf::crypto::KeyPairPtr node_kp, + const std::optional& node_cert) override { CCF_ASSERT_FMT( this_node == nullptr, @@ -117,7 +117,8 @@ namespace ccf new ThisNode{self_id, service_cert, node_kp, node_cert}); } - void set_endorsed_node_cert(const crypto::Pem& endorsed_node_cert) override + void set_endorsed_node_cert( + const ccf::crypto::Pem& endorsed_node_cert) override { std::lock_guard guard(lock); this_node->endorsed_node_cert = endorsed_node_cert; diff --git a/src/node/node_types.h b/src/node/node_types.h index 72f15892b944..7998cb891b50 100644 --- a/src/node/node_types.h +++ b/src/node/node_types.h @@ -107,13 +107,13 @@ namespace ccf struct MessageHash { MessageHash() = default; - MessageHash(ForwardedMsg msg_, crypto::Sha256Hash&& hash_) : + MessageHash(ForwardedMsg msg_, ccf::crypto::Sha256Hash&& hash_) : msg(msg_), hash(std::move(hash_)) {} ForwardedMsg msg; - crypto::Sha256Hash hash; + ccf::crypto::Sha256Hash hash; }; #pragma pack(pop) diff --git a/src/node/quote.cpp b/src/node/quote.cpp index e695fdd5115e..f475576ddcee 100644 --- a/src/node/quote.cpp +++ b/src/node/quote.cpp @@ -12,7 +12,7 @@ namespace ccf { bool verify_enclave_measurement_against_uvm_endorsements( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const pal::PlatformAttestationMeasurement& quote_measurement, const std::vector& uvm_endorsements) { @@ -44,7 +44,7 @@ namespace ccf } QuoteVerificationResult verify_enclave_measurement_against_store( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const pal::PlatformAttestationMeasurement& quote_measurement, const QuoteFormat& quote_format, const std::optional>& uvm_endorsements = std::nullopt) @@ -97,9 +97,9 @@ namespace ccf QuoteVerificationResult verify_quoted_node_public_key( const std::vector& expected_node_public_key, - const crypto::Sha256Hash& quoted_hash) + const ccf::crypto::Sha256Hash& quoted_hash) { - if (quoted_hash != crypto::Sha256Hash(expected_node_public_key)) + if (quoted_hash != ccf::crypto::Sha256Hash(expected_node_public_key)) { return QuoteVerificationResult::FailedInvalidQuotedPublicKey; } @@ -155,7 +155,7 @@ namespace ccf } QuoteVerificationResult verify_host_data_against_store( - kv::ReadOnlyTx& tx, const QuoteInfo& quote_info) + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info) { if (quote_info.format != QuoteFormat::amd_sev_snp_v1) { @@ -180,12 +180,12 @@ namespace ccf } QuoteVerificationResult AttestationProvider::verify_quote_against_store( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) { - crypto::Sha256Hash quoted_hash; + ccf::crypto::Sha256Hash quoted_hash; pal::PlatformAttestationReportData report_data; try { diff --git a/src/node/quote_endorsements_client.h b/src/node/quote_endorsements_client.h index 6805f62d108a..2a64b166df22 100644 --- a/src/node/quote_endorsements_client.h +++ b/src/node/quote_endorsements_client.h @@ -90,7 +90,7 @@ namespace ccf const EndpointInfo& endpoint) { { - http::Request r(endpoint.uri, HTTP_GET); + ::http::Request r(endpoint.uri, HTTP_GET); for (auto const& [k, v] : endpoint.params) { r.set_query_param(k, v); @@ -172,7 +172,7 @@ namespace ccf if (response_endpoint.response_is_der) { - auto raw = crypto::cert_der_to_pem(data).raw(); + auto raw = ccf::crypto::cert_der_to_pem(data).raw(); endorsements_pem.insert(endorsements_pem.end(), raw.begin(), raw.end()); } else if (response_endpoint.response_is_thim_json) diff --git a/src/node/receipt.cpp b/src/node/receipt.cpp index eacf2a729813..a6541815d960 100644 --- a/src/node/receipt.cpp +++ b/src/node/receipt.cpp @@ -154,7 +154,7 @@ namespace ccf schema["required"].push_back(name); schema["properties"] = nlohmann::json::object(); schema["properties"][name] = ds::openapi::components_ref_object( - ds::json::schema_name()); + ds::json::schema_name()); return schema; }; diff --git a/src/node/retired_nodes_cleanup.h b/src/node/retired_nodes_cleanup.h index 74ef487a2e47..dd2de0de0792 100644 --- a/src/node/retired_nodes_cleanup.h +++ b/src/node/retired_nodes_cleanup.h @@ -19,7 +19,7 @@ namespace ccf void send_cleanup_retired_nodes() { - http::Request request( + ::http::Request request( fmt::format( "/{}/{}", ccf::get_actor_prefix(ccf::ActorsType::nodes), diff --git a/src/node/rpc/acme_subsystem.h b/src/node/rpc/acme_subsystem.h index dabb59aae20e..001128a405ad 100644 --- a/src/node/rpc/acme_subsystem.h +++ b/src/node/rpc/acme_subsystem.h @@ -45,7 +45,7 @@ namespace ccf } } - virtual crypto::Pem network_cert() override + virtual ccf::crypto::Pem network_cert() override { return node_state.get_network_cert(); } @@ -60,15 +60,15 @@ namespace ccf const std::vector& body, std::function&)> callback, const std::vector& ca_certs = {}, const std::string& app_protocol = "HTTP1", bool authenticate_as_node_client_certificate = false) override { llhttp_method_t m = http_method_from_str(method.c_str()); - http::URL urlobj = http::parse_url_full(url); - http::Request r(urlobj.path, m); + ::http::URL urlobj = ::http::parse_url_full(url); + ::http::Request r(urlobj.path, m); if (!body.empty()) { r.set_body(&body); diff --git a/src/node/rpc/call_types.h b/src/node/rpc/call_types.h index 9b0e5b8e9f6d..898c95d52c70 100644 --- a/src/node/rpc/call_types.h +++ b/src/node/rpc/call_types.h @@ -70,7 +70,7 @@ namespace ccf struct Out { ServiceStatus service_status; - crypto::Pem service_certificate; + ccf::crypto::Pem service_certificate; std::optional current_view; std::optional primary_id; size_t recovery_count; diff --git a/src/node/rpc/claims.h b/src/node/rpc/claims.h index 49a8cb450c57..fe571cb77b75 100644 --- a/src/node/rpc/claims.h +++ b/src/node/rpc/claims.h @@ -12,23 +12,23 @@ namespace ccf return ClaimsDigest(); } - static crypto::Sha256Hash entry_leaf( + static ccf::crypto::Sha256Hash entry_leaf( const std::vector& write_set, - const std::optional& commit_evidence_digest, + const std::optional& commit_evidence_digest, const ClaimsDigest& claims_digest) { - crypto::Sha256Hash write_set_digest(write_set); + ccf::crypto::Sha256Hash write_set_digest(write_set); if (commit_evidence_digest.has_value()) { if (claims_digest.empty()) { - return crypto::Sha256Hash( + return ccf::crypto::Sha256Hash( write_set_digest, commit_evidence_digest.value()); } else { - return crypto::Sha256Hash( + return ccf::crypto::Sha256Hash( write_set_digest, commit_evidence_digest.value(), claims_digest.value()); @@ -38,11 +38,11 @@ namespace ccf { if (claims_digest.empty()) { - return crypto::Sha256Hash(write_set_digest); + return ccf::crypto::Sha256Hash(write_set_digest); } else { - return crypto::Sha256Hash(write_set_digest, claims_digest.value()); + return ccf::crypto::Sha256Hash(write_set_digest, claims_digest.value()); } } } diff --git a/src/node/rpc/custom_protocol_subsystem.h b/src/node/rpc/custom_protocol_subsystem.h index 1e1d3b373f75..a2a4c928b466 100644 --- a/src/node/rpc/custom_protocol_subsystem.h +++ b/src/node/rpc/custom_protocol_subsystem.h @@ -60,7 +60,7 @@ namespace ccf std::shared_ptr r = std::make_shared(); r->writer = node_state.get_writer_factory().create_writer_to_outside(); auto store = node_state.get_store(); - r->tx = std::make_shared(store.get()); + r->tx = std::make_shared(store.get()); r->ctx = std::make_shared( nullptr, *r->tx); return r; diff --git a/src/node/rpc/forwarder.h b/src/node/rpc/forwarder.h index 74685a9970a9..8e8040afe446 100644 --- a/src/node/rpc/forwarder.h +++ b/src/node/rpc/forwarder.h @@ -94,7 +94,7 @@ namespace ccf auto rpc_responder_shared = rpcresponder.lock(); if (rpc_responder_shared) { - auto response = http::Response(HTTP_STATUS_GATEWAY_TIMEOUT); + auto response = ::http::Response(HTTP_STATUS_GATEWAY_TIMEOUT); auto body = fmt::format( "Request was forwarded to node {}, but no response was received " "after {}ms", @@ -190,7 +190,7 @@ namespace ccf } template - std::shared_ptr recv_forwarded_command( + std::shared_ptr<::http::HttpRpcContext> recv_forwarded_command( const NodeId& from, const uint8_t* data, size_t size) { std::pair> r; @@ -237,7 +237,7 @@ namespace ccf return ccf::make_fwd_rpc_context( session, raw_request, r.first.frame_format); } - catch (const http::RequestTooLargeException& rexc) + catch (const ::http::RequestTooLargeException& rexc) { LOG_FAIL_FMT("Forwarded request exceeded limit: {}", rexc.what()); return nullptr; @@ -312,7 +312,7 @@ namespace ccf } std::shared_ptr get_forwarder_handler( - std::shared_ptr& ctx) + std::shared_ptr<::http::HttpRpcContext>& ctx) { if (ctx == nullptr) { @@ -328,7 +328,7 @@ namespace ccf } std::shared_ptr search = - http::fetch_rpc_handler(ctx, rpc_map_shared); + ::http::fetch_rpc_handler(ctx, rpc_map_shared); auto fwd_handler = std::dynamic_pointer_cast(search); if (!fwd_handler) diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index 7368a4f80021..a5a9318da33a 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -35,17 +35,17 @@ namespace ccf class RpcFrontend : public RpcHandler, public ForwardedRpcHandler { protected: - kv::Store& tables; + ccf::kv::Store& tables; endpoints::EndpointRegistry& endpoints; - ccfapp::AbstractNodeContext& node_context; + ccf::AbstractNodeContext& node_context; private: ccf::pal::Mutex open_lock; bool is_open_ = false; - kv::Consensus* consensus; + ccf::kv::Consensus* consensus; std::shared_ptr cmd_forwarder; - kv::TxHistory* history; + ccf::kv::TxHistory* history; size_t sig_tx_interval = 5000; std::chrono::milliseconds sig_ms_interval = std::chrono::milliseconds(1000); @@ -72,7 +72,7 @@ namespace ccf } endpoints::EndpointDefinitionPtr find_endpoint( - std::shared_ptr ctx, kv::CommittableTx& tx) + std::shared_ptr ctx, ccf::kv::CommittableTx& tx) { const auto endpoint = endpoints.find_endpoint(tx, *ctx); if (endpoint == nullptr) @@ -186,7 +186,7 @@ namespace ccf std::optional resolve_redirect_location( const RedirectionResolverConfig& resolver, - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const ccf::ListenInterfaceID& incoming_interface) { switch (resolver.kind) @@ -194,9 +194,11 @@ namespace ccf case (RedirectionResolutionKind::NodeByRole): { const auto role_it = resolver.target.find("role"); - const bool primary = + const bool seeking_primary = role_it == resolver.target.end() || role_it.value() == "primary"; - if (!primary) + const bool seeking_backup = + !seeking_primary && role_it.value() == "backup"; + if (!seeking_primary && !seeking_backup) { return std::nullopt; } @@ -207,14 +209,32 @@ namespace ccf incoming_interface : interface_it.value().get(); - const auto primary_id = consensus->primary(); - if (!primary_id.has_value()) + std::vector::const_iterator> + target_node_its; + const auto nodes = InternalTablesAccess::get_trusted_nodes(tx); + { + const auto primary_id = consensus->primary(); + if (seeking_primary && primary_id.has_value()) + { + target_node_its.push_back(nodes.find(primary_id.value())); + } + else if (seeking_backup) + { + for (auto it = nodes.begin(); it != nodes.end(); ++it) + { + if (it->first != primary_id) + { + target_node_its.push_back(it); + } + } + } + } + if (target_node_its.empty()) { return std::nullopt; } - const auto nodes = InternalTablesAccess::get_trusted_nodes(tx); - const auto node_it = nodes.find(primary_id.value()); + const auto node_it = target_node_its[rand() % target_node_its.size()]; if (node_it != nodes.end()) { const auto& interfaces = node_it->second.rpc_interfaces; @@ -242,26 +262,8 @@ namespace ccf return std::nullopt; } - RedirectionResolverConfig get_redirect_resolver_config( - endpoints::RedirectionStrategy strategy, - const ccf::NodeInfoNetwork_v2::NetInterface::Redirections& redirections) - { - switch (strategy) - { - case (ccf::endpoints::RedirectionStrategy::None): - { - return {}; - } - - case (ccf::endpoints::RedirectionStrategy::ToPrimary): - { - return redirections.to_primary; - } - } - } - bool check_redirect( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, std::shared_ptr ctx, const endpoints::EndpointDefinitionPtr& endpoint, const ccf::NodeInfoNetwork_v2::NetInterface::Redirections& redirections) @@ -282,7 +284,7 @@ namespace ccf if (!is_primary) { - auto resolver = get_redirect_resolver_config(rs, redirections); + auto resolver = redirections.to_primary; const auto listen_interface = ctx->get_session_context()->interface_id.value_or( @@ -310,6 +312,41 @@ namespace ccf return false; } + case (ccf::endpoints::RedirectionStrategy::ToBackup): + { + const bool is_backup = + (consensus != nullptr) && !consensus->can_replicate(); + + if (!is_backup) + { + auto resolver = redirections.to_backup; + + const auto listen_interface = + ctx->get_session_context()->interface_id.value_or( + PRIMARY_RPC_INTERFACE); + const auto location = + resolve_redirect_location(resolver, tx, listen_interface); + if (location.has_value()) + { + ctx->set_response_header( + http::headers::LOCATION, + fmt::format( + "https://{}{}", location.value(), ctx->get_request_url())); + ctx->set_response_status(HTTP_STATUS_TEMPORARY_REDIRECT); + return true; + } + + // Should have redirected, but don't know how to. Return an error + ctx->set_error( + HTTP_STATUS_SERVICE_UNAVAILABLE, + ccf::errors::BackupNotFound, + "Request should be redirected to backup, but receiving node " + "does not know any current backup address"); + return true; + } + return false; + } + default: { LOG_FAIL_FMT("Unhandled redirection strategy: {}", rs); @@ -381,7 +418,7 @@ namespace ccf std::unique_ptr get_authenticated_identity( std::shared_ptr ctx, - kv::CommittableTx& tx, + ccf::kv::CommittableTx& tx, const endpoints::EndpointDefinitionPtr& endpoint) { if (endpoint->authn_policies.empty()) @@ -456,7 +493,7 @@ namespace ccf void forward( std::shared_ptr ctx, - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const endpoints::EndpointDefinitionPtr& endpoint) { // HTTP/2 does not support forwarding @@ -595,7 +632,7 @@ namespace ccf } } - std::unique_ptr tx_p = tables.create_tx_ptr(); + std::unique_ptr tx_p = tables.create_tx_ptr(); set_root_on_proposals(*ctx, *tx_p); if (attempts > 0) @@ -734,12 +771,12 @@ namespace ccf } // else args owns a valid Tx relating to a non-pending response, which // should be applied - kv::CommittableTx& tx = *args.owned_tx; - kv::CommitResult result = tx.commit(ctx->claims, false); + ccf::kv::CommittableTx& tx = *args.owned_tx; + ccf::kv::CommitResult result = tx.commit(ctx->claims, false); switch (result) { - case kv::CommitResult::SUCCESS: + case ccf::kv::CommitResult::SUCCESS: { auto tx_id = tx.get_txid(); if (tx_id.has_value() && consensus != nullptr) @@ -789,12 +826,12 @@ namespace ccf return; } - case kv::CommitResult::FAIL_CONFLICT: + case ccf::kv::CommitResult::FAIL_CONFLICT: { break; } - case kv::CommitResult::FAIL_NO_REPLICATE: + case ccf::kv::CommitResult::FAIL_NO_REPLICATE: { ctx->clear_response_headers(); ctx->set_error( @@ -806,7 +843,7 @@ namespace ccf } } } - catch (const kv::CompactedVersionConflict& e) + catch (const ccf::kv::CompactedVersionConflict& e) { // The executing transaction failed because of a conflicting // compaction. Reset and retry @@ -837,7 +874,7 @@ namespace ccf return; } - catch (const kv::KvSerialiserException& e) + catch (const ccf::kv::KvSerialiserException& e) { // If serialising the committed transaction fails, there is no way // to recover safely (https://github.com/microsoft/CCF/issues/338). @@ -875,9 +912,9 @@ namespace ccf public: RpcFrontend( - kv::Store& tables_, + ccf::kv::Store& tables_, endpoints::EndpointRegistry& handlers_, - ccfapp::AbstractNodeContext& node_context_) : + ccf::AbstractNodeContext& node_context_) : tables(tables_), endpoints(handlers_), node_context(node_context_), @@ -913,7 +950,7 @@ namespace ccf } void set_root_on_proposals( - const ccf::RpcContextImpl& ctx, kv::CommittableTx& tx) + const ccf::RpcContextImpl& ctx, ccf::kv::CommittableTx& tx) { if (endpoints.request_needs_root(ctx)) { diff --git a/src/node/rpc/gov_effects.h b/src/node/rpc/gov_effects.h index 906189f091d9..5141559d234d 100644 --- a/src/node/rpc/gov_effects.h +++ b/src/node/rpc/gov_effects.h @@ -16,33 +16,33 @@ namespace ccf GovernanceEffects(AbstractNodeState& impl_) : impl(impl_) {} void transition_service_to_open( - kv::Tx& tx, ServiceIdentities identities) override + ccf::kv::Tx& tx, ServiceIdentities identities) override { impl.transition_service_to_open(tx, identities); } - bool rekey_ledger(kv::Tx& tx) override + bool rekey_ledger(ccf::kv::Tx& tx) override { return impl.rekey_ledger(tx); } - void trigger_recovery_shares_refresh(kv::Tx& tx) override + void trigger_recovery_shares_refresh(ccf::kv::Tx& tx) override { impl.trigger_recovery_shares_refresh(tx); } - void trigger_ledger_chunk(kv::Tx& tx) override + void trigger_ledger_chunk(ccf::kv::Tx& tx) override { impl.trigger_ledger_chunk(tx); } - void trigger_snapshot(kv::Tx& tx) override + void trigger_snapshot(ccf::kv::Tx& tx) override { impl.trigger_snapshot(tx); } void trigger_acme_refresh( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::optional>& interfaces = std::nullopt) override { diff --git a/src/node/rpc/gov_effects_interface.h b/src/node/rpc/gov_effects_interface.h index b755538ef0e2..a1e460a20ad3 100644 --- a/src/node/rpc/gov_effects_interface.h +++ b/src/node/rpc/gov_effects_interface.h @@ -20,18 +20,18 @@ namespace ccf struct ServiceIdentities { - std::optional previous; - crypto::Pem next; + std::optional previous; + ccf::crypto::Pem next; }; virtual void transition_service_to_open( - kv::Tx& tx, ServiceIdentities identities) = 0; - virtual bool rekey_ledger(kv::Tx& tx) = 0; - virtual void trigger_recovery_shares_refresh(kv::Tx& tx) = 0; - virtual void trigger_ledger_chunk(kv::Tx& tx) = 0; - virtual void trigger_snapshot(kv::Tx& tx) = 0; + ccf::kv::Tx& tx, ServiceIdentities identities) = 0; + virtual bool rekey_ledger(ccf::kv::Tx& tx) = 0; + virtual void trigger_recovery_shares_refresh(ccf::kv::Tx& tx) = 0; + virtual void trigger_ledger_chunk(ccf::kv::Tx& tx) = 0; + virtual void trigger_snapshot(ccf::kv::Tx& tx) = 0; virtual void trigger_acme_refresh( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::optional>& interfaces = std::nullopt) = 0; }; diff --git a/src/node/rpc/jwt_management.h b/src/node/rpc/jwt_management.h index 1e630381d53f..1af1cb4c8b52 100644 --- a/src/node/rpc/jwt_management.h +++ b/src/node/rpc/jwt_management.h @@ -24,7 +24,7 @@ namespace ccf { static void legacy_remove_jwt_public_signing_keys( - kv::Tx& tx, std::string issuer) + ccf::kv::Tx& tx, std::string issuer) { auto keys = tx.rw(Tables::Legacy::JWT_PUBLIC_SIGNING_KEYS); @@ -49,8 +49,8 @@ namespace ccf // setting keys from issuer A which will be used to validate iss claims for // issuer B, so this doesn't make sense (at least for now). - const auto issuer_domain = http::parse_url_full(issuer).host; - const auto constraint_domain = http::parse_url_full(constraint).host; + const auto issuer_domain = ::http::parse_url_full(issuer).host; + const auto constraint_domain = ::http::parse_url_full(constraint).host; if (constraint_domain.empty()) { @@ -73,7 +73,8 @@ namespace ccf return true; } - static void remove_jwt_public_signing_keys(kv::Tx& tx, std::string issuer) + static void remove_jwt_public_signing_keys( + ccf::kv::Tx& tx, std::string issuer) { // Unlike resetting JWT keys for a particular issuer, removing keys can be // safely done on both table revisions, as soon as the application shouldn't @@ -123,7 +124,7 @@ namespace ccf #endif static bool set_jwt_public_signing_keys( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::string& log_prefix, std::string issuer, const JwtIssuerMetadata& issuer_metadata, @@ -158,7 +159,7 @@ namespace ccf auto const& kid = jwk.kid.value(); try { - der = crypto::raw_from_b64(der_base64); + der = ccf::crypto::raw_from_b64(der_base64); } catch (const std::invalid_argument& e) { @@ -234,7 +235,7 @@ namespace ccf { try { - crypto::make_unique_verifier( + ccf::crypto::make_unique_verifier( (std::vector)der); // throws on error } catch (std::invalid_argument& exc) diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 344ec13df8ec..6d6262b8ae0b 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -70,15 +70,15 @@ namespace ccf struct KeyIdInfo { JwtIssuer issuer; - crypto::Pem cert; + ccf::crypto::Pem cert; }; DECLARE_JSON_TYPE(KeyIdInfo) DECLARE_JSON_REQUIRED_FIELDS(KeyIdInfo, issuer, cert) struct FullMemberDetails : public ccf::MemberDetails { - crypto::Pem cert; - std::optional public_encryption_key; + ccf::crypto::Pem cert; + std::optional public_encryption_key; }; DECLARE_JSON_TYPE(FullMemberDetails); DECLARE_JSON_REQUIRED_FIELDS( @@ -113,7 +113,7 @@ namespace ccf } void remove_all_other_non_open_proposals( - kv::Tx& tx, const ProposalId& proposal_id) + ccf::kv::Tx& tx, const ProposalId& proposal_id) { auto p = tx.rw(jsgov::Tables::PROPOSALS); auto pi = @@ -136,7 +136,7 @@ namespace ccf } ccf::jsgov::ProposalInfoSummary resolve_proposal( - kv::Tx& tx, + ccf::kv::Tx& tx, const ProposalId& proposal_id, const std::span& proposal_bytes, const std::string& constitution) @@ -359,13 +359,13 @@ namespace ccf } } - bool check_member_active(kv::ReadOnlyTx& tx, const MemberId& id) + bool check_member_active(ccf::kv::ReadOnlyTx& tx, const MemberId& id) { return check_member_status(tx, id, {MemberStatus::ACTIVE}); } bool check_member_status( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const MemberId& id, std::initializer_list allowed) { @@ -385,14 +385,16 @@ namespace ccf } void record_voting_history( - kv::Tx& tx, const MemberId& caller_id, const SignedReq& signed_request) + ccf::kv::Tx& tx, + const MemberId& caller_id, + const SignedReq& signed_request) { auto governance_history = tx.rw(network.governance_history); governance_history->put(caller_id, {signed_request}); } void record_cose_governance_history( - kv::Tx& tx, + ccf::kv::Tx& tx, const MemberId& caller_id, const std::span& cose_sign1) { @@ -402,7 +404,7 @@ namespace ccf } ProposalSubmissionStatus is_proposal_submission_acceptable( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::string& created_at, const std::vector& request_digest, const ccf::ProposalId& proposal_id, @@ -425,8 +427,8 @@ namespace ccf if (!replay_keys.empty()) { min_created_at = std::get<0>( - nonstd::split_1(replay_keys[replay_keys.size() / 2], ":")); - auto [key_ts, __] = nonstd::split_1(key, ":"); + ccf::nonstd::split_1(replay_keys[replay_keys.size() / 2], ":")); + auto [key_ts, __] = ccf::nonstd::split_1(key, ":"); if (key_ts < min_created_at) { return ProposalSubmissionStatus::TooOld; @@ -481,13 +483,15 @@ namespace ccf template void add_kv_wrapper_endpoint(T table) { - constexpr bool is_map = nonstd::is_specialization::value; + constexpr bool is_map = + ccf::nonstd::is_specialization::value; constexpr bool is_value = - nonstd::is_specialization::value; + ccf::nonstd::is_specialization::value; if constexpr (!(is_map || is_value)) { - static_assert(nonstd::dependent_false_v, "Unsupported table type"); + static_assert( + ccf::nonstd::dependent_false_v, "Unsupported table type"); } auto getter = @@ -500,7 +504,7 @@ namespace ccf { handle->foreach([&response_body](const auto& k, const auto& v) { if constexpr ( - std::is_same_v || + std::is_same_v || pal::is_attestation_measurement::value) { response_body[k.hex_str()] = v; @@ -573,7 +577,7 @@ namespace ccf void add_kv_wrapper_endpoints() { const auto all_gov_tables = network.get_all_builtin_governance_tables(); - nonstd::tuple_for_each( + ccf::nonstd::tuple_for_each( all_gov_tables, [this](auto table) { add_kv_wrapper_endpoint(table); }); } @@ -582,7 +586,7 @@ namespace ccf public: MemberEndpoints( - NetworkState& network_, ccfapp::AbstractNodeContext& context_) : + NetworkState& network_, ccf::AbstractNodeContext& context_) : GovEndpointRegistry(network_, context_), network(network_), share_manager(network_.ledger_secrets) @@ -815,7 +819,8 @@ namespace ccf j["state_digest"] = ma->state_digest; ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(j.dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -868,9 +873,10 @@ namespace ccf } auto rec_share = GetRecoveryShare::Out{ - crypto::b64_from_raw(encrypted_share.value())}; + ccf::crypto::b64_from_raw(encrypted_share.value())}; ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(rec_share).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -918,9 +924,10 @@ namespace ccf } auto rec_share = GetRecoveryShare::Out{ - crypto::b64_from_raw(encrypted_share.value())}; + ccf::crypto::b64_from_raw(encrypted_share.value())}; ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(rec_share).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -993,7 +1000,7 @@ namespace ccf } std::string share = params["share"]; - auto raw_recovery_share = crypto::raw_from_b64(share); + auto raw_recovery_share = ccf::crypto::raw_from_b64(share); OPENSSL_cleanse(const_cast(share.data()), share.size()); size_t submitted_shares_count = 0; @@ -1027,7 +1034,8 @@ namespace ccf submitted_shares_count, InternalTablesAccess::get_recovery_threshold(ctx.tx))}; ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(recovery_share).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -1064,7 +1072,8 @@ namespace ccf submitted_shares_count, InternalTablesAccess::get_recovery_threshold(ctx.tx))}; ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(recovery_share).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); }; @@ -1089,7 +1098,7 @@ namespace ccf for (const auto& metadata : v) { info.push_back(KeyIdInfo{ - metadata.issuer, crypto::cert_der_to_pem(metadata.cert)}); + metadata.issuer, ccf::crypto::cert_der_to_pem(metadata.cert)}); } kmap.emplace(k, std::move(info)); return true; @@ -1130,7 +1139,7 @@ namespace ccf if (cose_auth_id.has_value()) { std::span sig = cose_auth_id->signature; - request_digest = crypto::sha256(sig); + request_digest = ccf::crypto::sha256(sig); } ProposalId proposal_id; @@ -1152,7 +1161,7 @@ namespace ccf std::vector acc( root_at_read.value().h.begin(), root_at_read.value().h.end()); acc.insert(acc.end(), request_digest.begin(), request_digest.end()); - const crypto::Sha256Hash proposal_digest(acc); + const ccf::crypto::Sha256Hash proposal_digest(acc); proposal_id = proposal_digest.hex_str(); auto constitution = ctx.tx.ro(network.constitution)->get(); @@ -1341,7 +1350,8 @@ namespace ccf proposal_id, {member_id.value(), rv.state, {}, {}, std::nullopt, rv.failure}); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(rv).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -1530,7 +1540,8 @@ namespace ccf } ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(pi_.value()).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); }; @@ -1575,7 +1586,8 @@ namespace ccf ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(std::move(p.value())); }; @@ -1734,7 +1746,8 @@ namespace ccf pi_.value().failure = rv.failure; pi->put(proposal_id, pi_.value()); ctx.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON); + ccf::http::headers::CONTENT_TYPE, + http::headervalues::contenttype::JSON); ctx.rpc_ctx->set_response_body(nlohmann::json(rv).dump()); ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); return; @@ -1873,7 +1886,7 @@ namespace ccf public: MemberRpcFrontend( - NetworkState& network, ccfapp::AbstractNodeContext& context) : + NetworkState& network, ccf::AbstractNodeContext& context) : RpcFrontend(*network.tables, member_endpoints, context), member_endpoints(network, context) {} diff --git a/src/node/rpc/no_create_tx_claims_digest.cpp b/src/node/rpc/no_create_tx_claims_digest.cpp index aff03bc8c647..9484b50cecc4 100644 --- a/src/node/rpc/no_create_tx_claims_digest.cpp +++ b/src/node/rpc/no_create_tx_claims_digest.cpp @@ -3,10 +3,10 @@ #include -namespace ccfapp +namespace ccf { std::optional __attribute__((weak)) - get_create_tx_claims_digest(kv::ReadOnlyTx& tx) + get_create_tx_claims_digest(ccf::kv::ReadOnlyTx& tx) { return std::nullopt; } diff --git a/src/node/rpc/node_call_types.h b/src/node/rpc/node_call_types.h index e38862ce7819..5e70089fcc4f 100644 --- a/src/node/rpc/node_call_types.h +++ b/src/node/rpc/node_call_types.h @@ -28,12 +28,12 @@ namespace ccf { ccf::NodeId node_id; ccf::NodeStartupState state; - kv::Version last_signed_seqno; - kv::Version startup_seqno; + ccf::kv::Version last_signed_seqno; + ccf::kv::Version startup_seqno; // Only on recovery - std::optional recovery_target_seqno; - std::optional last_recovered_seqno; + std::optional recovery_target_seqno; + std::optional last_recovered_seqno; bool stop_notice; }; @@ -56,12 +56,12 @@ namespace ccf struct In { NodeId node_id; - crypto::Pem certificate_signing_request; - crypto::Pem node_endorsed_certificate; - crypto::Pem public_key; - crypto::Pem service_cert; + ccf::crypto::Pem certificate_signing_request; + ccf::crypto::Pem node_endorsed_certificate; + ccf::crypto::Pem public_key; + ccf::crypto::Pem service_cert; QuoteInfo quote_info; - crypto::Pem public_encryption_key; + ccf::crypto::Pem public_encryption_key; pal::PlatformAttestationMeasurement measurement; std::optional snp_security_policy = std::nullopt; // base64-encoded @@ -82,12 +82,13 @@ namespace ccf { NodeInfoNetwork node_info_network; QuoteInfo quote_info; - crypto::Pem public_encryption_key; + ccf::crypto::Pem public_encryption_key; // Always set by the joiner (node_state.h), but defaults to nullopt here // to make sure serialisation does take place now that it is OPTIONAL. std::optional consensus_type = std::nullopt; - std::optional startup_seqno = std::nullopt; - std::optional certificate_signing_request = std::nullopt; + std::optional startup_seqno = std::nullopt; + std::optional certificate_signing_request = + std::nullopt; nlohmann::json node_data = nullptr; }; @@ -101,7 +102,7 @@ namespace ccf struct NetworkInfo { bool public_only = false; - kv::Version last_recovered_signed_idx = kv::NoVersion; + ccf::kv::Version last_recovered_signed_idx = ccf::kv::NoVersion; ConsensusType consensus_type = ConsensusType::CFT; std::optional reconfiguration_type = std::nullopt; // Unused, but kept for backwards compatibility @@ -110,18 +111,18 @@ namespace ccf NetworkIdentity identity; std::optional service_status = std::nullopt; - std::optional endorsed_certificate = std::nullopt; + std::optional endorsed_certificate = std::nullopt; NetworkInfo() {} NetworkInfo( bool public_only, - kv::Version last_recovered_signed_idx, + ccf::kv::Version last_recovered_signed_idx, ReconfigurationType reconfiguration_type, const LedgerSecretsMap& ledger_secrets, const NetworkIdentity& identity, ServiceStatus service_status, - const std::optional& endorsed_certificate) : + const std::optional& endorsed_certificate) : public_only(public_only), last_recovered_signed_idx(last_recovered_signed_idx), reconfiguration_type(reconfiguration_type), diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 6ef867a53add..03b9e4f7037f 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -117,7 +117,7 @@ namespace ccf struct ConsensusConfigDetails { - kv::ConsensusDetails details; + ccf::kv::ConsensusDetails details; }; DECLARE_JSON_TYPE(ConsensusConfigDetails); @@ -125,7 +125,7 @@ namespace ccf struct SelfSignedNodeCertificateInfo { - crypto::Pem self_signed_certificate; + ccf::crypto::Pem self_signed_certificate; }; DECLARE_JSON_TYPE(SelfSignedNodeCertificateInfo); @@ -136,7 +136,7 @@ namespace ccf { struct Out { - crypto::Pem previous_service_identity; + ccf::crypto::Pem previous_service_identity; }; }; @@ -194,12 +194,12 @@ namespace ccf struct ExistingNodeInfo { NodeId node_id; - std::optional ledger_secret_seqno = std::nullopt; - std::optional endorsed_certificate = std::nullopt; + std::optional ledger_secret_seqno = std::nullopt; + std::optional endorsed_certificate = std::nullopt; }; std::optional check_node_exists( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::vector& self_signed_node_der, std::optional node_status = std::nullopt) { @@ -211,7 +211,7 @@ namespace ccf LOG_DEBUG_FMT( "Check node exists with certificate [{}]", self_signed_node_der); - auto pk_pem = crypto::public_key_pem_from_cert(self_signed_node_der); + auto pk_pem = ccf::crypto::public_key_pem_from_cert(self_signed_node_der); std::optional existing_node_info = std::nullopt; nodes->foreach([&existing_node_info, @@ -234,7 +234,7 @@ namespace ccf } std::optional check_conflicting_node_network( - kv::Tx& tx, const NodeInfoNetwork& node_info_network) + ccf::kv::Tx& tx, const NodeInfoNetwork& node_info_network) { auto nodes = tx.rw(network.nodes); @@ -261,7 +261,7 @@ namespace ccf } auto add_node( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::vector& node_der, const JoinNetworkNodeToNode::In& in, NodeStatus node_status, @@ -286,7 +286,7 @@ namespace ccf conflicting_node_id.value())); } - auto pubk_der = crypto::public_key_der_from_cert(node_der); + auto pubk_der = ccf::crypto::public_key_der_from_cert(node_der); NodeId joining_node_id = compute_node_id_from_pubk_der(pubk_der); pal::PlatformAttestationMeasurement measurement; @@ -299,7 +299,7 @@ namespace ccf return make_error(code, ccf::errors::InvalidQuote, message); } - std::optional ledger_secret_seqno = std::nullopt; + std::optional ledger_secret_seqno = std::nullopt; if (node_status == NodeStatus::TRUSTED) { ledger_secret_seqno = @@ -307,11 +307,12 @@ namespace ccf } // Note: All new nodes should specify a CSR from 2.x - auto client_public_key_pem = crypto::public_key_pem_from_cert(node_der); + auto client_public_key_pem = + ccf::crypto::public_key_pem_from_cert(node_der); if (in.certificate_signing_request.has_value()) { // Verify that client's public key matches the one specified in the CSR - auto csr_public_key_pem = crypto::public_key_pem_from_csr( + auto csr_public_key_pem = ccf::crypto::public_key_pem_from_csr( in.certificate_signing_request.value()); if (client_public_key_pem != csr_public_key_pem) { @@ -344,14 +345,14 @@ namespace ccf if (node_status == NodeStatus::TRUSTED) { // Joining node only submit a CSR from 2.x - std::optional endorsed_certificate = std::nullopt; + std::optional endorsed_certificate = std::nullopt; if (in.certificate_signing_request.has_value()) { // For a pre-open service, extract the validity period of self-signed // node certificate and use it verbatim in endorsed certificate auto [valid_from, valid_to] = - crypto::make_verifier(node_der)->validity_period(); - endorsed_certificate = crypto::create_endorsed_cert( + ccf::crypto::make_verifier(node_der)->validity_period(); + endorsed_certificate = ccf::crypto::create_endorsed_cert( in.certificate_signing_request.value(), valid_from, valid_to, @@ -394,8 +395,7 @@ namespace ccf } public: - NodeEndpoints( - NetworkState& network_, ccfapp::AbstractNodeContext& context_) : + NodeEndpoints(NetworkState& network_, ccf::AbstractNodeContext& context_) : CommonEndpointRegistry(get_actor_prefix(ActorsType::nodes), context_), network(network_), node_operation(*context_.get_subsystem()) @@ -1538,12 +1538,13 @@ namespace ccf else { // On recovery, force a new ledger chunk - auto tx_ = static_cast(&ctx.tx); + auto tx_ = static_cast(&ctx.tx); if (tx_ == nullptr) { throw std::logic_error("Could not cast tx to CommittableTx"); } - tx_->set_flag(kv::CommittableTx::Flag::LEDGER_CHUNK_BEFORE_THIS_TX); + tx_->set_flag( + ccf::kv::CommittableTx::Flag::LEDGER_CHUNK_BEFORE_THIS_TX); } auto endorsed_certificates = @@ -1581,7 +1582,7 @@ namespace ccf } std::optional digest = - ccfapp::get_create_tx_claims_digest(ctx.tx); + ccf::get_create_tx_claims_digest(ctx.tx); if (digest.has_value()) { auto digest_value = digest.value(); @@ -1798,8 +1799,7 @@ namespace ccf NodeEndpoints node_endpoints; public: - NodeRpcFrontend( - NetworkState& network, ccfapp::AbstractNodeContext& context) : + NodeRpcFrontend(NetworkState& network, ccf::AbstractNodeContext& context) : RpcFrontend(*network.tables, node_endpoints, context), node_endpoints(network, context) {} diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index 12ae666fac84..6acddb4026ef 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -24,16 +24,17 @@ namespace ccf virtual ~AbstractNodeState() {} virtual void transition_service_to_open( - kv::Tx& tx, AbstractGovernanceEffects::ServiceIdentities identities) = 0; - virtual bool rekey_ledger(kv::Tx& tx) = 0; - virtual void trigger_recovery_shares_refresh(kv::Tx& tx) = 0; - virtual void trigger_ledger_chunk(kv::Tx& tx) = 0; - virtual void trigger_snapshot(kv::Tx& tx) = 0; + ccf::kv::Tx& tx, + AbstractGovernanceEffects::ServiceIdentities identities) = 0; + virtual bool rekey_ledger(ccf::kv::Tx& tx) = 0; + virtual void trigger_recovery_shares_refresh(ccf::kv::Tx& tx) = 0; + virtual void trigger_ledger_chunk(ccf::kv::Tx& tx) = 0; + virtual void trigger_snapshot(ccf::kv::Tx& tx) = 0; virtual void trigger_host_process_launch( const std::vector& args, const std::vector& input) = 0; virtual void trigger_acme_refresh( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::optional>& interfaces = std::nullopt) = 0; virtual void install_custom_acme_challenge_handler( @@ -46,20 +47,20 @@ namespace ccf virtual bool is_reading_public_ledger() const = 0; virtual bool is_reading_private_ledger() const = 0; virtual bool is_part_of_network() const = 0; - virtual kv::Version get_last_recovered_signed_idx() = 0; - virtual void initiate_private_recovery(kv::Tx& tx) = 0; + virtual ccf::kv::Version get_last_recovered_signed_idx() = 0; + virtual void initiate_private_recovery(ccf::kv::Tx& tx) = 0; virtual ExtendedState state() = 0; virtual QuoteVerificationResult verify_quote( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) = 0; - virtual kv::Version get_startup_snapshot_seqno() = 0; + virtual ccf::kv::Version get_startup_snapshot_seqno() = 0; virtual SessionMetrics get_session_metrics() = 0; virtual size_t get_jwt_attempts() = 0; - virtual crypto::Pem get_self_signed_certificate() = 0; + virtual ccf::crypto::Pem get_self_signed_certificate() = 0; virtual const StartupConfig& get_node_config() const = 0; - virtual crypto::Pem get_network_cert() = 0; + virtual ccf::crypto::Pem get_network_cert() = 0; virtual void stop_notice() = 0; virtual bool has_received_stop_notice() = 0; virtual bool is_member_frontend_open() = 0; @@ -67,16 +68,16 @@ namespace ccf virtual bool is_accessible_to_members() const = 0; virtual void make_http_request( - const http::URL& url, - http::Request&& req, - std::function< - bool(http_status status, http::HeaderMap&&, std::vector&&)> + const ::http::URL& url, + ::http::Request&& req, + std::function&&)> callback, const std::vector& ca_certs = {}, const std::string& app_protocol = "HTTP1", bool use_node_client_certificate = false) = 0; - virtual std::shared_ptr get_store() = 0; + virtual std::shared_ptr get_store() = 0; virtual ringbuffer::AbstractWriterFactory& get_writer_factory() = 0; }; } diff --git a/src/node/rpc/node_operation.h b/src/node/rpc/node_operation.h index 65d6f62e5659..99bd399d80de 100644 --- a/src/node/rpc/node_operation.h +++ b/src/node/rpc/node_operation.h @@ -65,12 +65,12 @@ namespace ccf return impl.can_replicate(); } - kv::Version get_last_recovered_signed_idx() override + ccf::kv::Version get_last_recovered_signed_idx() override { return impl.get_last_recovered_signed_idx(); } - kv::Version get_startup_snapshot_seqno() override + ccf::kv::Version get_startup_snapshot_seqno() override { return impl.get_startup_snapshot_seqno(); } @@ -86,7 +86,7 @@ namespace ccf } QuoteVerificationResult verify_quote( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) override @@ -95,12 +95,12 @@ namespace ccf tx, quote_info, expected_node_public_key_der, measurement); } - void initiate_private_recovery(kv::Tx& tx) override + void initiate_private_recovery(ccf::kv::Tx& tx) override { impl.initiate_private_recovery(tx); } - crypto::Pem get_self_signed_node_certificate() override + ccf::crypto::Pem get_self_signed_node_certificate() override { return impl.get_self_signed_certificate(); } diff --git a/src/node/rpc/node_operation_interface.h b/src/node/rpc/node_operation_interface.h index 4b09c5efb424..77edb89fb87b 100644 --- a/src/node/rpc/node_operation_interface.h +++ b/src/node/rpc/node_operation_interface.h @@ -15,8 +15,8 @@ namespace ccf { using ExtendedState = std::tuple< NodeStartupState, - std::optional /* recovery_target_seqno */, - std::optional /* last_recovered_seqno */>; + std::optional /* recovery_target_seqno */, + std::optional /* last_recovered_seqno */>; class AbstractNodeOperation : public ccf::AbstractNodeSubSystem { @@ -42,20 +42,20 @@ namespace ccf virtual bool can_replicate() = 0; - virtual kv::Version get_last_recovered_signed_idx() = 0; - virtual kv::Version get_startup_snapshot_seqno() = 0; + virtual ccf::kv::Version get_last_recovered_signed_idx() = 0; + virtual ccf::kv::Version get_startup_snapshot_seqno() = 0; virtual SessionMetrics get_session_metrics() = 0; virtual size_t get_jwt_attempts() = 0; virtual QuoteVerificationResult verify_quote( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) = 0; - virtual void initiate_private_recovery(kv::Tx& tx) = 0; + virtual void initiate_private_recovery(ccf::kv::Tx& tx) = 0; - virtual crypto::Pem get_self_signed_node_certificate() = 0; + virtual ccf::crypto::Pem get_self_signed_node_certificate() = 0; }; } \ No newline at end of file diff --git a/src/node/rpc/test/frontend_test.cpp b/src/node/rpc/test/frontend_test.cpp index 42e741dae104..21d5fe93b600 100644 --- a/src/node/rpc/test/frontend_test.cpp +++ b/src/node/rpc/test/frontend_test.cpp @@ -8,7 +8,6 @@ #include "ccf/ds/logger.h" #include "ccf/json_handler.h" #include "ccf/kv/map.h" -#include "ccf/serdes.h" #include "crypto/openssl/hash.h" #include "ds/files.h" #include "enclave/enclave_time.h" @@ -42,7 +41,7 @@ class TSimpleFrontend : public RpcFrontend public: Registry registry; - TSimpleFrontend(kv::Store& tables, ccfapp::AbstractNodeContext& context) : + TSimpleFrontend(ccf::kv::Store& tables, ccf::AbstractNodeContext& context) : RpcFrontend(tables, registry, context), registry(context) {} @@ -55,7 +54,8 @@ class BaseTestFrontend : public SimpleUserRpcFrontend public: ccf::StubNodeContext context; - BaseTestFrontend(kv::Store& tables) : SimpleUserRpcFrontend(tables, context) + BaseTestFrontend(ccf::kv::Store& tables) : + SimpleUserRpcFrontend(tables, context) {} // For testing only, we don't need to specify auth policies everywhere and @@ -73,7 +73,7 @@ class BaseTestFrontend : public SimpleUserRpcFrontend class TestUserFrontend : public BaseTestFrontend { public: - TestUserFrontend(kv::Store& tables) : BaseTestFrontend(tables) + TestUserFrontend(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -101,7 +101,8 @@ class TestUserFrontend : public BaseTestFrontend class TestJsonWrappedEndpointFunction : public BaseTestFrontend { public: - TestJsonWrappedEndpointFunction(kv::Store& tables) : BaseTestFrontend(tables) + TestJsonWrappedEndpointFunction(ccf::kv::Store& tables) : + BaseTestFrontend(tables) { open(); @@ -112,7 +113,7 @@ class TestJsonWrappedEndpointFunction : public BaseTestFrontend auto echo_query_function = [this](auto& ctx, nlohmann::json&&) { const auto parsed_query = - http::parse_query(ctx.rpc_ctx->get_request_query()); + ccf::http::parse_query(ctx.rpc_ctx->get_request_query()); return make_success(std::move(parsed_query)); }; make_endpoint( @@ -150,7 +151,7 @@ class TestJsonWrappedEndpointFunction : public BaseTestFrontend class TestRestrictedVerbsFrontend : public BaseTestFrontend { public: - TestRestrictedVerbsFrontend(kv::Store& tables) : BaseTestFrontend(tables) + TestRestrictedVerbsFrontend(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -175,9 +176,9 @@ class TestRestrictedVerbsFrontend : public BaseTestFrontend class TestExplicitCommitability : public BaseTestFrontend { public: - kv::Map values; + ccf::kv::Map values; - TestExplicitCommitability(kv::Store& tables) : + TestExplicitCommitability(ccf::kv::Store& tables) : BaseTestFrontend(tables), values("test_values") { @@ -185,7 +186,7 @@ class TestExplicitCommitability : public BaseTestFrontend auto maybe_commit = [this](ccf::endpoints::EndpointContext& ctx) { const auto parsed = - serdes::unpack(ctx.rpc_ctx->get_request_body(), default_pack); + nlohmann::json::parse(ctx.rpc_ctx->get_request_body()); const auto new_value = parsed["value"].get(); auto vs = ctx.tx.rw(values); @@ -208,7 +209,7 @@ class TestExplicitCommitability : public BaseTestFrontend class TestAlternativeHandlerTypes : public BaseTestFrontend { public: - TestAlternativeHandlerTypes(kv::Store& tables) : BaseTestFrontend(tables) + TestAlternativeHandlerTypes(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -236,7 +237,7 @@ class TestAlternativeHandlerTypes : public BaseTestFrontend class TestTemplatedPaths : public BaseTestFrontend { public: - TestTemplatedPaths(kv::Store& tables) : BaseTestFrontend(tables) + TestTemplatedPaths(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -252,7 +253,7 @@ class TestTemplatedPaths : public BaseTestFrontend class TestDecodedTemplatedPaths : public BaseTestFrontend { public: - TestDecodedTemplatedPaths(kv::Store& tables) : BaseTestFrontend(tables) + TestDecodedTemplatedPaths(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -292,7 +293,7 @@ class TestNoCertsFrontend : public RpcFrontend ccf::endpoints::EndpointRegistry endpoints; public: - TestNoCertsFrontend(kv::Store& tables) : + TestNoCertsFrontend(ccf::kv::Store& tables) : RpcFrontend(tables, endpoints, context), endpoints("test") { @@ -317,13 +318,13 @@ class RpcContextRecorder { public: // session->caller_cert may be DER or PEM, we always convert to PEM - crypto::Pem last_caller_cert; + ccf::crypto::Pem last_caller_cert; std::optional last_caller_id = std::nullopt; void record_ctx(ccf::endpoints::EndpointContext& ctx) { - last_caller_cert = - crypto::cert_der_to_pem(ctx.rpc_ctx->get_session_context()->caller_cert); + last_caller_cert = ccf::crypto::cert_der_to_pem( + ctx.rpc_ctx->get_session_context()->caller_cert); if (const auto uci = ctx.try_get_caller()) { last_caller_id = uci->user_id; @@ -343,7 +344,7 @@ class TestForwardingUserFrontEnd : public BaseTestFrontend, public RpcContextRecorder { public: - TestForwardingUserFrontEnd(kv::Store& tables) : BaseTestFrontend(tables) + TestForwardingUserFrontEnd(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -412,20 +413,20 @@ class TestForwardingMemberFrontEnd : public MemberRpcFrontend, } }; -auto create_simple_request( - const std::string& method = "/empty_function", - serdes::Pack pack = default_pack) +auto create_simple_request(const std::string& method = "/empty_function") { - http::Request request(method); + ::http::Request request(method); request.set_header( - http::headers::CONTENT_TYPE, ccf::jsonhandler::pack_to_content_type(pack)); + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); return request; } -http::SimpleResponseProcessor::Response parse_response(const vector& v) +::http::SimpleResponseProcessor::Response parse_response( + const vector& v) { - http::SimpleResponseProcessor processor; - http::ResponseParser parser(processor); + ::http::SimpleResponseProcessor processor; + ::http::ResponseParser parser(processor); parser.execute(v.data(), v.size()); REQUIRE(processor.received.size() == 1); @@ -433,24 +434,24 @@ http::SimpleResponseProcessor::Response parse_response(const vector& v) return processor.received.front(); } -nlohmann::json parse_response_body( - const vector& body, serdes::Pack pack = default_pack) +nlohmann::json parse_response_body(const vector& body) { - return serdes::unpack(body, pack); + return nlohmann::json::parse(body); } // callers used throughout auto user_caller = kp -> self_sign("CN=name", valid_from, valid_to); -auto user_caller_der = crypto::make_verifier(user_caller) -> cert_der(); +auto user_caller_der = ccf::crypto::make_verifier(user_caller) -> cert_der(); -auto member_caller_der = crypto::make_verifier(member_cert) -> cert_der(); +auto member_caller_der = ccf::crypto::make_verifier(member_cert) -> cert_der(); auto node_caller = kp -> self_sign("CN=node", valid_from, valid_to); -auto node_caller_der = crypto::make_verifier(node_caller) -> cert_der(); +auto node_caller_der = ccf::crypto::make_verifier(node_caller) -> cert_der(); -auto kp_other = crypto::make_key_pair(); +auto kp_other = ccf::crypto::make_key_pair(); auto invalid_caller = kp_other -> self_sign("CN=name", valid_from, valid_to); -auto invalid_caller_der = crypto::make_verifier(invalid_caller) -> cert_der(); +auto invalid_caller_der = + ccf::crypto::make_verifier(invalid_caller) -> cert_der(); auto anonymous_caller_der = std::vector(); @@ -475,7 +476,8 @@ void prepare_callers(NetworkState& network) // It is necessary to set a consensus before committing the first transaction, // so that the KV batching done before calling into replicate() stays in // order. - auto backup_consensus = std::make_shared(); + auto backup_consensus = + std::make_shared(); network.tables->set_consensus(backup_consensus); auto tx = network.tables->create_tx(); @@ -487,7 +489,7 @@ void prepare_callers(NetworkState& network) user_id = InternalTablesAccess::add_user(tx, {user_caller}); member_id = InternalTablesAccess::add_member(tx, member_cert); invalid_member_id = InternalTablesAccess::add_member(tx, invalid_caller); - CHECK(tx.commit() == kv::CommitResult::SUCCESS); + CHECK(tx.commit() == ccf::kv::CommitResult::SUCCESS); } TEST_CASE("SignedReq to and from json") @@ -656,106 +658,103 @@ TEST_CASE("JsonWrappedEndpointFunction") NetworkState network; prepare_callers(network); TestJsonWrappedEndpointFunction frontend(*network.tables); - for (const auto pack_type : {serdes::Pack::Text, serdes::Pack::MsgPack}) + {{INFO("Calling echo, with params in body"); + auto echo_call = create_simple_request("/echo"); + const nlohmann::json j_body = { + {"data", {"nested", "Some string"}}, {"other", "Another string"}}; + const auto serialized_body = j_body.dump(); + echo_call.set_body(serialized_body); + const auto serialized_call = echo_call.build_request(); + + auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); + frontend.process(rpc_ctx); + auto response = parse_response(rpc_ctx->serialise_response()); + CHECK(response.status == HTTP_STATUS_OK); + + const auto response_body = parse_response_body(response.body); + CHECK(response_body == j_body); +} + +{ + INFO("Calling echo_query, with params in query"); + auto echo_call = create_simple_request("/echo_parsed_query"); + const std::map query_params = { + {"foo", "helloworld"}, + {"bar", "1"}, + {"fooz", "\"2\""}, + {"baz", "\"awkward\"\"escapes"}}; + for (const auto& [k, v] : query_params) { - { - INFO("Calling echo, with params in body"); - auto echo_call = create_simple_request("/echo", pack_type); - const nlohmann::json j_body = { - {"data", {"nested", "Some string"}}, {"other", "Another string"}}; - const auto serialized_body = serdes::pack(j_body, pack_type); - echo_call.set_body(serialized_body.data(), serialized_body.size()); - const auto serialized_call = echo_call.build_request(); - - auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); - frontend.process(rpc_ctx); - auto response = parse_response(rpc_ctx->serialise_response()); - CHECK(response.status == HTTP_STATUS_OK); + echo_call.set_query_param(k, v); + } - const auto response_body = parse_response_body(response.body, pack_type); - CHECK(response_body == j_body); - } + const auto serialized_call = echo_call.build_request(); - { - INFO("Calling echo_query, with params in query"); - auto echo_call = create_simple_request("/echo_parsed_query", pack_type); - const std::map query_params = { - {"foo", "helloworld"}, - {"bar", "1"}, - {"fooz", "\"2\""}, - {"baz", "\"awkward\"\"escapes"}}; - for (const auto& [k, v] : query_params) - { - echo_call.set_query_param(k, v); - } + auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); + frontend.process(rpc_ctx); + auto response = parse_response(rpc_ctx->serialise_response()); + CHECK(response.status == HTTP_STATUS_OK); - const auto serialized_call = echo_call.build_request(); + const auto response_body = parse_response_body(response.body); + const auto response_map = response_body.get(); + CHECK(response_map == query_params); +} - auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); - frontend.process(rpc_ctx); - auto response = parse_response(rpc_ctx->serialise_response()); - CHECK(response.status == HTTP_STATUS_OK); +{ + INFO("Calling get_caller"); + const auto get_caller = create_simple_request("/get_caller"); + const auto serialized_call = get_caller.build_request(); - const auto response_body = parse_response_body(response.body, pack_type); - const auto response_map = response_body.get(); - CHECK(response_map == query_params); - } + auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); + frontend.process(rpc_ctx); + auto response = parse_response(rpc_ctx->serialise_response()); + CHECK(response.status == HTTP_STATUS_OK); - { - INFO("Calling get_caller"); - const auto get_caller = create_simple_request("/get_caller", pack_type); - const auto serialized_call = get_caller.build_request(); + const auto response_body = parse_response_body(response.body); + CHECK(response_body == user_id); +} +} - auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); - frontend.process(rpc_ctx); - auto response = parse_response(rpc_ctx->serialise_response()); - CHECK(response.status == HTTP_STATUS_OK); +{ + INFO("Calling failable, without failing"); + auto dont_fail = create_simple_request("/failable"); + const auto serialized_call = dont_fail.build_request(); - const auto response_body = parse_response_body(response.body, pack_type); - CHECK(response_body == user_id); - } - } + auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); + frontend.process(rpc_ctx); + auto response = parse_response(rpc_ctx->serialise_response()); + CHECK(response.status == HTTP_STATUS_OK); +} +{ + for (const auto err : { + HTTP_STATUS_INTERNAL_SERVER_ERROR, + HTTP_STATUS_BAD_REQUEST, + (http_status)418 // Teapot + }) { - INFO("Calling failable, without failing"); - auto dont_fail = create_simple_request("/failable"); - const auto serialized_call = dont_fail.build_request(); + INFO("Calling failable, with error"); + const auto msg = fmt::format("An error message about {}", err); + auto fail = create_simple_request("/failable"); + const nlohmann::json j_body = { + {"error", {{"code", err}, {"message", msg}}}}; + const auto serialized_body = j_body.dump(); + fail.set_body(serialized_body); + const auto serialized_call = fail.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); frontend.process(rpc_ctx); auto response = parse_response(rpc_ctx->serialise_response()); - CHECK(response.status == HTTP_STATUS_OK); - } - - { - for (const auto err : { - HTTP_STATUS_INTERNAL_SERVER_ERROR, - HTTP_STATUS_BAD_REQUEST, - (http_status)418 // Teapot - }) - { - INFO("Calling failable, with error"); - const auto msg = fmt::format("An error message about {}", err); - auto fail = create_simple_request("/failable"); - const nlohmann::json j_body = { - {"error", {{"code", err}, {"message", msg}}}}; - const auto serialized_body = serdes::pack(j_body, default_pack); - fail.set_body(serialized_body.data(), serialized_body.size()); - const auto serialized_call = fail.build_request(); - - auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_call); - frontend.process(rpc_ctx); - auto response = parse_response(rpc_ctx->serialise_response()); - CHECK(response.status == err); - CHECK( - response.headers[http::headers::CONTENT_TYPE] == - http::headervalues::contenttype::JSON); - const std::string body_s(response.body.begin(), response.body.end()); - auto body_j = nlohmann::json::parse(body_s); - CHECK(body_j["error"]["message"] == msg); - } + CHECK(response.status == err); + CHECK( + response.headers[ccf::http::headers::CONTENT_TYPE] == + ccf::http::headervalues::contenttype::JSON); + const std::string body_s(response.body.begin(), response.body.end()); + auto body_j = nlohmann::json::parse(body_s); + CHECK(body_j["error"]["message"] == msg); } } +} TEST_CASE("Restricted verbs") { @@ -773,7 +772,7 @@ TEST_CASE("Restricted verbs") HTTP_STATUS_METHOD_NOT_ALLOWED; { - http::Request get("get_only", verb); + ::http::Request get("get_only", verb); const auto serialized_get = get.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_get); frontend.process(rpc_ctx); @@ -786,7 +785,7 @@ TEST_CASE("Restricted verbs") else { CHECK(response.status == other_verb_status); - const auto it = response.headers.find(http::headers::ALLOW); + const auto it = response.headers.find(ccf::http::headers::ALLOW); REQUIRE(it != response.headers.end()); const auto v = it->second; CHECK(v.find(llhttp_method_name(HTTP_GET)) != std::string::npos); @@ -794,7 +793,7 @@ TEST_CASE("Restricted verbs") } { - http::Request post("post_only", verb); + ::http::Request post("post_only", verb); const auto serialized_post = post.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_post); frontend.process(rpc_ctx); @@ -807,7 +806,7 @@ TEST_CASE("Restricted verbs") else { CHECK(response.status == other_verb_status); - const auto it = response.headers.find(http::headers::ALLOW); + const auto it = response.headers.find(ccf::http::headers::ALLOW); REQUIRE(it != response.headers.end()); const auto v = it->second; CHECK(v.find(llhttp_method_name(HTTP_POST)) != std::string::npos); @@ -815,7 +814,7 @@ TEST_CASE("Restricted verbs") } { - http::Request put_or_delete("put_or_delete", verb); + ::http::Request put_or_delete("put_or_delete", verb); const auto serialized_put_or_delete = put_or_delete.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_put_or_delete); @@ -829,7 +828,7 @@ TEST_CASE("Restricted verbs") else { CHECK(response.status == other_verb_status); - const auto it = response.headers.find(http::headers::ALLOW); + const auto it = response.headers.find(ccf::http::headers::ALLOW); REQUIRE(it != response.headers.end()); const auto v = it->second; CHECK(v.find(llhttp_method_name(HTTP_PUT)) != std::string::npos); @@ -866,7 +865,7 @@ TEST_CASE("Explicit commitability") { auto tx = network.tables->create_tx(); tx.rw(frontend.values)->put(0, next_value); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } for (const auto status : all_statuses) @@ -877,12 +876,12 @@ TEST_CASE("Explicit commitability") INFO("Without override..."); const auto new_value = ++next_value; - http::Request request("maybe_commit", HTTP_POST); + ::http::Request request("maybe_commit", HTTP_POST); const nlohmann::json request_body = { {"value", new_value}, {"status", status}}; - const auto serialized_body = serdes::pack(request_body, default_pack); - request.set_body(&serialized_body); + const auto serialized_body = request_body.dump(); + request.set_body(serialized_body); const auto serialized_request = request.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_request); @@ -913,12 +912,12 @@ TEST_CASE("Explicit commitability") { const auto new_value = ++next_value; - http::Request request("maybe_commit", HTTP_POST); + ::http::Request request("maybe_commit", HTTP_POST); const nlohmann::json request_body = { {"value", new_value}, {"apply", apply}, {"status", status}}; - const auto serialized_body = serdes::pack(request_body, default_pack); - request.set_body(&serialized_body); + const auto serialized_body = request_body.dump(); + request.set_body(serialized_body); const auto serialized_request = request.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_request); @@ -963,7 +962,7 @@ TEST_CASE("Alternative endpoints") for (auto verb : {HTTP_GET, HTTP_POST}) { - http::Request read_only("read_only", verb); + ::http::Request read_only("read_only", verb); const auto serialized_read_only = read_only.build_request(); auto rpc_ctx = ccf::make_rpc_context(user_session, serialized_read_only); @@ -1057,7 +1056,8 @@ TEST_CASE("Forwarding" * doctest::test_suite("forwarding")) TestForwardingUserFrontEnd user_frontend_primary(*network_primary.tables); TestForwardingUserFrontEnd user_frontend_backup(*network_backup.tables); - auto primary_consensus = std::make_shared(); + auto primary_consensus = + std::make_shared(); network_primary.tables->set_consensus(primary_consensus); auto channel_stub = std::make_shared(); @@ -1065,7 +1065,8 @@ TEST_CASE("Forwarding" * doctest::test_suite("forwarding")) auto rpc_map = std::weak_ptr(); auto backup_forwarder = std::make_shared>( rpc_responder, channel_stub, rpc_map); - auto backup_consensus = std::make_shared(); + auto backup_consensus = + std::make_shared(); network_backup.tables->set_consensus(backup_consensus); auto simple_call = create_simple_request(); @@ -1113,7 +1114,7 @@ TEST_CASE("Forwarding" * doctest::test_suite("forwarding")) auto forwarded_msg = channel_stub->get_pop_back(); auto fwd_ctx = backup_forwarder->recv_forwarded_command( - kv::test::FirstBackupNodeId, + ccf::kv::test::FirstBackupNodeId, forwarded_msg.data(), forwarded_msg.size()); @@ -1145,7 +1146,7 @@ TEST_CASE("Forwarding" * doctest::test_suite("forwarding")) auto forwarded_msg = channel_stub->get_pop_back(); auto fwd_ctx = backup_forwarder->recv_forwarded_command( - kv::test::FirstBackupNodeId, + ccf::kv::test::FirstBackupNodeId, forwarded_msg.data(), forwarded_msg.size()); @@ -1203,7 +1204,8 @@ TEST_CASE("Nodefrontend forwarding" * doctest::test_suite("forwarding")) auto channel_stub = std::make_shared(); - auto primary_consensus = std::make_shared(); + auto primary_consensus = + std::make_shared(); network_primary.tables->set_consensus(primary_consensus); auto rpc_responder = std::weak_ptr(); @@ -1211,7 +1213,8 @@ TEST_CASE("Nodefrontend forwarding" * doctest::test_suite("forwarding")) auto backup_forwarder = std::make_shared>( rpc_responder, channel_stub, rpc_map); node_frontend_backup.set_cmd_forwarder(backup_forwarder); - auto backup_consensus = std::make_shared(); + auto backup_consensus = + std::make_shared(); network_backup.tables->set_consensus(backup_consensus); auto write_req = create_simple_request(); @@ -1227,7 +1230,9 @@ TEST_CASE("Nodefrontend forwarding" * doctest::test_suite("forwarding")) auto forwarded_msg = channel_stub->get_pop_back(); auto fwd_ctx = backup_forwarder->recv_forwarded_command( - kv::test::FirstBackupNodeId, forwarded_msg.data(), forwarded_msg.size()); + ccf::kv::test::FirstBackupNodeId, + forwarded_msg.data(), + forwarded_msg.size()); node_frontend_primary.process_forwarded(fwd_ctx); auto response = parse_response(fwd_ctx->serialise_response()); @@ -1250,7 +1255,8 @@ TEST_CASE("Userfrontend forwarding" * doctest::test_suite("forwarding")) auto channel_stub = std::make_shared(); - auto primary_consensus = std::make_shared(); + auto primary_consensus = + std::make_shared(); network_primary.tables->set_consensus(primary_consensus); auto rpc_responder = std::weak_ptr(); @@ -1258,7 +1264,8 @@ TEST_CASE("Userfrontend forwarding" * doctest::test_suite("forwarding")) auto backup_forwarder = std::make_shared>( rpc_responder, channel_stub, rpc_map); user_frontend_backup.set_cmd_forwarder(backup_forwarder); - auto backup_consensus = std::make_shared(); + auto backup_consensus = + std::make_shared(); network_backup.tables->set_consensus(backup_consensus); auto write_req = create_simple_request(); @@ -1272,7 +1279,9 @@ TEST_CASE("Userfrontend forwarding" * doctest::test_suite("forwarding")) auto forwarded_msg = channel_stub->get_pop_back(); auto fwd_ctx = backup_forwarder->recv_forwarded_command( - kv::test::FirstBackupNodeId, forwarded_msg.data(), forwarded_msg.size()); + ccf::kv::test::FirstBackupNodeId, + forwarded_msg.data(), + forwarded_msg.size()); user_frontend_primary.process_forwarded(fwd_ctx); auto response = parse_response(fwd_ctx->serialise_response()); @@ -1297,7 +1306,8 @@ TEST_CASE("Memberfrontend forwarding" * doctest::test_suite("forwarding")) TestForwardingMemberFrontEnd member_frontend_backup(network_backup, context); auto channel_stub = std::make_shared(); - auto primary_consensus = std::make_shared(); + auto primary_consensus = + std::make_shared(); network_primary.tables->set_consensus(primary_consensus); auto rpc_responder = std::weak_ptr(); @@ -1305,7 +1315,8 @@ TEST_CASE("Memberfrontend forwarding" * doctest::test_suite("forwarding")) auto backup_forwarder = std::make_shared>( rpc_responder, channel_stub, rpc_map); member_frontend_backup.set_cmd_forwarder(backup_forwarder); - auto backup_consensus = std::make_shared(); + auto backup_consensus = + std::make_shared(); network_backup.tables->set_consensus(backup_consensus); auto write_req = create_simple_request(); @@ -1319,7 +1330,9 @@ TEST_CASE("Memberfrontend forwarding" * doctest::test_suite("forwarding")) auto forwarded_msg = channel_stub->get_pop_back(); auto fwd_ctx = backup_forwarder->recv_forwarded_command( - kv::test::FirstBackupNodeId, forwarded_msg.data(), forwarded_msg.size()); + ccf::kv::test::FirstBackupNodeId, + forwarded_msg.data(), + forwarded_msg.size()); member_frontend_primary.process_forwarded(fwd_ctx); auto response = parse_response(fwd_ctx->serialise_response()); @@ -1332,10 +1345,10 @@ TEST_CASE("Memberfrontend forwarding" * doctest::test_suite("forwarding")) class TestConflictFrontend : public BaseTestFrontend { public: - using Values = kv::Map; + using Values = ccf::kv::Map; size_t execution_count = 0; - TestConflictFrontend(kv::Store& tables) : BaseTestFrontend(tables) + TestConflictFrontend(ccf::kv::Store& tables) : BaseTestFrontend(tables) { open(); @@ -1355,7 +1368,7 @@ class TestConflictFrontend : public BaseTestFrontend auto tx = this->tables.create_tx(); auto conflict_map = tx.template rw("test_values_conflict"); conflict_map->put(0, 42); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // Indicate that the execution conflicted ctx.rpc_ctx->set_response_header("test-has-conflicted", "true"); @@ -1426,7 +1439,7 @@ TEST_CASE("Retry on conflict") class TestManualConflictsRegistry : public UserEndpointRegistry { public: - using MyVals = kv::Map; + using MyVals = ccf::kv::Map; static constexpr auto SRC = "source"; static constexpr auto DST = "destination"; @@ -1483,7 +1496,7 @@ class TestManualConflictsRegistry : public UserEndpointRegistry } } - TestManualConflictsRegistry(ccfapp::AbstractNodeContext& context) : + TestManualConflictsRegistry(ccf::AbstractNodeContext& context) : UserEndpointRegistry(context) { auto pausable = [this](auto& ctx) { @@ -1528,7 +1541,7 @@ class TestManualConflictsFrontend ccf::StubNodeContext context; - TestManualConflictsFrontend(kv::Store& tables) : + TestManualConflictsFrontend(ccf::kv::Store& tables) : TSimpleFrontend(tables, context) { open(); @@ -1546,18 +1559,18 @@ TEST_CASE("Manual conflicts") auto call_pausable = [&]( std::shared_ptr session, http_status expected_status) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto req = create_simple_request("/pausable"); auto serialized_call = req.build_request(); auto rpc_ctx = ccf::make_rpc_context(session, serialized_call); frontend.process(rpc_ctx); auto response = parse_response(rpc_ctx->serialise_response()); CHECK(response.status == expected_status); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; auto get_metrics = [&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto req = create_simple_request("/pausable/metrics"); req.set_method(HTTP_GET); auto serialized_call = req.build_request(); @@ -1570,36 +1583,36 @@ TEST_CASE("Manual conflicts") ret.calls = body["calls"].get(); ret.retries = body["retries"].get(); ret.errors = body["errors"].get(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return ret; }; auto get_value = [&](const std::string& table = TF::DST) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto tx = network.tables->create_tx(); auto handle = tx.ro(table); auto ret = handle->get(TF::KEY); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); - crypto::openssl_sha256_shutdown(); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); + ccf::crypto::openssl_sha256_shutdown(); return ret; }; auto update_value = [&](size_t n, const std::string& table = TF::SRC, size_t key = TF::KEY) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto tx = network.tables->create_tx(); using TF = TestManualConflictsFrontend; auto handle = tx.wo(table); handle->put(key, n); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); - crypto::openssl_sha256_shutdown(); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); + ccf::crypto::openssl_sha256_shutdown(); }; auto run_test = [&]( std::function&& read_write_op, std::shared_ptr session = user_session, http_status expected_status = HTTP_STATUS_OK) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); frontend.registry.before_read.ready = false; frontend.registry.after_read.ready = false; frontend.registry.before_write.ready = false; @@ -1616,7 +1629,7 @@ TEST_CASE("Manual conflicts") frontend.registry.after_write.wait(); worker.join(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; { @@ -1712,7 +1725,7 @@ TEST_CASE("Manual conflicts") using TF = TestManualConflictsFrontend; auto handle = tx.wo(TF::SRC); handle->remove(TF::KEY); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); }); const auto metrics_after = get_metrics(); @@ -1735,7 +1748,7 @@ TEST_CASE("Manual conflicts") using TF = TestManualConflictsFrontend; auto handle = tx.wo(TF::SRC); handle->clear(); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); }); const auto metrics_after = get_metrics(); @@ -1758,7 +1771,7 @@ TEST_CASE("Manual conflicts") [&]() { auto tx = network.tables->create_tx(); InternalTablesAccess::remove_user(tx, user_id); - CHECK(tx.commit() == kv::CommitResult::SUCCESS); + CHECK(tx.commit() == ccf::kv::CommitResult::SUCCESS); }, user_session, HTTP_STATUS_UNAUTHORIZED); @@ -1781,11 +1794,11 @@ int main(int argc, char** argv) std::chrono::system_clock::now().time_since_epoch()); ::threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/rpc/test/frontend_test_infra.h b/src/node/rpc/test/frontend_test_infra.h index 715a66e5fdd0..815dbb7d4264 100644 --- a/src/node/rpc/test/frontend_test_infra.h +++ b/src/node/rpc/test/frontend_test_infra.h @@ -5,7 +5,6 @@ #include "ccf/app_interface.h" #include "ccf/crypto/rsa_key_pair.h" #include "ccf/ds/logger.h" -#include "ccf/serdes.h" #include "ccf/service/signed_req.h" #include "ds/files.h" #include "kv/test/null_encryptor.h" @@ -19,31 +18,28 @@ #include #include -using namespace ccfapp; +using namespace ccf; using namespace ccf; using namespace std; -using namespace serdes; using namespace nlohmann; -using TResponse = http::SimpleResponseProcessor::Response; +using TResponse = ::http::SimpleResponseProcessor::Response; // used throughout constexpr size_t certificate_validity_period_days = 365; using namespace std::literals; auto valid_from = ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); -auto valid_to = crypto::compute_cert_valid_to_string( +auto valid_to = ccf::crypto::compute_cert_valid_to_string( valid_from, certificate_validity_period_days); -auto kp = crypto::make_key_pair(); +auto kp = ccf::crypto::make_key_pair(); auto member_cert = kp -> self_sign("CN=name_member", valid_from, valid_to); -auto verifier_mem = crypto::make_verifier(member_cert); +auto verifier_mem = ccf::crypto::make_verifier(member_cert); auto user_cert = kp -> self_sign("CN=name_user", valid_from, valid_to); -auto dummy_enc_pubk = crypto::make_rsa_key_pair() -> public_key_pem(); - -auto encryptor = std::make_shared(); +auto dummy_enc_pubk = ccf::crypto::make_rsa_key_pair() -> public_key_pem(); -constexpr auto default_pack = serdes::Pack::Text; +auto encryptor = std::make_shared(); template T parse_response_body(const TResponse& r) @@ -51,7 +47,7 @@ T parse_response_body(const TResponse& r) nlohmann::json body_j; try { - body_j = serdes::unpack(r.body, serdes::Pack::Text); + body_j = nlohmann::json::parse(r.body); } catch (const nlohmann::json::parse_error& e) { @@ -81,29 +77,28 @@ void check_error_message(const TResponse& r, const std::string& msg) std::vector create_request( const json& params, const string& method_name, llhttp_method verb = HTTP_POST) { - http::Request r(fmt::format("/gov/{}", method_name), verb); - const auto body = params.is_null() ? std::vector() : - serdes::pack(params, default_pack); - r.set_body(&body); + ::http::Request r(fmt::format("/gov/{}", method_name), verb); + const auto body = params.is_null() ? std::string() : params.dump(); + r.set_body(body); return r.build_request(); } auto frontend_process( MemberRpcFrontend& frontend, const std::vector& serialized_request, - const crypto::Pem& caller) + const ccf::crypto::Pem& caller) { auto session = std::make_shared( - ccf::InvalidSessionId, crypto::make_verifier(caller)->cert_der()); + ccf::InvalidSessionId, ccf::crypto::make_verifier(caller)->cert_der()); auto rpc_ctx = ccf::make_rpc_context(session, serialized_request); - http::extract_actor(*rpc_ctx); + ::http::extract_actor(*rpc_ctx); frontend.process(rpc_ctx); DOCTEST_CHECK(!rpc_ctx->response_is_pending); auto serialized_response = rpc_ctx->serialise_response(); - http::SimpleResponseProcessor processor; - http::ResponseParser parser(processor); + ::http::SimpleResponseProcessor processor; + ::http::ResponseParser parser(processor); parser.execute(serialized_response.data(), serialized_response.size()); DOCTEST_REQUIRE(processor.received.size() == 1); @@ -111,7 +106,7 @@ auto frontend_process( return processor.received.front(); } -auto get_cert(uint64_t member_id, crypto::KeyPairPtr& kp_mem) +auto get_cert(uint64_t member_id, ccf::crypto::KeyPairPtr& kp_mem) { return kp_mem->self_sign( "CN=new member" + to_string(member_id), valid_from, valid_to); @@ -124,7 +119,7 @@ std::unique_ptr make_test_network_ident() ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); return std::make_unique( "CN=CCF test network", - crypto::service_identity_curve_choice, + ccf::crypto::service_identity_curve_choice, valid_from, 2); } @@ -133,9 +128,9 @@ void init_network(NetworkState& network) { network.tables->set_encryptor(encryptor); auto history = std::make_shared( - *network.tables, kv::test::PrimaryNodeId, *kp); + *network.tables, ccf::kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); network.tables->set_consensus(consensus); network.identity = make_test_network_ident(); } \ No newline at end of file diff --git a/src/node/rpc/test/node_frontend_test.cpp b/src/node/rpc/test/node_frontend_test.cpp index f912ea1f903e..81324870b315 100644 --- a/src/node/rpc/test/node_frontend_test.cpp +++ b/src/node/rpc/test/node_frontend_test.cpp @@ -4,7 +4,6 @@ #include "ccf/crypto/pem.h" #include "ccf/crypto/verifier.h" #include "ccf/ds/logger.h" -#include "ccf/serdes.h" #include "crypto/openssl/hash.h" #include "frontend_test_infra.h" #include "kv/test/null_encryptor.h" @@ -15,9 +14,8 @@ using namespace ccf; using namespace nlohmann; -using namespace serdes; -using TResponse = http::SimpleResponseProcessor::Response; +using TResponse = ::http::SimpleResponseProcessor::Response; auto node_id = 0; @@ -25,13 +23,11 @@ TResponse frontend_process( NodeRpcFrontend& frontend, const json& json_params, const std::string& method, - const crypto::Pem& caller) + const ccf::crypto::Pem& caller) { - http::Request r(method); - const auto body = json_params.is_null() ? - std::vector() : - serdes::pack(json_params, Pack::Text); - r.set_body(&body); + ::http::Request r(method); + const auto body = json_params.is_null() ? std::string() : json_params.dump(); + r.set_body(body); auto serialise_request = r.build_request(); auto session = @@ -42,8 +38,8 @@ TResponse frontend_process( CHECK(!rpc_ctx->response_is_pending); const auto serialised_response = rpc_ctx->serialise_response(); - http::SimpleResponseProcessor processor; - http::ResponseParser parser(processor); + ::http::SimpleResponseProcessor processor; + ::http::ResponseParser parser(processor); parser.execute(serialised_response.data(), serialised_response.size()); REQUIRE(processor.received.size() == 1); @@ -65,7 +61,7 @@ void require_ledger_secrets_equal( TEST_CASE("Add a node to an opening service") { NetworkState network; - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); auto gen_tx = network.tables->create_tx(); InternalTablesAccess::init_configuration( @@ -80,15 +76,15 @@ TEST_CASE("Add a node to an opening service") frontend.open(); // New node should not be given ledger secret past this one via join request - kv::Version up_to_ledger_secret_seqno = 4; + ccf::kv::Version up_to_ledger_secret_seqno = 4; network.ledger_secrets->set_secret( up_to_ledger_secret_seqno, make_ledger_secret()); // Node certificate - crypto::KeyPairPtr kp = crypto::make_key_pair(); + ccf::crypto::KeyPairPtr kp = ccf::crypto::make_key_pair(); const auto caller = kp->self_sign("CN=Joiner", valid_from, valid_to); const auto node_public_encryption_key = - crypto::make_key_pair()->public_key_pem(); + ccf::crypto::make_key_pair()->public_key_pem(); INFO("Add first node before a service exists"); { @@ -103,7 +99,7 @@ TEST_CASE("Add a node to an opening service") InternalTablesAccess::create_service( gen_tx, network.identity->cert, ccf::TxID{}); - REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(gen_tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx = network.tables->create_tx(); INFO("Add first node which should be trusted straight away"); @@ -127,7 +123,7 @@ TEST_CASE("Add a node to an opening service") CHECK(response.network_info->endorsed_certificate == std::nullopt); auto pk_der = kp->public_key_der(); - const NodeId node_id = crypto::Sha256Hash(pk_der).hex_str(); + const NodeId node_id = ccf::crypto::Sha256Hash(pk_der).hex_str(); auto nodes = tx.rw(network.nodes); auto node_info = nodes->get(node_id); @@ -163,8 +159,8 @@ TEST_CASE("Add a node to an opening service") INFO( "Adding a different node with the same node network details should fail"); { - crypto::KeyPairPtr kp = crypto::make_key_pair(); - auto v = crypto::make_verifier( + ccf::crypto::KeyPairPtr kp = ccf::crypto::make_key_pair(); + auto v = ccf::crypto::make_verifier( kp->self_sign("CN=Other Joiner", valid_from, valid_to)); const auto new_caller = v->cert_pem(); @@ -185,7 +181,7 @@ TEST_CASE("Add a node to an open service") { NetworkState network; auto gen_tx = network.tables->create_tx(); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); network.identity = make_test_network_ident(); @@ -198,7 +194,7 @@ TEST_CASE("Add a node to an open service") frontend.open(); // New node should not be given ledger secret past this one via join request - kv::Version up_to_ledger_secret_seqno = 4; + ccf::kv::Version up_to_ledger_secret_seqno = 4; network.ledger_secrets->set_secret( up_to_ledger_secret_seqno, make_ledger_secret()); @@ -208,19 +204,20 @@ TEST_CASE("Add a node to an open service") InternalTablesAccess::activate_member( gen_tx, InternalTablesAccess::add_member( - gen_tx, {member_cert, crypto::make_rsa_key_pair()->public_key_pem()})); + gen_tx, + {member_cert, ccf::crypto::make_rsa_key_pair()->public_key_pem()})); REQUIRE(InternalTablesAccess::open_service(gen_tx)); - REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(gen_tx.commit() == ccf::kv::CommitResult::SUCCESS); // Node certificate - crypto::KeyPairPtr kp = crypto::make_key_pair(); + ccf::crypto::KeyPairPtr kp = ccf::crypto::make_key_pair(); const auto caller = kp->self_sign("CN=Joiner", valid_from, valid_to); std::optional node_info; auto tx = network.tables->create_tx(); const auto node_public_encryption_key = - crypto::make_key_pair()->public_key_pem(); + ccf::crypto::make_key_pair()->public_key_pem(); JoinNetworkNodeToNode::In join_input; join_input.public_encryption_key = node_public_encryption_key; @@ -237,7 +234,7 @@ TEST_CASE("Add a node to an open service") CHECK(!response.network_info.has_value()); auto pk_der = kp->public_key_der(); - const NodeId node_id = crypto::Sha256Hash(pk_der).hex_str(); + const NodeId node_id = ccf::crypto::Sha256Hash(pk_der).hex_str(); auto nodes = tx.rw(network.nodes); node_info = nodes->get(node_id); CHECK(node_info.has_value()); @@ -248,9 +245,9 @@ TEST_CASE("Add a node to an open service") INFO( "Adding a different node with the same node network details should fail"); { - crypto::KeyPairPtr kp = crypto::make_key_pair(); - auto v = - crypto::make_verifier(kp->self_sign("CN=Joiner", valid_from, valid_to)); + ccf::crypto::KeyPairPtr kp = ccf::crypto::make_key_pair(); + auto v = ccf::crypto::make_verifier( + kp->self_sign("CN=Joiner", valid_from, valid_to)); const auto new_caller = v->cert_pem(); // Network node info is empty (same as before) @@ -283,11 +280,12 @@ TEST_CASE("Add a node to an open service") auto joining_node_id = ccf::compute_node_id_from_kp(kp); InternalTablesAccess::trust_node( tx, joining_node_id, network.ledger_secrets->get_latest(tx).first); - const auto dummy_endorsed_certificate = crypto::make_key_pair()->self_sign( - "CN=dummy endorsed certificate", valid_from, valid_to); + const auto dummy_endorsed_certificate = + ccf::crypto::make_key_pair()->self_sign( + "CN=dummy endorsed certificate", valid_from, valid_to); auto endorsed_certificate = tx.rw(network.node_endorsed_certificates); endorsed_certificate->put(joining_node_id, {dummy_endorsed_certificate}); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); // In the meantime, a new ledger secret is added. The new ledger secret // should not be passed to the new joiner via the join @@ -318,11 +316,11 @@ TEST_CASE("Add a node to an open service") int main(int argc, char** argv) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index f73de6ecac7f..fd539f19e47a 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -66,9 +66,9 @@ namespace ccf return true; } - kv::Version get_last_recovered_signed_idx() override + ccf::kv::Version get_last_recovered_signed_idx() override { - return kv::NoVersion; + return ccf::kv::NoVersion; } SessionMetrics get_session_metrics() override @@ -82,7 +82,7 @@ namespace ccf } QuoteVerificationResult verify_quote( - kv::ReadOnlyTx& tx, + ccf::kv::ReadOnlyTx& tx, const QuoteInfo& quote_info, const std::vector& expected_node_public_key_der, pal::PlatformAttestationMeasurement& measurement) override @@ -90,17 +90,17 @@ namespace ccf return QuoteVerificationResult::Verified; } - kv::Version get_startup_snapshot_seqno() override + ccf::kv::Version get_startup_snapshot_seqno() override { return 0; } - void initiate_private_recovery(kv::Tx& tx) override + void initiate_private_recovery(ccf::kv::Tx& tx) override { throw std::logic_error("Unimplemented"); } - crypto::Pem get_self_signed_node_certificate() override + ccf::crypto::Pem get_self_signed_node_certificate() override { return {}; } @@ -109,33 +109,33 @@ namespace ccf class StubGovernanceEffects : public ccf::AbstractGovernanceEffects { public: - void transition_service_to_open(kv::Tx& tx, ServiceIdentities) override + void transition_service_to_open(ccf::kv::Tx& tx, ServiceIdentities) override { return; } - bool rekey_ledger(kv::Tx& tx) override + bool rekey_ledger(ccf::kv::Tx& tx) override { return true; } - void trigger_recovery_shares_refresh(kv::Tx& tx) override + void trigger_recovery_shares_refresh(ccf::kv::Tx& tx) override { return; } - void trigger_ledger_chunk(kv::Tx& tx) override + void trigger_ledger_chunk(ccf::kv::Tx& tx) override { return; } - void trigger_snapshot(kv::Tx& tx) override + void trigger_snapshot(ccf::kv::Tx& tx) override { return; } void trigger_acme_refresh( - kv::Tx& tx, + ccf::kv::Tx& tx, const std::optional>& interfaces = std::nullopt) override { @@ -161,9 +161,11 @@ namespace ccf historical::ExpiryDuration seconds_until_expiry) {} + void set_soft_cache_limit(historical::CacheSize cache_limit){}; + void track_deletes_on_missing_keys(bool track) {} - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( historical::RequestHandle handle, ccf::SeqNo seqno, historical::ExpiryDuration seconds_until_expiry) @@ -171,7 +173,7 @@ namespace ccf return nullptr; } - kv::ReadOnlyStorePtr get_store_at( + ccf::kv::ReadOnlyStorePtr get_store_at( historical::RequestHandle handle, ccf::SeqNo seqno) { return nullptr; @@ -191,7 +193,7 @@ namespace ccf return nullptr; } - std::vector get_store_range( + std::vector get_store_range( historical::RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno, @@ -200,7 +202,7 @@ namespace ccf return {}; } - std::vector get_store_range( + std::vector get_store_range( historical::RequestHandle handle, ccf::SeqNo start_seqno, ccf::SeqNo end_seqno) @@ -225,7 +227,7 @@ namespace ccf return {}; } - std::vector get_stores_for( + std::vector get_stores_for( historical::RequestHandle handle, const SeqNoCollection& seqnos, historical::ExpiryDuration seconds_until_expiry) @@ -233,7 +235,7 @@ namespace ccf return {}; } - std::vector get_stores_for( + std::vector get_stores_for( historical::RequestHandle handle, const SeqNoCollection& seqnos) { return {}; @@ -259,7 +261,7 @@ namespace ccf } }; - struct StubNodeContext : public ccfapp::AbstractNodeContext + struct StubNodeContext : public ccf::AbstractNodeContext { public: std::shared_ptr node_operation = nullptr; @@ -284,7 +286,7 @@ namespace ccf ccf::NodeId get_node_id() const override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } }; } diff --git a/src/node/rpc/user_frontend.h b/src/node/rpc/user_frontend.h index 0e273ee67a04..73051816da28 100644 --- a/src/node/rpc/user_frontend.h +++ b/src/node/rpc/user_frontend.h @@ -18,7 +18,7 @@ namespace ccf UserRpcFrontend( NetworkState& network, std::unique_ptr&& endpoints_, - ccfapp::AbstractNodeContext& context_) : + ccf::AbstractNodeContext& context_) : RpcFrontend(*network.tables, *endpoints_, context_), endpoints(std::move(endpoints_)) {} diff --git a/src/node/secret_broadcast.h b/src/node/secret_broadcast.h index 92d0e62e6f5d..fba11c73b4b7 100644 --- a/src/node/secret_broadcast.h +++ b/src/node/secret_broadcast.h @@ -18,7 +18,7 @@ namespace ccf static void broadcast_some( NetworkState& network, NodeId self, - kv::Tx& tx, + ccf::kv::Tx& tx, const LedgerSecretsMap& some_ledger_secrets) { auto secrets = tx.rw(network.secrets); @@ -33,8 +33,8 @@ namespace ccf { ledger_secrets_for_node.push_back( {s.first, - crypto::ckm_rsa_pkcs_oaep_wrap( - crypto::make_rsa_public_key(ni.encryption_pub_key), + ccf::crypto::ckm_rsa_pkcs_oaep_wrap( + ccf::crypto::make_rsa_public_key(ni.encryption_pub_key), s.second->raw_key), s.second->previous_secret_stored_version}); } @@ -46,7 +46,9 @@ namespace ccf } static void broadcast_new( - NetworkState& network, kv::Tx& tx, LedgerSecretPtr&& new_ledger_secret) + NetworkState& network, + ccf::kv::Tx& tx, + LedgerSecretPtr&& new_ledger_secret) { auto secrets = tx.rw(network.secrets); @@ -58,8 +60,8 @@ namespace ccf ledger_secrets_for_node.push_back( {std::nullopt, - crypto::ckm_rsa_pkcs_oaep_wrap( - crypto::make_rsa_public_key(ni.encryption_pub_key), + ccf::crypto::ckm_rsa_pkcs_oaep_wrap( + ccf::crypto::make_rsa_public_key(ni.encryption_pub_key), new_ledger_secret->raw_key), new_ledger_secret->previous_secret_stored_version}); @@ -70,10 +72,10 @@ namespace ccf } static std::vector decrypt( - const crypto::RSAKeyPairPtr& encryption_key, + const ccf::crypto::RSAKeyPairPtr& encryption_key, const std::vector& cipher) { - return crypto::ckm_rsa_pkcs_oaep_unwrap(encryption_key, cipher); + return ccf::crypto::ckm_rsa_pkcs_oaep_unwrap(encryption_key, cipher); } }; } \ No newline at end of file diff --git a/src/node/secret_share.h b/src/node/secret_share.h index e8939e2679b6..69f7c377b6ee 100644 --- a/src/node/secret_share.h +++ b/src/node/secret_share.h @@ -17,7 +17,7 @@ extern "C" /// SSS assumes that there is a function of this prototype int randombytes(void* buf, size_t n) { - crypto::EntropyPtr entropy = crypto::get_entropy(); + ccf::crypto::EntropyPtr entropy = ccf::crypto::get_entropy(); entropy->random((unsigned char*)buf, n); return 0; } diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 45d071dd6c1a..bdf9b24bc2d6 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -22,12 +22,12 @@ namespace ccf class LedgerSecretWrappingKey { private: - static constexpr auto KZ_KEY_SIZE = crypto::GCM_DEFAULT_KEY_SIZE; + static constexpr auto KZ_KEY_SIZE = ccf::crypto::GCM_DEFAULT_KEY_SIZE; bool has_wrapped = false; size_t num_shares; size_t recovery_threshold; std::vector data; // Referred to as "kz" in TR - std::vector shares; + std::vector shares; public: LedgerSecretWrappingKey(size_t num_shares_, size_t recovery_threshold_) : @@ -35,20 +35,20 @@ namespace ccf recovery_threshold(recovery_threshold_) { shares.resize(num_shares); - crypto::sharing::Share secret; - crypto::sharing::sample_secret_and_shares( + ccf::crypto::sharing::Share secret; + ccf::crypto::sharing::sample_secret_and_shares( secret, shares, recovery_threshold); data = secret.key(KZ_KEY_SIZE); } LedgerSecretWrappingKey( - std::vector&& shares_, + std::vector&& shares_, size_t recovery_threshold_) : recovery_threshold(recovery_threshold_) { shares = shares_; - crypto::sharing::Share secret; - crypto::sharing::recover_unauthenticated_secret( + ccf::crypto::sharing::Share secret; + ccf::crypto::sharing::recover_unauthenticated_secret( secret, shares, recovery_threshold); data = secret.key(KZ_KEY_SIZE); } @@ -81,7 +81,7 @@ namespace ccf std::vector> get_shares() const { std::vector> shares_; - for (const crypto::sharing::Share& share : shares) + for (const ccf::crypto::sharing::Share& share : shares) { shares_.emplace_back(share.serialise()); } @@ -104,9 +104,9 @@ namespace ccf "Ledger secret wrapping key has already wrapped once"); } - crypto::GcmCipher encrypted_ls(ledger_secret->raw_key.size()); + ccf::crypto::GcmCipher encrypted_ls(ledger_secret->raw_key.size()); - crypto::make_key_aes_gcm(data)->encrypt( + ccf::crypto::make_key_aes_gcm(data)->encrypt( encrypted_ls.hdr.get_iv(), // iv is always 0 here as the share wrapping // key is never re-used for encryption ledger_secret->raw_key, @@ -122,11 +122,11 @@ namespace ccf LedgerSecretPtr unwrap( const std::vector& wrapped_latest_ledger_secret) { - crypto::GcmCipher encrypted_ls; + ccf::crypto::GcmCipher encrypted_ls; encrypted_ls.deserialise(wrapped_latest_ledger_secret); std::vector decrypted_ls; - if (!crypto::make_key_aes_gcm(data)->decrypt( + if (!ccf::crypto::make_key_aes_gcm(data)->decrypt( encrypted_ls.hdr.get_iv(), encrypted_ls.hdr.tag, encrypted_ls.cipher, @@ -157,7 +157,7 @@ namespace ccf std::shared_ptr ledger_secrets; EncryptedSharesMap compute_encrypted_shares( - kv::Tx& tx, const LedgerSecretWrappingKey& ls_wrapping_key) + ccf::kv::Tx& tx, const LedgerSecretWrappingKey& ls_wrapping_key) { EncryptedSharesMap encrypted_shares; auto shares = ls_wrapping_key.get_shares(); @@ -168,7 +168,7 @@ namespace ccf size_t share_index = 0; for (auto const& [member_id, enc_pub_key] : active_recovery_members_info) { - auto member_enc_pubk = crypto::make_rsa_public_key(enc_pub_key); + auto member_enc_pubk = ccf::crypto::make_rsa_public_key(enc_pub_key); auto raw_share = std::vector( shares[share_index].begin(), shares[share_index].end()); encrypted_shares[member_id] = member_enc_pubk->rsa_oaep_wrap(raw_share); @@ -181,7 +181,7 @@ namespace ccf } void shuffle_recovery_shares( - kv::Tx& tx, const LedgerSecretPtr& latest_ledger_secret) + ccf::kv::Tx& tx, const LedgerSecretPtr& latest_ledger_secret) { auto active_recovery_members_info = InternalTablesAccess::get_active_recovery_members(tx); @@ -224,11 +224,11 @@ namespace ccf } void set_recovery_shares_info( - kv::Tx& tx, + ccf::kv::Tx& tx, const LedgerSecretPtr& latest_ledger_secret, const std::optional& previous_ledger_secret = std::nullopt, - std::optional latest_ls_version = std::nullopt) + std::optional latest_ls_version = std::nullopt) { // First, generate a fresh ledger secrets wrapping key and wrap the // latest ledger secret with it. Then, encrypt the penultimate ledger @@ -243,12 +243,12 @@ namespace ccf Tables::ENCRYPTED_PAST_LEDGER_SECRET); std::vector encrypted_previous_secret = {}; - kv::Version version_previous_secret = kv::NoVersion; + ccf::kv::Version version_previous_secret = ccf::kv::NoVersion; if (previous_ledger_secret.has_value()) { version_previous_secret = previous_ledger_secret->first; - crypto::GcmCipher encrypted_previous_ls( + ccf::crypto::GcmCipher encrypted_previous_ls( previous_ledger_secret->second->raw_key.size()); encrypted_previous_ls.hdr.set_random_iv(); @@ -278,7 +278,7 @@ namespace ccf const LedgerSecretPtr& current_ledger_secret) { // Submitted recovery shares are encrypted with the latest ledger secret. - crypto::GcmCipher encrypted_submitted_share(submitted_share.size()); + ccf::crypto::GcmCipher encrypted_submitted_share(submitted_share.size()); encrypted_submitted_share.hdr.set_random_iv(); @@ -296,7 +296,7 @@ namespace ccf const std::vector& encrypted_submitted_share, LedgerSecretPtr&& current_ledger_secret) { - crypto::GcmCipher encrypted_share; + ccf::crypto::GcmCipher encrypted_share; encrypted_share.deserialise(encrypted_submitted_share); std::vector decrypted_share; @@ -310,13 +310,14 @@ namespace ccf return decrypted_share; } - LedgerSecretWrappingKey combine_from_encrypted_submitted_shares(kv::Tx& tx) + LedgerSecretWrappingKey combine_from_encrypted_submitted_shares( + ccf::kv::Tx& tx) { auto encrypted_submitted_shares = tx.rw( Tables::ENCRYPTED_SUBMITTED_SHARES); auto config = tx.rw(Tables::CONFIGURATION); - std::vector new_shares = {}; + std::vector new_shares = {}; std::vector old_shares = {}; // Defensively allow shares in both formats for the time being, even if we // get a mix, and so long as we have enough of one or the other, attempt @@ -330,7 +331,7 @@ namespace ccf encrypted_share, ledger_secrets->get_latest(tx).second); switch (decrypted_share.size()) { - case crypto::sharing::Share::serialised_size: + case ccf::crypto::sharing::Share::serialised_size: { new_shares.emplace_back(decrypted_share); break; @@ -353,7 +354,7 @@ namespace ccf "is neither a new-style share of {} bytes nor an old-style " "share of {} bytes", decrypted_share.size(), - crypto::sharing::Share::serialised_size, + ccf::crypto::sharing::Share::serialised_size, SecretSharing::SHARE_LENGTH)); } } @@ -396,7 +397,7 @@ namespace ccf * * @param tx Store transaction object */ - void issue_recovery_shares(kv::Tx& tx) + void issue_recovery_shares(ccf::kv::Tx& tx) { auto [latest, penultimate] = ledger_secrets->get_latest_and_penultimate(tx); @@ -415,7 +416,8 @@ namespace ccf * derived from the hook at which the ledger secret is applied to the * store. */ - void issue_recovery_shares(kv::Tx& tx, LedgerSecretPtr new_ledger_secret) + void issue_recovery_shares( + ccf::kv::Tx& tx, LedgerSecretPtr new_ledger_secret) { set_recovery_shares_info( tx, new_ledger_secret, ledger_secrets->get_latest(tx)); @@ -427,13 +429,13 @@ namespace ccf * * @param tx Store transaction object */ - void shuffle_recovery_shares(kv::Tx& tx) + void shuffle_recovery_shares(ccf::kv::Tx& tx) { shuffle_recovery_shares(tx, ledger_secrets->get_latest(tx).second); } static std::optional get_encrypted_share( - kv::ReadOnlyTx& tx, const MemberId& member_id) + ccf::kv::ReadOnlyTx& tx, const MemberId& member_id) { auto recovery_shares_info = tx.ro(Tables::SHARES)->get(); @@ -453,7 +455,7 @@ namespace ccf } LedgerSecretsMap restore_recovery_shares_info( - kv::Tx& tx, + ccf::kv::Tx& tx, const RecoveredEncryptedLedgerSecrets& recovery_ledger_secrets) { // First, re-assemble the ledger secret wrapping key from the submitted @@ -526,7 +528,7 @@ namespace ccf } size_t submit_recovery_share( - kv::Tx& tx, + ccf::kv::Tx& tx, MemberId member_id, const std::vector& submitted_recovery_share) { @@ -547,7 +549,7 @@ namespace ccf return encrypted_submitted_shares->size(); } - static void clear_submitted_recovery_shares(kv::Tx& tx) + static void clear_submitted_recovery_shares(ccf::kv::Tx& tx) { auto encrypted_submitted_shares = tx.rw( Tables::ENCRYPTED_SUBMITTED_SHARES); diff --git a/src/node/snapshot_serdes.h b/src/node/snapshot_serdes.h index 1e9cb706a4f2..c768987043bb 100644 --- a/src/node/snapshot_serdes.h +++ b/src/node/snapshot_serdes.h @@ -18,17 +18,17 @@ namespace ccf struct StartupSnapshotInfo { std::vector raw; - kv::Version seqno; + ccf::kv::Version seqno; // Store used to verify a snapshot (either created fresh when a node joins // from a snapshot or points to the main store when recovering from a // snapshot) - std::shared_ptr store = nullptr; + std::shared_ptr store = nullptr; StartupSnapshotInfo( - const std::shared_ptr& store_, + const std::shared_ptr& store_, std::vector&& raw_, - kv::Version seqno_) : + ccf::kv::Version seqno_) : raw(std::move(raw_)), seqno(seqno_), store(store_) @@ -36,18 +36,19 @@ namespace ccf }; static void deserialise_snapshot( - const std::shared_ptr& store, + const std::shared_ptr& store, const std::vector& snapshot, - kv::ConsensusHookPtrs& hooks, - std::vector* view_history = nullptr, + ccf::kv::ConsensusHookPtrs& hooks, + std::vector* view_history = nullptr, bool public_only = false, std::optional> prev_service_identity = std::nullopt) { const auto* data = snapshot.data(); auto size = snapshot.size(); - auto tx_hdr = serialized::peek(data, size); - auto store_snapshot_size = sizeof(kv::SerialisedEntryHeader) + tx_hdr.size; + auto tx_hdr = serialized::peek(data, size); + auto store_snapshot_size = + sizeof(ccf::kv::SerialisedEntryHeader) + tx_hdr.size; auto receipt_data = data + store_snapshot_size; auto receipt_size = size - store_snapshot_size; @@ -67,7 +68,7 @@ namespace ccf } auto snapshot_digest = - crypto::Sha256Hash({snapshot.data(), store_snapshot_size}); + ccf::crypto::Sha256Hash({snapshot.data(), store_snapshot_size}); auto snapshot_digest_claim = receipt->leaf_components.claims_digest.value(); if (snapshot_digest != snapshot_digest_claim) { @@ -80,13 +81,13 @@ namespace ccf auto root = receipt->calculate_root(); auto raw_sig = receipt->signature; - auto v = crypto::make_unique_verifier(receipt->cert); + auto v = ccf::crypto::make_unique_verifier(receipt->cert); if (!v->verify_hash( root.h.data(), root.h.size(), receipt->signature.data(), receipt->signature.size(), - crypto::MDType::SHA256)) + ccf::crypto::MDType::SHA256)) { throw std::logic_error( "Signature verification failed for snapshot receipt"); @@ -94,7 +95,7 @@ namespace ccf if (prev_service_identity) { - crypto::Pem prev_pem(*prev_service_identity); + ccf::crypto::Pem prev_pem(*prev_service_identity); if (!v->verify_certificate( {&prev_pem}, {}, /* ignore_time */ @@ -114,7 +115,7 @@ namespace ccf auto rc = store->deserialise_snapshot( snapshot.data(), store_snapshot_size, hooks, view_history, public_only); - if (rc != kv::ApplyResult::PASS) + if (rc != ccf::kv::ApplyResult::PASS) { throw std::logic_error(fmt::format("Failed to apply snapshot: {}", rc)); } @@ -125,10 +126,10 @@ namespace ccf }; static std::unique_ptr initialise_from_snapshot( - const std::shared_ptr& store, + const std::shared_ptr& store, std::vector&& snapshot, - kv::ConsensusHookPtrs& hooks, - std::vector* view_history = nullptr, + ccf::kv::ConsensusHookPtrs& hooks, + std::vector* view_history = nullptr, bool public_only = false, std::optional> previous_service_identity = std::nullopt) @@ -148,11 +149,11 @@ namespace ccf const std::vector& sig, const std::vector& tree, const NodeId& node_id, - const crypto::Pem& node_cert, - kv::Version seqno, - const crypto::Sha256Hash& write_set_digest, + const ccf::crypto::Pem& node_cert, + ccf::kv::Version seqno, + const ccf::crypto::Sha256Hash& write_set_digest, const std::string& commit_evidence, - crypto::Sha256Hash&& claims_digest) + ccf::crypto::Sha256Hash&& claims_digest) { ccf::MerkleTreeHistory history(tree); auto proof = history.get_proof(seqno); diff --git a/src/node/snapshotter.h b/src/node/snapshotter.h index 9d8f1504e673..cabc8f371384 100644 --- a/src/node/snapshotter.h +++ b/src/node/snapshotter.h @@ -20,7 +20,7 @@ namespace ccf { class Snapshotter : public std::enable_shared_from_this, - public kv::AbstractSnapshotter + public ccf::kv::AbstractSnapshotter { private: static constexpr auto max_tx_interval = std::numeric_limits::max(); @@ -34,17 +34,17 @@ namespace ccf ccf::pal::Mutex lock; - std::shared_ptr store; + std::shared_ptr store; // Snapshots are never generated by default (e.g. during public recovery) size_t snapshot_tx_interval = max_tx_interval; struct SnapshotInfo { - kv::Version version; - crypto::Sha256Hash write_set_digest; + ccf::kv::Version version; + ccf::crypto::Sha256Hash write_set_digest; std::string commit_evidence; - crypto::Sha256Hash snapshot_digest; + ccf::crypto::Sha256Hash snapshot_digest; std::vector serialised_snapshot; // Prevents the receipt from being passed to the host (on commit) in case @@ -54,7 +54,7 @@ namespace ccf std::optional<::consensus::Index> evidence_idx = std::nullopt; std::optional node_id = std::nullopt; - std::optional node_cert = std::nullopt; + std::optional node_cert = std::nullopt; std::optional> sig = std::nullopt; std::optional> tree = std::nullopt; @@ -100,7 +100,7 @@ namespace ccf struct SnapshotMsg { std::shared_ptr self; - std::unique_ptr snapshot; + std::unique_ptr snapshot; uint32_t generation_count; }; @@ -111,7 +111,7 @@ namespace ccf } void snapshot_( - std::unique_ptr snapshot, + std::unique_ptr snapshot, uint32_t generation_count) { if (pending_snapshots.size() >= max_pending_snapshots_count) @@ -130,20 +130,20 @@ namespace ccf auto tx = store->create_tx(); auto evidence = tx.rw(Tables::SNAPSHOT_EVIDENCE); - auto snapshot_hash = crypto::Sha256Hash(serialised_snapshot); + auto snapshot_hash = ccf::crypto::Sha256Hash(serialised_snapshot); evidence->put({snapshot_hash, snapshot_version}); ccf::ClaimsDigest cd; cd.set(std::move(snapshot_hash)); - crypto::Sha256Hash ws_digest; + ccf::crypto::Sha256Hash ws_digest; std::string commit_evidence; auto capture_ws_digest_and_commit_evidence = [&ws_digest, &commit_evidence]( const std::vector& write_set, const std::string& commit_evidence_) { new (&ws_digest) - crypto::Sha256Hash({write_set.data(), write_set.size()}); + ccf::crypto::Sha256Hash({write_set.data(), write_set.size()}); commit_evidence = commit_evidence_; }; @@ -157,7 +157,7 @@ namespace ccf auto rc = tx.commit(cd, false, nullptr, capture_ws_digest_and_commit_evidence); - if (rc != kv::CommitResult::SUCCESS) + if (rc != ccf::kv::CommitResult::SUCCESS) { LOG_FAIL_FMT( "Could not commit snapshot evidence for seqno {}: {}", @@ -232,7 +232,7 @@ namespace ccf public: Snapshotter( ringbuffer::AbstractWriterFactory& writer_factory_, - std::shared_ptr& store_, + std::shared_ptr& store_, size_t snapshot_tx_interval_) : writer_factory(writer_factory_), store(store_), @@ -346,7 +346,7 @@ namespace ccf next_snapshot_indices.back().idx); bool forced = store->flag_enabled_unsafe( - kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); + ccf::kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); ::consensus::Index last_unforced_idx = last_snapshot_idx; for (auto it = next_snapshot_indices.rbegin(); @@ -367,7 +367,7 @@ namespace ccf LOG_TRACE_FMT( "{} {} as snapshot index", !due ? "Forced" : "Recorded", idx); store->unset_flag_unsafe( - kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); + ccf::kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); return due; } @@ -378,7 +378,7 @@ namespace ccf ::consensus::Index idx, const std::vector& sig, const NodeId& node_id, - const crypto::Pem& node_cert) + const ccf::crypto::Pem& node_cert) { std::lock_guard guard(lock); @@ -464,7 +464,7 @@ namespace ccf // serialisation on another thread (round-robin). Otherwise, only record // that a snapshot was generated. - kv::ScopedStoreMapsLock maps_lock(store); + ccf::kv::ScopedStoreMapsLock maps_lock(store); std::lock_guard guard(lock); update_indices(idx); diff --git a/src/node/test/channels.cpp b/src/node/test/channels.cpp index 8b3d6e793cca..462b1fa9ec06 100644 --- a/src/node/test/channels.cpp +++ b/src/node/test/channels.cpp @@ -69,7 +69,7 @@ using MsgType = std::array; static const NodeId nid1 = std::string("nid1"); static const NodeId nid2 = std::string("nid2"); -static constexpr auto default_curve = crypto::CurveID::SECP384R1; +static constexpr auto default_curve = ccf::crypto::CurveID::SECP384R1; static std::pair make_validity_pair(bool expired) { @@ -88,25 +88,27 @@ static std::pair make_validity_pair(bool expired) } } -static crypto::Pem generate_self_signed_cert( - const crypto::KeyPairPtr& kp, const std::string& name, bool expired = false) +static ccf::crypto::Pem generate_self_signed_cert( + const ccf::crypto::KeyPairPtr& kp, + const std::string& name, + bool expired = false) { const auto [valid_from, validity_days] = make_validity_pair(expired); - return crypto::create_self_signed_cert( + return ccf::crypto::create_self_signed_cert( kp, name, {}, valid_from, validity_days); } -static crypto::Pem generate_endorsed_cert( - const crypto::KeyPairPtr& kp, +static ccf::crypto::Pem generate_endorsed_cert( + const ccf::crypto::KeyPairPtr& kp, const std::string& name, - const crypto::KeyPairPtr& issuer_kp, - const crypto::Pem& issuer_cert, + const ccf::crypto::KeyPairPtr& issuer_kp, + const ccf::crypto::Pem& issuer_cert, bool expired = false) { const auto [valid_from, validity_days] = make_validity_pair(expired); - return crypto::create_endorsed_cert( + return ccf::crypto::create_endorsed_cert( kp, name, {}, @@ -233,20 +235,20 @@ NodeOutboundMsg get_first( TEST_CASE_FIXTURE(IORingbuffersFixture, "Client/Server key exchange") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node2", network_kp, service_cert); - auto v = crypto::make_verifier(channel1_cert); + auto v = ccf::crypto::make_verifier(channel1_cert); REQUIRE(v->verify_certificate({&service_cert})); - v = crypto::make_verifier(channel2_cert); + v = ccf::crypto::make_verifier(channel2_cert); REQUIRE(v->verify_certificate({&service_cert})); REQUIRE(!make_verifier(channel2_cert)->is_self_signed()); @@ -443,14 +445,14 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Client/Server key exchange") TEST_CASE_FIXTURE(IORingbuffersFixture, "Replay and out-of-order") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node2", network_kp, service_cert); @@ -619,10 +621,10 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Replay and out-of-order") TEST_CASE_FIXTURE(IORingbuffersFixture, "Host connections") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel_kp = crypto::make_key_pair(default_curve); + auto channel_kp = ccf::crypto::make_key_pair(default_curve); auto channel_cert = generate_endorsed_cert(channel_kp, "CN=Node", network_kp, service_cert); @@ -668,14 +670,14 @@ static std::vector> get_all_msgs( TEST_CASE_FIXTURE(IORingbuffersFixture, "Concurrent key exchange init") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node1", network_kp, service_cert); @@ -762,47 +764,47 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Concurrent key exchange init") struct CurveChoices { - crypto::CurveID network; - crypto::CurveID node_1; - crypto::CurveID node_2; + ccf::crypto::CurveID network; + ccf::crypto::CurveID node_1; + ccf::crypto::CurveID node_2; }; TEST_CASE_FIXTURE(IORingbuffersFixture, "Full NodeToNode test") { constexpr auto all_256 = CurveChoices{ - crypto::CurveID::SECP256R1, - crypto::CurveID::SECP256R1, - crypto::CurveID::SECP256R1}; + ccf::crypto::CurveID::SECP256R1, + ccf::crypto::CurveID::SECP256R1, + ccf::crypto::CurveID::SECP256R1}; constexpr auto all_384 = CurveChoices{ - crypto::CurveID::SECP384R1, - crypto::CurveID::SECP384R1, - crypto::CurveID::SECP384R1}; + ccf::crypto::CurveID::SECP384R1, + ccf::crypto::CurveID::SECP384R1, + ccf::crypto::CurveID::SECP384R1}; // One backup on a different curve constexpr auto mixed_0 = CurveChoices{ - crypto::CurveID::SECP256R1, - crypto::CurveID::SECP256R1, - crypto::CurveID::SECP384R1}; + ccf::crypto::CurveID::SECP256R1, + ccf::crypto::CurveID::SECP256R1, + ccf::crypto::CurveID::SECP384R1}; // Both backups on a different curve constexpr auto mixed_1 = CurveChoices{ - crypto::CurveID::SECP384R1, - crypto::CurveID::SECP256R1, - crypto::CurveID::SECP256R1}; + ccf::crypto::CurveID::SECP384R1, + ccf::crypto::CurveID::SECP256R1, + ccf::crypto::CurveID::SECP256R1}; size_t i = 0; for (const auto& curves : {all_256, all_384, mixed_0, mixed_1}) { LOG_DEBUG_FMT("Iteration: {}", i++); - auto network_kp = crypto::make_key_pair(curves.network); + auto network_kp = ccf::crypto::make_key_pair(curves.network); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); auto ni1 = std::string("N1"); - auto channel1_kp = crypto::make_key_pair(curves.node_1); + auto channel1_kp = ccf::crypto::make_key_pair(curves.node_1); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); auto ni2 = std::string("N2"); - auto channel2_kp = crypto::make_key_pair(curves.node_2); + auto channel2_kp = ccf::crypto::make_key_pair(curves.node_2); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node2", network_kp, service_cert); @@ -888,14 +890,14 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Full NodeToNode test") TEST_CASE_FIXTURE(IORingbuffersFixture, "Interrupted key exchange") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node1", network_kp, service_cert); @@ -1039,14 +1041,14 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Stuttering handshake") MsgType aad; aad.fill(0x10); - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node1", network_kp, service_cert); @@ -1112,9 +1114,9 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Stuttering handshake") TEST_CASE_FIXTURE(IORingbuffersFixture, "Expired certs") { - auto network_kp = crypto::make_key_pair(default_curve); - auto channel1_kp = crypto::make_key_pair(default_curve); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=MyNetwork"); auto channel1_cert = @@ -1171,14 +1173,14 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Expired certs") TEST_CASE_FIXTURE(IORingbuffersFixture, "Robust key exchange") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node1", network_kp, service_cert); @@ -1404,7 +1406,7 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Robust key exchange") // key rotation exchanges happening during the sequence TEST_CASE_FIXTURE(IORingbuffersFixture, "Key rotation") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); MsgType aad; @@ -1427,7 +1429,7 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Key rotation") ringbuffer::AbstractWriterFactory& writer_factory, QueueWithLock& send_queue, ReceivedMessages& received_results) { - auto kp = crypto::make_key_pair(default_curve); + auto kp = ccf::crypto::make_key_pair(default_curve); auto cert = generate_endorsed_cert( kp, fmt::format("CN={}", my_node_id), network_kp, service_cert); @@ -1591,14 +1593,14 @@ TEST_CASE_FIXTURE(IORingbuffersFixture, "Key rotation") TEST_CASE_FIXTURE(IORingbuffersFixture, "Timeout idle channels") { - auto network_kp = crypto::make_key_pair(default_curve); + auto network_kp = ccf::crypto::make_key_pair(default_curve); auto service_cert = generate_self_signed_cert(network_kp, "CN=Network"); - auto channel1_kp = crypto::make_key_pair(default_curve); + auto channel1_kp = ccf::crypto::make_key_pair(default_curve); auto channel1_cert = generate_endorsed_cert(channel1_kp, "CN=Node1", network_kp, service_cert); - auto channel2_kp = crypto::make_key_pair(default_curve); + auto channel2_kp = ccf::crypto::make_key_pair(default_curve); auto channel2_cert = generate_endorsed_cert(channel2_kp, "CN=Node2", network_kp, service_cert); diff --git a/src/node/test/encryptor.cpp b/src/node/test/encryptor.cpp index 4a26eb338d08..d6a6fc0b05b0 100644 --- a/src/node/test/encryptor.cpp +++ b/src/node/test/encryptor.cpp @@ -16,29 +16,29 @@ #include #include -kv::ConsensusHookPtrs hooks; -using StringString = kv::Map; +ccf::kv::ConsensusHookPtrs hooks; +using StringString = ccf::kv::Map; -void commit_one(kv::Store& store, StringString& map) +void commit_one(ccf::kv::Store& store, StringString& map) { auto tx = store.create_tx(); auto m = tx.rw(map); m->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } bool encrypt_round_trip( ccf::NodeEncryptor& encryptor, std::vector& plain, - kv::Version version) + ccf::kv::Version version) { std::vector aad; std::vector header; std::vector cipher(plain.size()); std::vector decrypted(plain.size()); - kv::Term term = 1; - kv::Term ret_term = 0; + ccf::kv::Term term = 1; + ccf::kv::Term ret_term = 0; encryptor.encrypt(plain, aad, header, cipher, {term, version}); encryptor.decrypt(cipher, aad, header, decrypted, version, ret_term); @@ -114,8 +114,8 @@ TEST_CASE("Subsequent ciphers from same plaintext are different") std::vector serialised_header; std::vector serialised_header2; std::vector additional_data; // No additional data - kv::Version version = 10; - kv::Term term = 1; + ccf::kv::Version version = 10; + ccf::kv::Term term = 1; encryptor.encrypt( plain, additional_data, serialised_header, cipher, {term, version}); @@ -141,8 +141,8 @@ TEST_CASE("Ciphers at same seqno with different terms are different") std::vector serialised_header; std::vector serialised_header2; std::vector additional_data; // No additional data - kv::Version version = 10; - kv::Term term = 1; + ccf::kv::Version version = 10; + ccf::kv::Term term = 1; encryptor.encrypt( plain, additional_data, serialised_header, cipher, {term, version}); @@ -167,8 +167,8 @@ TEST_CASE("Ciphers at same seqno/term with and without snapshot are different") std::vector serialised_header; std::vector serialised_header2; std::vector additional_data; // No additional data - kv::Version version = 10; - kv::Term term = 1; + ccf::kv::Version version = 10; + ccf::kv::Term term = 1; encryptor.encrypt( plain, @@ -176,7 +176,7 @@ TEST_CASE("Ciphers at same seqno/term with and without snapshot are different") serialised_header, cipher, {term, version}, - kv::EntryType::Snapshot); + ccf::kv::EntryType::Snapshot); encryptor.encrypt( plain, @@ -184,7 +184,7 @@ TEST_CASE("Ciphers at same seqno/term with and without snapshot are different") serialised_header2, cipher2, {term, version}, - kv::EntryType::WriteSet); + ccf::kv::EntryType::WriteSet); // Ciphers are different because IV is different REQUIRE(cipher != cipher2); @@ -201,8 +201,8 @@ TEST_CASE("Additional data") std::vector cipher; std::vector serialised_header; std::vector additional_data(256, 0x10); - kv::Version version = 10; - kv::Term term = 1; + ccf::kv::Version version = 10; + ccf::kv::Term term = 1; // Encrypting plain at version 10 encryptor.encrypt( @@ -236,10 +236,10 @@ TEST_CASE("Additional data") TEST_CASE("KV encryption/decryption") { - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); StringString map("map"); - kv::Store primary_store; - kv::Store backup_store; + ccf::kv::Store primary_store; + ccf::kv::Store backup_store; std::shared_ptr primary_ledger_secrets; std::shared_ptr backup_ledger_secrets; @@ -270,7 +270,7 @@ TEST_CASE("KV encryption/decryption") commit_one(primary_store, map); REQUIRE( backup_store.deserialize(*consensus->get_latest_data())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); } INFO("Rekeys"); @@ -295,17 +295,17 @@ TEST_CASE("KV encryption/decryption") REQUIRE( backup_store.deserialize(*consensus->get_latest_data())->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); } } } TEST_CASE("Backup catchup from many ledger secrets") { - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); StringString map("map"); - kv::Store primary_store; - kv::Store backup_store; + ccf::kv::Store primary_store; + ccf::kv::Store backup_store; std::shared_ptr primary_ledger_secrets; std::shared_ptr backup_ledger_secrets; @@ -352,7 +352,7 @@ TEST_CASE("Backup catchup from many ledger secrets") { REQUIRE( backup_store.deserialize(*std::get<1>(next_entry.value()))->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); auto tx_id = backup_store.current_txid(); tx_id.version--; @@ -362,7 +362,7 @@ TEST_CASE("Backup catchup from many ledger secrets") REQUIRE( backup_store.deserialize(*std::get<1>(next_entry.value()))->apply() == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); next_entry = consensus->pop_oldest_entry(); } @@ -371,10 +371,10 @@ TEST_CASE("Backup catchup from many ledger secrets") TEST_CASE("KV integrity verification") { - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); StringString map("map"); - kv::Store primary_store; - kv::Store backup_store; + ccf::kv::Store primary_store; + ccf::kv::Store backup_store; auto ledger_secrets = std::make_shared(); ledger_secrets->init(); @@ -400,14 +400,14 @@ TEST_CASE("KV integrity verification") auto r = backup_store.deserialize(latest_data.value()); auto rr = r->apply(); - REQUIRE(rr == kv::ApplyResult::FAIL); + REQUIRE(rr == ccf::kv::ApplyResult::FAIL); } TEST_CASE("Encryptor rollback") { StringString map("map"); constexpr auto store_term = 2; - kv::Store store; + ccf::kv::Store store; store.initialise_term(2); auto ledger_secrets = std::make_shared(); @@ -448,12 +448,12 @@ TEST_CASE("Encryptor rollback") int main(int argc, char** argv) { - logger::config::default_init(); - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/test/endorsements.cpp b/src/node/test/endorsements.cpp index de4f3524d94b..743d615e4b96 100644 --- a/src/node/test/endorsements.cpp +++ b/src/node/test/endorsements.cpp @@ -67,12 +67,12 @@ TEST_CASE("Check ECDSA Test endorsement") int main(int argc, char** argv) { - logger::config::default_init(); - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/test/historical_queries.cpp b/src/node/test/historical_queries.cpp index a077b0bcca55..61f2584dc4ec 100644 --- a/src/node/test/historical_queries.cpp +++ b/src/node/test/historical_queries.cpp @@ -25,33 +25,33 @@ std::unique_ptr threading::ThreadMessaging::singleton = nullptr; -using NumToString = kv::Map; +using NumToString = ccf::kv::Map; constexpr size_t certificate_validity_period_days = 365; using namespace std::literals; auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); -auto valid_to = crypto::compute_cert_valid_to_string( +auto valid_to = ccf::crypto::compute_cert_valid_to_string( valid_from, certificate_validity_period_days); struct TestState { - std::shared_ptr kv_store = nullptr; + std::shared_ptr kv_store = nullptr; std::shared_ptr ledger_secrets = nullptr; - crypto::KeyPairPtr node_kp = nullptr; + ccf::crypto::KeyPairPtr node_kp = nullptr; }; TestState create_and_init_state(bool initialise_ledger_rekey = true) { TestState ts; - ts.kv_store = std::make_shared(); - ts.kv_store->set_consensus(std::make_shared()); - auto encryptor = std::make_shared(); + ts.kv_store = std::make_shared(); + ts.kv_store->set_consensus(std::make_shared()); + auto encryptor = std::make_shared(); ts.kv_store->set_encryptor(encryptor); - ts.node_kp = crypto::make_key_pair(); + ts.node_kp = ccf::crypto::make_key_pair(); // Make history to produce signatures const ccf::NodeId node_id = std::string("node_id"); @@ -71,7 +71,7 @@ TestState create_and_init_state(bool initialise_ledger_rekey = true) ni.cert = ts.node_kp->self_sign("CN=Test node", valid_from, valid_to); ni.status = ccf::NodeStatus::TRUSTED; nodes->put(node_id, ni); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } // Create ledger secrets to test decrypting entries across rekeys @@ -96,21 +96,21 @@ TestState create_and_init_state(bool initialise_ledger_rekey = true) auto member_public_encryption_keys = tx.rw( ccf::Tables::MEMBER_ENCRYPTION_PUBLIC_KEYS); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto cert = kp->self_sign("CN=member", valid_from, valid_to); auto member_id = - crypto::Sha256Hash(crypto::cert_pem_to_der(cert)).hex_str(); + ccf::crypto::Sha256Hash(ccf::crypto::cert_pem_to_der(cert)).hex_str(); member_info->put(member_id, {ccf::MemberStatus::ACTIVE}); member_public_encryption_keys->put( - member_id, crypto::make_rsa_key_pair()->public_key_pem()); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + member_id, ccf::crypto::make_rsa_key_pair()->public_key_pem()); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } return ts; } -kv::Version write_transactions(kv::Store& kv_store, size_t tx_count) +ccf::kv::Version write_transactions(ccf::kv::Store& kv_store, size_t tx_count) { const auto begin = kv_store.current_version(); const auto end = begin + tx_count; @@ -123,22 +123,22 @@ kv::Version write_transactions(kv::Store& kv_store, size_t tx_count) public_map->put(i, s); private_map->put(i, s); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } REQUIRE(kv_store.current_version() == end); return kv_store.current_version(); } -kv::Version write_transactions_and_signature( - kv::Store& kv_store, size_t tx_count) +ccf::kv::Version write_transactions_and_signature( + ccf::kv::Store& kv_store, size_t tx_count) { write_transactions(kv_store, tx_count); kv_store.get_history()->emit_signature(); auto consensus = - dynamic_cast(kv_store.get_consensus().get()); + dynamic_cast(kv_store.get_consensus().get()); REQUIRE(consensus != nullptr); REQUIRE(consensus->get_committed_seqno() == kv_store.current_version()); @@ -149,8 +149,8 @@ kv::Version write_transactions_and_signature( return kv_store.current_version(); } -kv::Version rekey( - kv::Store& kv_store, +ccf::kv::Version rekey( + ccf::kv::Store& kv_store, const std::shared_ptr& ledger_secrets) { ccf::ShareManager share_manager(ledger_secrets); @@ -158,7 +158,7 @@ kv::Version rekey( auto tx = kv_store.create_tx(); auto new_ledger_secret = ccf::make_ledger_secret(); share_manager.issue_recovery_shares(tx, new_ledger_secret); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto tx_version = tx.commit_version(); @@ -170,7 +170,8 @@ kv::Version rekey( return tx_version; } -void validate_business_transaction(kv::ReadOnlyStorePtr store, ccf::SeqNo seqno) +void validate_business_transaction( + ccf::kv::ReadOnlyStorePtr store, ccf::SeqNo seqno) { REQUIRE(store != nullptr); @@ -211,9 +212,9 @@ void validate_business_transaction( } std::map> construct_host_ledger( - std::shared_ptr c) + std::shared_ptr c) { - auto consensus = dynamic_cast(c.get()); + auto consensus = dynamic_cast(c.get()); REQUIRE(consensus != nullptr); INFO("Rebuild ledger as seen by host"); @@ -238,13 +239,26 @@ std::map> construct_host_ledger( return ledger; } +size_t get_cache_limit_for_entries( + const std::vector>& entries) +{ + return ccf::historical::soft_to_raw_ratio * + std::accumulate( + entries.begin(), + entries.end(), + 0ll, + [&](size_t prev, const std::vector& entry) { + return prev + entry.size(); + }); +} + TEST_CASE("StateCache point queries") { auto state = create_and_init_state(); auto& kv_store = *state.kv_store; - kv::Version low_signature_transaction; - kv::Version high_signature_transaction; + ccf::kv::Version low_signature_transaction; + ccf::kv::Version high_signature_transaction; { INFO("Build some interesting state in the store"); @@ -516,7 +530,7 @@ TEST_CASE("StateCache get store vs get state") auto state = create_and_init_state(); auto& kv_store = *state.kv_store; - kv::Version signature_transaction; + ccf::kv::Version signature_transaction; { INFO("Build some interesting state in the store"); @@ -727,7 +741,7 @@ TEST_CASE("StateCache range queries") auto state = create_and_init_state(); auto& kv_store = *state.kv_store; - std::vector signature_versions; + std::vector signature_versions; const auto begin_seqno = kv_store.current_version() + 1; @@ -751,7 +765,7 @@ TEST_CASE("StateCache range queries") return accepted; }; - auto signing_version = [&signature_versions](kv::Version seqno) { + auto signing_version = [&signature_versions](ccf::kv::Version seqno) { const auto begin = signature_versions.begin(); const auto end = signature_versions.end(); @@ -769,8 +783,8 @@ TEST_CASE("StateCache range queries") std::random_device rd; std::mt19937 g(rd()); auto fetch_and_validate_range = [&]( - kv::Version range_start, - kv::Version range_end) { + ccf::kv::Version range_start, + ccf::kv::Version range_end) { constexpr auto this_handle = 42; { auto stores = cache.get_store_range(this_handle, range_start, range_end); @@ -855,7 +869,7 @@ TEST_CASE("Incremental progress") auto state = create_and_init_state(); auto& kv_store = *state.kv_store; - std::vector signature_versions; + std::vector signature_versions; const auto begin_seqno = kv_store.current_version() + 1; @@ -950,12 +964,140 @@ TEST_CASE("Incremental progress") } } +TEST_CASE("StateCache soft zero limit with increasing") +{ + // Try get two states. Shouldn't be able to retrieve anything with 0 cache + // limit. After increasing to the size of first state only that one is + // available, but later overwritten by the second one, and finally all are + // evicted after setting again to 0. + // + // ! DISCLAIMER ! If you change this bear in mind that each attempt to get the + // store promotes the handle, and so requests eviction order changes, + // therefore this test in particular is quite fragile. + + auto state = create_and_init_state(); + auto& kv_store = *state.kv_store; + + auto seq_low = write_transactions_and_signature(kv_store, 1); + auto seq_high = write_transactions_and_signature(kv_store, 1); + REQUIRE(kv_store.current_version() == seq_high); + + auto ledger = construct_host_ledger(state.kv_store->get_consensus()); + REQUIRE(ledger.size() == seq_high); + + auto stub_writer = std::make_shared(); + ccf::historical::StateCache cache( + kv_store, state.ledger_secrets, stub_writer); + + cache.set_soft_cache_limit(0); + + REQUIRE(!cache.get_state_at(0, seq_low)); + cache.tick(std::chrono::milliseconds(100)); + + cache.handle_ledger_entry(seq_low, ledger.at(seq_low)); + cache.tick(std::chrono::milliseconds(100)); + + // Ledger entry fecthed and instantly removed because of the parent request + // eviction due to zero cache limit. + REQUIRE(!cache.get_state_at(0, seq_low)); + + cache.set_soft_cache_limit( + get_cache_limit_for_entries({ledger.at(seq_low), ledger.at(seq_high)}) - 1); + + cache.handle_ledger_entry(seq_low, ledger.at(seq_low)); + cache.tick(std::chrono::milliseconds(100)); + + REQUIRE(cache.get_state_at(0, seq_low)); + REQUIRE(!cache.get_state_at(1, seq_high)); + + cache.tick(std::chrono::milliseconds(100)); + + cache.handle_ledger_entry(seq_high, ledger.at(seq_high)); + + // Both available because tick hasn't been called yet. + REQUIRE(cache.get_state_at(0, seq_low)); + REQUIRE(cache.get_state_at(1, seq_high)); + + cache.tick(std::chrono::milliseconds(100)); + + REQUIRE(!cache.get_state_at(0, seq_low)); + REQUIRE(cache.get_state_at(1, seq_high)); + + cache.set_soft_cache_limit(0); + cache.tick(std::chrono::milliseconds(100)); + + REQUIRE(!cache.get_state_at(0, seq_low)); + REQUIRE(!cache.get_state_at(1, seq_high)); +} + +TEST_CASE("StateCache dropping state explicitly") +{ + // Adding two states to the cache (limit is hit). Drop state2 and add state3. + // State1 should remain available, as well as state3, state2 was force-evicted + // from LRU because it's been dropped explicitly. + // + // ! DISCLAIMER ! If you change this bear in mind that each attempt to get the + // store promotes the handle, and so requests eviction order changes, + // therefore this test in particular is quite fragile. + + auto state = create_and_init_state(); + auto& kv_store = *state.kv_store; + + auto seq_low = write_transactions_and_signature(kv_store, 1); + auto seq_mid = write_transactions_and_signature(kv_store, 1); + auto seq_high = write_transactions_and_signature(kv_store, 1); + REQUIRE(kv_store.current_version() == seq_high); + + auto ledger = construct_host_ledger(state.kv_store->get_consensus()); + REQUIRE(ledger.size() == seq_high); + + auto stub_writer = std::make_shared(); + ccf::historical::StateCache cache( + kv_store, state.ledger_secrets, stub_writer); + + REQUIRE(!cache.get_state_at(0, seq_low)); + REQUIRE(!cache.get_state_at(1, seq_mid)); + + cache.tick(std::chrono::milliseconds(100)); + + cache.handle_ledger_entry(seq_low, ledger.at(seq_low)); + cache.handle_ledger_entry(seq_mid, ledger.at(seq_mid)); + + REQUIRE(cache.get_state_at(0, seq_low)); + REQUIRE(cache.get_state_at(1, seq_mid)); + + cache.set_soft_cache_limit( + get_cache_limit_for_entries( + {ledger.at(seq_low), ledger.at(seq_mid), ledger.at(seq_high)}) - + 1); + + cache.tick(std::chrono::milliseconds(100)); + + REQUIRE(!cache.get_state_at(2, seq_high)); + cache.tick(std::chrono::milliseconds(100)); + + // Ledger entries not provided yet, so previous are not evicted. + REQUIRE(cache.get_state_at(0, seq_low)); + REQUIRE(cache.get_state_at(1, seq_mid)); + REQUIRE(!cache.get_state_at(2, seq_high)); + + cache.drop_cached_states(1); + cache.tick(std::chrono::milliseconds(100)); + + cache.handle_ledger_entry(seq_high, ledger.at(seq_high)); + cache.tick(std::chrono::milliseconds(100)); + + REQUIRE(cache.get_state_at(0, seq_low)); + REQUIRE(!cache.get_state_at(1, seq_mid)); + REQUIRE(cache.get_state_at(2, seq_high)); +} + TEST_CASE("StateCache sparse queries") { auto state = create_and_init_state(); auto& kv_store = *state.kv_store; - std::vector signature_versions; + std::vector signature_versions; const auto begin_seqno = kv_store.current_version() + 1; @@ -979,7 +1121,7 @@ TEST_CASE("StateCache sparse queries") return accepted; }; - auto signing_version = [&signature_versions](kv::Version seqno) { + auto signing_version = [&signature_versions](ccf::kv::Version seqno) { const auto begin = signature_versions.begin(); const auto end = signature_versions.end(); @@ -1088,7 +1230,7 @@ TEST_CASE("StateCache concurrent access") auto& kv_store = *state.kv_store; const auto default_handle = 0; - std::vector signature_versions; + std::vector signature_versions; const auto begin_seqno = kv_store.current_version() + 1; @@ -1112,7 +1254,7 @@ TEST_CASE("StateCache concurrent access") std::atomic finished = false; std::thread host_thread([&]() { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto ledger = construct_host_ledger(state.kv_store->get_consensus()); size_t last_handled_write = 0; @@ -1163,7 +1305,7 @@ TEST_CASE("StateCache concurrent access") cache.tick(std::chrono::milliseconds(100)); std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }); constexpr auto per_thread_queries = 30; @@ -1219,7 +1361,7 @@ TEST_CASE("StateCache concurrent access") }; auto validate_all_stores = - [&](const std::vector& stores) { + [&](const std::vector& stores) { for (auto& store : stores) { REQUIRE(store != nullptr); @@ -1252,7 +1394,7 @@ TEST_CASE("StateCache concurrent access") auto query_random_point_store = [&](ccf::SeqNo target_seqno, size_t handle, const auto& error_printer) { - kv::ReadOnlyStorePtr store; + ccf::kv::ReadOnlyStorePtr store; auto fetch_result = [&]() { store = cache.get_store_at(handle, target_seqno); }; @@ -1279,7 +1421,7 @@ TEST_CASE("StateCache concurrent access") ccf::SeqNo range_end, size_t handle, const auto& error_printer) { - std::vector stores; + std::vector stores; auto fetch_result = [&]() { stores = cache.get_store_range(handle, range_start, range_end); }; @@ -1295,7 +1437,7 @@ TEST_CASE("StateCache concurrent access") size_t handle, const auto& error_printer) { std::vector states; - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto fetch_result = [&]() { states = cache.get_state_range(handle, range_start, range_end); }; @@ -1303,14 +1445,14 @@ TEST_CASE("StateCache concurrent access") REQUIRE(fetch_until_timeout(fetch_result, check_result, error_printer)); REQUIRE(states.size() == range_end - range_start + 1); validate_all_states(states); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; auto query_random_sparse_set_stores = [&]( const ccf::SeqNoCollection& seqnos, size_t handle, const auto& error_printer) { - std::vector stores; + std::vector stores; auto fetch_result = [&]() { stores = cache.get_stores_for(handle, seqnos); }; @@ -1325,7 +1467,7 @@ TEST_CASE("StateCache concurrent access") size_t handle, const auto& error_printer) { std::vector states; - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto fetch_result = [&]() { states = cache.get_states_for(handle, seqnos); }; @@ -1333,7 +1475,7 @@ TEST_CASE("StateCache concurrent access") REQUIRE(fetch_until_timeout(fetch_result, check_result, error_printer)); REQUIRE(states.size() == seqnos.size()); validate_all_states(states); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); }; auto run_n_queries = [&](size_t handle) { @@ -1591,7 +1733,7 @@ TEST_CASE("StateCache concurrent access") TEST_CASE("Recover historical ledger secrets") { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); auto state = create_and_init_state(); auto& kv_store = *state.kv_store; @@ -1708,17 +1850,17 @@ TEST_CASE("Recover historical ledger secrets") validate_business_transaction(historical_state, first_seqno); } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } int main(int argc, char** argv) { threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/test/history.cpp b/src/node/test/history.cpp index dac4199a69c6..2a367bbfe222 100644 --- a/src/node/test/history.cpp +++ b/src/node/test/history.cpp @@ -21,22 +21,22 @@ std::unique_ptr threading::ThreadMessaging::singleton = nullptr; -using MapT = kv::Map; +using MapT = ccf::kv::Map; -class DummyConsensus : public kv::test::StubConsensus +class DummyConsensus : public ccf::kv::test::StubConsensus { public: - kv::Store* store; + ccf::kv::Store* store; - DummyConsensus(kv::Store* store_) : store(store_) {} + DummyConsensus(ccf::kv::Store* store_) : store(store_) {} - bool replicate(const kv::BatchVector& entries, ccf::View view) override + bool replicate(const ccf::kv::BatchVector& entries, ccf::View view) override { if (store) { REQUIRE(entries.size() == 1); return store->deserialize(*std::get<1>(entries[0]))->apply() != - kv::ApplyResult::FAIL; + ccf::kv::ApplyResult::FAIL; } return true; } @@ -51,39 +51,39 @@ class DummyConsensus : public kv::test::StubConsensus return 0; } - std::optional primary() override + std::optional primary() override { - return kv::test::FirstBackupNodeId; + return ccf::kv::test::FirstBackupNodeId; } - kv::NodeId id() override + ccf::kv::NodeId id() override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } }; TEST_CASE("Check signature verification") { - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); const auto self_signed = make_self_signed_cert(kp); - kv::Store primary_store; + ccf::kv::Store primary_store; primary_store.set_encryptor(encryptor); constexpr auto store_term = 2; - std::shared_ptr primary_history = + std::shared_ptr primary_history = std::make_shared( - primary_store, kv::test::PrimaryNodeId, *kp); + primary_store, ccf::kv::test::PrimaryNodeId, *kp); primary_history->set_endorsed_certificate(self_signed); primary_store.set_history(primary_history); primary_store.initialise_term(store_term); - kv::Store backup_store; + ccf::kv::Store backup_store; backup_store.set_encryptor(encryptor); - std::shared_ptr backup_history = + std::shared_ptr backup_history = std::make_shared( - backup_store, kv::test::FirstBackupNodeId, *kp); + backup_store, ccf::kv::test::FirstBackupNodeId, *kp); backup_history->set_endorsed_certificate(self_signed); backup_store.set_history(backup_history); backup_store.initialise_term(store_term); @@ -91,11 +91,11 @@ TEST_CASE("Check signature verification") ccf::Nodes nodes(ccf::Tables::NODES); ccf::Signatures signatures(ccf::Tables::SIGNATURES); - std::shared_ptr consensus = + std::shared_ptr consensus = std::make_shared(&backup_store); primary_store.set_consensus(consensus); - std::shared_ptr null_consensus = + std::shared_ptr null_consensus = std::make_shared(nullptr); backup_store.set_consensus(null_consensus); @@ -106,8 +106,8 @@ TEST_CASE("Check signature verification") ccf::NodeInfo ni; ni.encryption_pub_key = kp->public_key_pem(); ni.cert = self_signed; - tx->put(kv::test::PrimaryNodeId, ni); - REQUIRE(txs.commit() == kv::CommitResult::SUCCESS); + tx->put(ccf::kv::test::PrimaryNodeId, ni); + REQUIRE(txs.commit() == ccf::kv::CommitResult::SUCCESS); } INFO("Issue signature, and verify successfully on backup"); @@ -120,34 +120,34 @@ TEST_CASE("Check signature verification") { auto txs = primary_store.create_tx(); auto sigs = txs.rw(signatures); - ccf::PrimarySignature bogus(kv::test::PrimaryNodeId, 0); + ccf::PrimarySignature bogus(ccf::kv::test::PrimaryNodeId, 0); bogus.sig = std::vector(256, 1); sigs->put(bogus); - REQUIRE(txs.commit() == kv::CommitResult::FAIL_NO_REPLICATE); + REQUIRE(txs.commit() == ccf::kv::CommitResult::FAIL_NO_REPLICATE); } } TEST_CASE("Check signing works across rollback") { - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); const auto self_signed = make_self_signed_cert(kp); - kv::Store primary_store; + ccf::kv::Store primary_store; primary_store.set_encryptor(encryptor); constexpr auto store_term = 2; - std::shared_ptr primary_history = + std::shared_ptr primary_history = std::make_shared( - primary_store, kv::test::PrimaryNodeId, *kp); + primary_store, ccf::kv::test::PrimaryNodeId, *kp); primary_history->set_endorsed_certificate(self_signed); primary_store.set_history(primary_history); primary_store.initialise_term(store_term); - kv::Store backup_store; - std::shared_ptr backup_history = + ccf::kv::Store backup_store; + std::shared_ptr backup_history = std::make_shared( - backup_store, kv::test::FirstBackupNodeId, *kp); + backup_store, ccf::kv::test::FirstBackupNodeId, *kp); backup_history->set_endorsed_certificate(self_signed); backup_store.set_history(backup_history); backup_store.set_encryptor(encryptor); @@ -155,10 +155,10 @@ TEST_CASE("Check signing works across rollback") ccf::Nodes nodes(ccf::Tables::NODES); - std::shared_ptr consensus = + std::shared_ptr consensus = std::make_shared(&backup_store); primary_store.set_consensus(consensus); - std::shared_ptr null_consensus = + std::shared_ptr null_consensus = std::make_shared(nullptr); backup_store.set_consensus(null_consensus); @@ -169,8 +169,8 @@ TEST_CASE("Check signing works across rollback") ccf::NodeInfo ni; ni.encryption_pub_key = kp->public_key_pem(); ni.cert = self_signed; - tx->put(kv::test::PrimaryNodeId, ni); - REQUIRE(txs.commit() == kv::CommitResult::SUCCESS); + tx->put(ccf::kv::test::PrimaryNodeId, ni); + REQUIRE(txs.commit() == ccf::kv::CommitResult::SUCCESS); } auto v1_proof = primary_history->get_proof(primary_store.current_version()); @@ -180,8 +180,8 @@ TEST_CASE("Check signing works across rollback") auto txs = primary_store.create_tx(); auto tx = txs.rw(nodes); ccf::NodeInfo ni; - tx->put(kv::test::FirstBackupNodeId, ni); - REQUIRE(txs.commit() == kv::CommitResult::SUCCESS); + tx->put(ccf::kv::test::FirstBackupNodeId, ni); + REQUIRE(txs.commit() == ccf::kv::CommitResult::SUCCESS); } primary_store.rollback({store_term, 1}, primary_store.commit_view()); @@ -212,15 +212,15 @@ TEST_CASE("Check signing works across rollback") } } -class CompactingConsensus : public kv::test::StubConsensus +class CompactingConsensus : public ccf::kv::test::StubConsensus { public: - kv::Store* store; + ccf::kv::Store* store; size_t count = 0; - CompactingConsensus(kv::Store* store_) : store(store_) {} + CompactingConsensus(ccf::kv::Store* store_) : store(store_) {} - bool replicate(const kv::BatchVector& entries, ccf::View view) override + bool replicate(const ccf::kv::BatchVector& entries, ccf::View view) override { for (auto& [version, data, committable, hooks] : entries) { @@ -241,36 +241,36 @@ class CompactingConsensus : public kv::test::StubConsensus return 0; } - std::optional primary() override + std::optional primary() override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } - kv::NodeId id() override + ccf::kv::NodeId id() override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } - ccf::View get_view(kv::Version version) override + ccf::View get_view(ccf::kv::Version version) override { return 2; } }; -class TestPendingTx : public kv::PendingTx +class TestPendingTx : public ccf::kv::PendingTx { ccf::TxID txid; - kv::Store& store; + ccf::kv::Store& store; MapT& other_table; public: - TestPendingTx(ccf::TxID txid_, kv::Store& store_, MapT& other_table_) : + TestPendingTx(ccf::TxID txid_, ccf::kv::Store& store_, MapT& other_table_) : txid(txid_), store(store_), other_table(other_table_) {} - kv::PendingTxInfo call() override + ccf::kv::PendingTxInfo call() override { auto txr = store.create_reserved_tx(txid); auto txrv = txr.rw(other_table); @@ -283,8 +283,8 @@ TEST_CASE( "Batches containing but not ending on a committable transaction should not " "halt replication") { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); std::shared_ptr consensus = std::make_shared(&store); @@ -298,7 +298,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 1); } @@ -309,7 +309,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 1); store.commit( @@ -322,27 +322,29 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 4); } } -class RollbackConsensus : public kv::test::StubConsensus +class RollbackConsensus : public ccf::kv::test::StubConsensus { public: - kv::Store* store; + ccf::kv::Store* store; size_t count = 0; - kv::Version rollback_at; - kv::Version rollback_to; + ccf::kv::Version rollback_at; + ccf::kv::Version rollback_to; RollbackConsensus( - kv::Store* store_, kv::Version rollback_at_, kv::Version rollback_to_) : + ccf::kv::Store* store_, + ccf::kv::Version rollback_at_, + ccf::kv::Version rollback_to_) : store(store_), rollback_at(rollback_at_), rollback_to(rollback_to_) {} - bool replicate(const kv::BatchVector& entries, ccf::View view) override + bool replicate(const ccf::kv::BatchVector& entries, ccf::View view) override { for (auto& [version, data, committable, hook] : entries) { @@ -363,14 +365,14 @@ class RollbackConsensus : public kv::test::StubConsensus return 0; } - std::optional primary() override + std::optional primary() override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } - kv::NodeId id() override + ccf::kv::NodeId id() override { - return kv::test::PrimaryNodeId; + return ccf::kv::test::PrimaryNodeId; } ccf::View get_view(ccf::SeqNo seqno) override @@ -387,8 +389,8 @@ class RollbackConsensus : public kv::test::StubConsensus TEST_CASE( "Check that empty rollback during replicate does not cause replication halts") { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); std::shared_ptr consensus = std::make_shared(&store, 2, 2); @@ -401,7 +403,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 1); } @@ -410,7 +412,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 2); } @@ -419,7 +421,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 3); } } @@ -427,8 +429,8 @@ TEST_CASE( TEST_CASE( "Check that rollback during replicate does not cause replication halts") { - kv::Store store; - auto encryptor = std::make_shared(); + ccf::kv::Store store; + auto encryptor = std::make_shared(); store.set_encryptor(encryptor); std::shared_ptr consensus = std::make_shared(&store, 2, 1); @@ -441,7 +443,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 1); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 1); } @@ -450,7 +452,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 2); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 2); } @@ -459,7 +461,7 @@ TEST_CASE( auto tx = store.create_tx(); auto txv = tx.rw(table); txv->put(0, 3); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); REQUIRE(consensus->count == 3); } } @@ -467,11 +469,11 @@ TEST_CASE( int main(int argc, char** argv) { threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/test/history_bench.cpp b/src/node/test/history_bench.cpp index bafeb01d2705..8fad0325b23f 100644 --- a/src/node/test/history_bench.cpp +++ b/src/node/test/history_bench.cpp @@ -20,7 +20,7 @@ namespace threading using namespace ccf; -class DummyConsensus : public kv::test::StubConsensus +class DummyConsensus : public ccf::kv::test::StubConsensus { public: DummyConsensus() {} @@ -59,7 +59,7 @@ static void hash_only(picobench::state& s) { (void)_; auto data = txs[idx++]; - crypto::Sha256Hash h(data); + ccf::crypto::Sha256Hash h(data); do_not_optimize(h); clobber_memory(); } @@ -71,14 +71,16 @@ static void append(picobench::state& s) { ::srand(42); - kv::Store store; - auto kp = crypto::make_key_pair(); + ccf::kv::Store store; + auto kp = ccf::crypto::make_key_pair(); - std::shared_ptr consensus = std::make_shared(); + std::shared_ptr consensus = + std::make_shared(); store.set_consensus(consensus); - std::shared_ptr history = - std::make_shared(store, kv::test::PrimaryNodeId, *kp); + std::shared_ptr history = + std::make_shared( + store, ccf::kv::test::PrimaryNodeId, *kp); store.set_history(history); std::vector> txs; @@ -108,14 +110,16 @@ static void append_compact(picobench::state& s) { ::srand(42); - kv::Store store; - auto kp = crypto::make_key_pair(); + ccf::kv::Store store; + auto kp = ccf::crypto::make_key_pair(); - std::shared_ptr consensus = std::make_shared(); + std::shared_ptr consensus = + std::make_shared(); store.set_consensus(consensus); - std::shared_ptr history = - std::make_shared(store, kv::test::PrimaryNodeId, *kp); + std::shared_ptr history = + std::make_shared( + store, ccf::kv::test::PrimaryNodeId, *kp); store.set_history(history); std::vector> txs; @@ -161,13 +165,13 @@ PICOBENCH(append_compact<1000>).iterations(sizes).samples(10); int main(int argc, char* argv[]) { - logger::config::level() = LoggerLevel::FATAL; + ccf::logger::config::level() = LoggerLevel::FATAL; ::threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); picobench::runner runner; runner.parse_cmd_line(argc, argv); auto ret = runner.run(); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); return ret; } diff --git a/src/node/test/merkle_bench.cpp b/src/node/test/merkle_bench.cpp index 89512db6f664..11628eda42c9 100644 --- a/src/node/test/merkle_bench.cpp +++ b/src/node/test/merkle_bench.cpp @@ -25,13 +25,13 @@ inline void clobber_memory() static void append_retract(picobench::state& s) { ccf::MerkleTreeHistory t; - vector hashes; + vector hashes; std::random_device r; for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); hashes.emplace_back(h); @@ -58,13 +58,13 @@ static void append_retract(picobench::state& s) static void append_flush(picobench::state& s) { ccf::MerkleTreeHistory t; - vector hashes; + vector hashes; std::random_device r; for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); hashes.emplace_back(h); @@ -88,13 +88,13 @@ static void append_flush(picobench::state& s) static void append_get_proof_verify(picobench::state& s) { ccf::MerkleTreeHistory t; - vector hashes; + vector hashes; std::random_device r; for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); hashes.emplace_back(h); @@ -120,13 +120,13 @@ static void append_get_proof_verify(picobench::state& s) static void append_get_proof_verify_v(picobench::state& s) { ccf::MerkleTreeHistory t; - vector hashes; + vector hashes; std::random_device r; for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); hashes.emplace_back(h); @@ -157,8 +157,8 @@ static void serialise_deserialise(picobench::state& s) for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); t.append(h); } @@ -176,8 +176,8 @@ static void serialised_size(picobench::state& s) for (size_t i = 0; i < s.iterations(); ++i) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = r(); t.append(h); } @@ -191,8 +191,8 @@ static void serialised_size(picobench::state& s) s.iterations(), buf.size(), bph, - (bph - crypto::Sha256Hash::SIZE) * 100 / - crypto::Sha256Hash::SIZE) + (bph - ccf::crypto::Sha256Hash::SIZE) * 100 / + ccf::crypto::Sha256Hash::SIZE) << std::endl; } @@ -219,9 +219,9 @@ PICOBENCH(serialised_size) int main(int argc, char* argv[]) { picobench::runner runner; - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); runner.parse_cmd_line(argc, argv); auto ret = runner.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return ret; } \ No newline at end of file diff --git a/src/node/test/merkle_mem.cpp b/src/node/test/merkle_mem.cpp index 149871be8e7e..f39268c12b32 100644 --- a/src/node/test/merkle_mem.cpp +++ b/src/node/test/merkle_mem.cpp @@ -25,10 +25,10 @@ static size_t get_maxrss() return r.ru_maxrss; } -static crypto::Sha256Hash random_hash(std::random_device& rdev) +static ccf::crypto::Sha256Hash random_hash(std::random_device& rdev) { - crypto::Sha256Hash h; - for (size_t j = 0; j < crypto::Sha256Hash::SIZE; j++) + ccf::crypto::Sha256Hash h; + for (size_t j = 0; j < ccf::crypto::Sha256Hash::SIZE; j++) h.h[j] = rdev(); return h; } diff --git a/src/node/test/merkle_test.cpp b/src/node/test/merkle_test.cpp index 5e8bd287fdff..11377d3635a3 100644 --- a/src/node/test/merkle_test.cpp +++ b/src/node/test/merkle_test.cpp @@ -6,9 +6,9 @@ #define DOCTEST_CONFIG_IMPLEMENT #include -crypto::Sha256Hash rand_hash() +ccf::crypto::Sha256Hash rand_hash() { - crypto::Sha256Hash hash; + ccf::crypto::Sha256Hash hash; uint8_t* data = hash.h.data(); for (size_t i = 0; i < hash.h.size(); ++i) { @@ -88,8 +88,9 @@ TEST_CASE("Retrieving leaves") { constexpr size_t hash_count = 1'000; - std::map hashes; - hashes[0] = crypto::Sha256Hash(); // Default index 0 always contains all-0s + std::map hashes; + hashes[0] = + ccf::crypto::Sha256Hash(); // Default index 0 always contains all-0s ccf::MerkleTreeHistory history; @@ -165,7 +166,7 @@ TEST_CASE("First root") INFO("Empty root"); ccf::MerkleTreeHistory tree; const auto empty_root = tree.get_root(); - REQUIRE(empty_root == crypto::Sha256Hash()); + REQUIRE(empty_root == ccf::crypto::Sha256Hash()); REQUIRE(tree.get_leaf(0) == empty_root); } @@ -181,10 +182,10 @@ TEST_CASE("First root") int main(int argc, char** argv) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return res; } diff --git a/src/node/test/receipt.cpp b/src/node/test/receipt.cpp index 30e293597b77..063ee06ceb6f 100644 --- a/src/node/test/receipt.cpp +++ b/src/node/test/receipt.cpp @@ -15,11 +15,11 @@ std::random_device rand_device; std::default_random_engine rand_engine(rand_device()); -crypto::Sha256Hash rand_digest() +ccf::crypto::Sha256Hash rand_digest() { std::uniform_int_distribution dist; - crypto::Sha256Hash ret; + ccf::crypto::Sha256Hash ret; std::generate( ret.h.begin(), ret.h.end(), [&]() { return dist(rand_engine); }); @@ -30,11 +30,11 @@ void populate_receipt(std::shared_ptr receipt) { using namespace std::literals; const auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 1h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 1h); const auto valid_to = - ds::to_x509_time_string(std::chrono::system_clock::now() + 1h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() + 1h); - auto node_kp = crypto::make_key_pair(); + auto node_kp = ccf::crypto::make_key_pair(); auto node_cert = node_kp->self_sign("CN=node", valid_from, valid_to); receipt->cert = node_cert; @@ -54,11 +54,11 @@ void populate_receipt(std::shared_ptr receipt) if (dir == ccf::ProofReceipt::ProofStep::Left) { - current_digest = crypto::Sha256Hash(digest, current_digest); + current_digest = ccf::crypto::Sha256Hash(digest, current_digest); } else { - current_digest = crypto::Sha256Hash(current_digest, digest); + current_digest = ccf::crypto::Sha256Hash(current_digest, digest); } } @@ -68,7 +68,7 @@ void populate_receipt(std::shared_ptr receipt) const auto num_endorsements = rand() % 3; for (auto i = 0; i < num_endorsements; ++i) { - auto service_kp = crypto::make_key_pair(); + auto service_kp = ccf::crypto::make_key_pair(); auto service_cert = service_kp->self_sign("CN=service", valid_from, valid_to); const auto csr = node_kp->create_csr(fmt::format("CN=Test{}", i)); @@ -169,7 +169,7 @@ TEST_CASE("JSON parsing" * doctest::test_suite("receipt")) TEST_CASE("JSON roundtrip" * doctest::test_suite("receipt")) { - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); { std::shared_ptr r = nullptr; nlohmann::json j; @@ -194,5 +194,5 @@ TEST_CASE("JSON roundtrip" * doctest::test_suite("receipt")) compare_receipts(p_receipt, parsed); } } - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); } \ No newline at end of file diff --git a/src/node/test/secret_share.cpp b/src/node/test/secret_share.cpp index ed9faf672088..c936ee6699ee 100644 --- a/src/node/test/secret_share.cpp +++ b/src/node/test/secret_share.cpp @@ -15,8 +15,8 @@ TEST_CASE("Simple test") INFO("Data to split must be have fixed length"); { - auto random = - crypto::get_entropy()->random(ccf::SecretSharing::SECRET_TO_SPLIT_LENGTH); + auto random = ccf::crypto::get_entropy()->random( + ccf::SecretSharing::SECRET_TO_SPLIT_LENGTH); std::copy_n( random.begin(), ccf::SecretSharing::SECRET_TO_SPLIT_LENGTH, diff --git a/src/node/test/snapshot.cpp b/src/node/test/snapshot.cpp index e2b06c7444ab..90d6f2b06c53 100644 --- a/src/node/test/snapshot.cpp +++ b/src/node/test/snapshot.cpp @@ -20,14 +20,14 @@ std::unique_ptr TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) { - auto source_consensus = std::make_shared(); - kv::Store source_store; - auto encryptor = std::make_shared(); + auto source_consensus = std::make_shared(); + ccf::kv::Store source_store; + auto encryptor = std::make_shared(); source_store.set_encryptor(encryptor); source_store.set_consensus(source_consensus); - ccf::NodeId source_node_id = kv::test::PrimaryNodeId; - auto source_node_kp = crypto::make_key_pair(); + ccf::NodeId source_node_id = ccf::kv::test::PrimaryNodeId; + auto source_node_kp = ccf::crypto::make_key_pair(); auto source_history = std::make_shared( source_store, source_node_id, *source_node_kp); @@ -36,10 +36,10 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) source_store.set_history(source_history); source_store.initialise_term(2); - kv::Map string_map("public:string_map"); + ccf::kv::Map string_map("public:string_map"); size_t transactions_count = 3; - kv::Version snapshot_version = kv::NoVersion; + ccf::kv::Version snapshot_version = ccf::kv::NoVersion; INFO("Apply transactions to original store"); { @@ -48,7 +48,7 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) auto tx = source_store.create_tx(); auto map = tx.rw(string_map); map->put(fmt::format("key#{}", i), "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } @@ -82,7 +82,7 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) target_tree.append(ccf::entry_leaf( serialised_signature, - crypto::Sha256Hash("ce:2.4:"), + ccf::crypto::Sha256Hash("ce:2.4:"), ccf::empty_claims())); REQUIRE( target_tree.get_root() == source_history->get_replicated_state_root()); @@ -90,15 +90,15 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) INFO("Snapshot at signature"); { - kv::Store target_store; - auto encryptor = std::make_shared(); + ccf::kv::Store target_store; + auto encryptor = std::make_shared(); target_store.set_encryptor(encryptor); INFO("Setup target store"); { - auto target_node_kp = crypto::make_key_pair(); + auto target_node_kp = ccf::crypto::make_key_pair(); auto target_history = std::make_shared( - target_store, kv::test::PrimaryNodeId, *target_node_kp); + target_store, ccf::kv::test::PrimaryNodeId, *target_node_kp); target_history->set_endorsed_certificate( make_self_signed_cert(target_node_kp)); target_store.set_history(target_history); @@ -108,43 +108,45 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) INFO("Apply snapshot taken before any signature was emitted"); { - std::unique_ptr snapshot = nullptr; + std::unique_ptr snapshot = + nullptr; { - kv::ScopedStoreMapsLock maps_lock(&source_store); + ccf::kv::ScopedStoreMapsLock maps_lock(&source_store); snapshot = source_store.snapshot_unsafe_maps(snapshot_version - 1); } auto serialised_snapshot = source_store.serialise_snapshot(std::move(snapshot)); // There is no signature to read to seed the target history - std::vector view_history; - kv::ConsensusHookPtrs hooks; + std::vector view_history; + ccf::kv::ConsensusHookPtrs hooks; REQUIRE( target_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks, - &view_history) == kv::ApplyResult::FAIL); + &view_history) == ccf::kv::ApplyResult::FAIL); } INFO("Apply snapshot taken at signature"); { - std::unique_ptr snapshot = nullptr; + std::unique_ptr snapshot = + nullptr; { - kv::ScopedStoreMapsLock maps_lock(&source_store); + ccf::kv::ScopedStoreMapsLock maps_lock(&source_store); snapshot = source_store.snapshot_unsafe_maps(snapshot_version); } auto serialised_snapshot = source_store.serialise_snapshot(std::move(snapshot)); - std::vector view_history; - kv::ConsensusHookPtrs hooks; + std::vector view_history; + ccf::kv::ConsensusHookPtrs hooks; REQUIRE( target_store.deserialise_snapshot( serialised_snapshot.data(), serialised_snapshot.size(), hooks, - &view_history) == kv::ApplyResult::PASS); + &view_history) == ccf::kv::ApplyResult::PASS); // Merkle history and view history thus far are restored when applying // snapshot @@ -160,7 +162,7 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) auto tx = source_store.create_tx(); auto map = tx.rw(string_map); map->put("key", "value"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); auto serialised_tx = source_consensus->get_latest_data().value(); @@ -176,11 +178,11 @@ TEST_CASE("Snapshot with merkle tree" * doctest::test_suite("snapshot")) int main(int argc, char** argv) { threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/test/snapshotter.cpp b/src/node/test/snapshotter.cpp index 273927f89708..914fdd3a3957 100644 --- a/src/node/test/snapshotter.cpp +++ b/src/node/test/snapshotter.cpp @@ -21,9 +21,9 @@ std::unique_ptr threading::ThreadMessaging::singleton = nullptr; constexpr auto buffer_size = 1024 * 16; -auto kp = crypto::make_key_pair(); +auto kp = ccf::crypto::make_key_pair(); -using StringString = kv::Map; +using StringString = ccf::kv::Map; using rb_msg = std::pair; auto read_ringbuffer_out(ringbuffer::Circuit& circuit) @@ -92,11 +92,12 @@ void issue_transactions(ccf::NetworkState& network, size_t tx_count) auto tx = network.tables->create_tx(); auto map = tx.rw("public:map"); map->put("foo", "bar"); - REQUIRE(tx.commit() == kv::CommitResult::SUCCESS); + REQUIRE(tx.commit() == ccf::kv::CommitResult::SUCCESS); } } -size_t read_latest_snapshot_evidence(const std::shared_ptr& store) +size_t read_latest_snapshot_evidence( + const std::shared_ptr& store) { auto tx = store->create_read_only_tx(); auto h = tx.ro(ccf::Tables::SNAPSHOT_EVIDENCE); @@ -114,11 +115,11 @@ bool record_signature( size_t idx) { std::vector dummy_signature(128, 43); - crypto::Pem node_cert; + ccf::crypto::Pem node_cert; bool requires_snapshot = snapshotter->record_committable(idx); snapshotter->record_signature( - idx, dummy_signature, kv::test::PrimaryNodeId, node_cert); + idx, dummy_signature, ccf::kv::test::PrimaryNodeId, node_cert); snapshotter->record_serialised_tree(idx, history->serialise_tree(idx)); return requires_snapshot; @@ -135,17 +136,17 @@ void record_snapshot_evidence( TEST_CASE("Regular snapshotting") { - logger::config::default_init(); + ccf::logger::config::default_init(); ccf::NetworkState network; - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); auto history = std::make_shared( - *network.tables.get(), kv::test::PrimaryNodeId, *kp); + *network.tables.get(), ccf::kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); network.tables->initialise_term(2); network.tables->set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); auto in_buffer = std::make_unique(buffer_size); @@ -302,13 +303,13 @@ TEST_CASE("Regular snapshotting") TEST_CASE("Rollback before snapshot is committed") { ccf::NetworkState network; - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); auto history = std::make_shared( - *network.tables.get(), kv::test::PrimaryNodeId, *kp); + *network.tables.get(), ccf::kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); network.tables->initialise_term(2); network.tables->set_consensus(consensus); - auto encryptor = std::make_shared(); + auto encryptor = std::make_shared(); network.tables->set_encryptor(encryptor); auto in_buffer = std::make_unique(buffer_size); @@ -394,7 +395,7 @@ TEST_CASE("Rollback before snapshot is committed") size_t snapshot_idx = network.tables->current_version(); network.tables->set_flag( - kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); + ccf::kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE); REQUIRE_FALSE(record_signature(history, snapshotter, snapshot_idx)); snapshotter->commit(snapshot_idx, true); @@ -410,7 +411,7 @@ TEST_CASE("Rollback before snapshot is committed") REQUIRE(snapshotter->write_snapshot(snapshot, snapshot_count)); REQUIRE(!network.tables->flag_enabled( - kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE)); + ccf::kv::AbstractStore::Flag::SNAPSHOT_AT_NEXT_SIGNATURE)); // Commit evidence issue_transactions(network, 1); @@ -429,13 +430,13 @@ TEST_CASE("Rollback before snapshot is committed") // https://github.com/microsoft/CCF/issues/3796 TEST_CASE("Rekey ledger while snapshot is in progress") { - logger::config::default_init(); + ccf::logger::config::default_init(); ccf::NetworkState network; - auto consensus = std::make_shared(); + auto consensus = std::make_shared(); auto history = std::make_shared( - *network.tables.get(), kv::test::PrimaryNodeId, *kp); + *network.tables.get(), ccf::kv::test::PrimaryNodeId, *kp); network.tables->set_history(history); network.tables->initialise_term(2); network.tables->set_consensus(consensus); @@ -467,7 +468,7 @@ TEST_CASE("Rekey ledger while snapshot is in progress") auto sigs = tx.rw(ccf::Tables::SIGNATURES); auto trees = tx.rw(ccf::Tables::SERIALISED_MERKLE_TREE); - sigs->put({kv::test::PrimaryNodeId, 0, 0, {}, {}, {}, {}}); + sigs->put({ccf::kv::test::PrimaryNodeId, 0, 0, {}, {}, {}, {}}); auto tree = history->serialise_tree(snapshot_idx - 1); trees->put(tree); tx.commit(); @@ -501,7 +502,7 @@ TEST_CASE("Rekey ledger while snapshot is in progress") // Snapshot can be deserialised to backup store ccf::NetworkState backup_network; auto backup_history = std::make_shared( - *backup_network.tables.get(), kv::test::FirstBackupNodeId, *kp); + *backup_network.tables.get(), ccf::kv::test::FirstBackupNodeId, *kp); backup_network.tables->set_history(backup_history); auto tx = network.tables->create_read_only_tx(); @@ -511,23 +512,23 @@ TEST_CASE("Rekey ledger while snapshot is in progress") std::make_shared(backup_ledger_secrets); backup_network.tables->set_encryptor(backup_encryptor); - kv::ConsensusHookPtrs hooks; - std::vector view_history; + ccf::kv::ConsensusHookPtrs hooks; + std::vector view_history; REQUIRE( backup_network.tables->deserialise_snapshot( snapshot.data(), snapshot.size(), hooks, &view_history) == - kv::ApplyResult::PASS); + ccf::kv::ApplyResult::PASS); } } int main(int argc, char** argv) { threading::ThreadMessaging::init(1); - crypto::openssl_sha256_init(); + ccf::crypto::openssl_sha256_init(); doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); if (context.shouldExit()) return res; return res; diff --git a/src/node/tx_receipt_impl.h b/src/node/tx_receipt_impl.h index 97bdcdbc1f70..c7ad613f84c6 100644 --- a/src/node/tx_receipt_impl.h +++ b/src/node/tx_receipt_impl.h @@ -15,25 +15,27 @@ namespace ccf HistoryTree::Hash root = {}; std::shared_ptr path = {}; ccf::NodeId node_id = {}; - std::optional node_cert = std::nullopt; - std::optional write_set_digest = std::nullopt; + std::optional node_cert = std::nullopt; + std::optional write_set_digest = std::nullopt; std::optional commit_evidence = std::nullopt; ccf::ClaimsDigest claims_digest = {}; - std::optional> service_endorsements = std::nullopt; + std::optional> service_endorsements = + std::nullopt; TxReceiptImpl( const std::vector& signature_, const HistoryTree::Hash& root_, std::shared_ptr path_, const NodeId& node_id_, - const std::optional& node_cert_, - const std::optional& write_set_digest_ = std::nullopt, + const std::optional& node_cert_, + const std::optional& write_set_digest_ = + std::nullopt, // Optional to support historical transactions, where it may be absent const std::optional& commit_evidence_ = std::nullopt, // May not be set on historical transactions const ccf::ClaimsDigest& claims_digest_ = ccf::no_claims(), - const std::optional>& service_endorsements_ = - std::nullopt) : + const std::optional>& + service_endorsements_ = std::nullopt) : signature(signature_), root(root_), path(path_), diff --git a/src/node/uvm_endorsements.h b/src/node/uvm_endorsements.h index 6e94a95dce0c..3de89e14213a 100644 --- a/src/node/uvm_endorsements.h +++ b/src/node/uvm_endorsements.h @@ -251,10 +251,10 @@ namespace ccf } static std::span verify_uvm_endorsements_signature( - const crypto::Pem& leaf_cert_pub_key, + const ccf::crypto::Pem& leaf_cert_pub_key, const std::vector& uvm_endorsements_raw) { - auto verifier = crypto::make_cose_verifier_from_key(leaf_cert_pub_key); + auto verifier = ccf::crypto::make_cose_verifier_from_key(leaf_cert_pub_key); std::span payload; if (!verifier->verify(uvm_endorsements_raw, payload)) @@ -282,7 +282,7 @@ namespace ccf std::string pem_chain; for (auto const& c : phdr.x5_chain) { - pem_chain += crypto::cert_der_to_pem(c).str(); + pem_chain += ccf::crypto::cert_der_to_pem(c).str(); } const auto& did = phdr.iss; @@ -297,25 +297,26 @@ namespace ccf did_document_str)); } - crypto::Pem pubk; + ccf::crypto::Pem pubk; for (auto const& vm : did_document.verification_method) { if (vm.controller == did && vm.public_key_jwk.has_value()) { - auto jwk = vm.public_key_jwk.value().get(); + auto jwk = vm.public_key_jwk.value().get(); switch (jwk.kty) { - case crypto::JsonWebKeyType::RSA: + case ccf::crypto::JsonWebKeyType::RSA: { auto rsa_jwk = - vm.public_key_jwk->get(); - pubk = crypto::make_rsa_public_key(rsa_jwk)->public_key_pem(); + vm.public_key_jwk->get(); + pubk = ccf::crypto::make_rsa_public_key(rsa_jwk)->public_key_pem(); break; } - case crypto::JsonWebKeyType::EC: + case ccf::crypto::JsonWebKeyType::EC: { - auto ec_jwk = vm.public_key_jwk->get(); - pubk = crypto::make_public_key(ec_jwk)->public_key_pem(); + auto ec_jwk = + vm.public_key_jwk->get(); + pubk = ccf::crypto::make_public_key(ec_jwk)->public_key_pem(); break; } default: diff --git a/src/quic/test/main.cpp b/src/quic/test/main.cpp index 93a40fad4404..6d248d1ee62e 100644 --- a/src/quic/test/main.cpp +++ b/src/quic/test/main.cpp @@ -8,7 +8,7 @@ #include using namespace std; -using namespace crypto; +using namespace ccf::crypto; TEST_CASE("check QUIC OpenSSL library call") { diff --git a/src/service/internal_tables_access.h b/src/service/internal_tables_access.h index 9b1abe268ba7..660c797278cb 100644 --- a/src/service/internal_tables_access.h +++ b/src/service/internal_tables_access.h @@ -34,7 +34,7 @@ namespace ccf // instantiated InternalTablesAccess() = delete; - static void retire_active_nodes(kv::Tx& tx) + static void retire_active_nodes(ccf::kv::Tx& tx) { auto nodes = tx.rw(Tables::NODES); @@ -54,7 +54,7 @@ namespace ccf } static bool is_recovery_member( - kv::ReadOnlyTx& tx, const MemberId& member_id) + ccf::kv::ReadOnlyTx& tx, const MemberId& member_id) { auto member_encryption_public_keys = tx.ro( @@ -63,7 +63,8 @@ namespace ccf return member_encryption_public_keys->get(member_id).has_value(); } - static bool is_active_member(kv::ReadOnlyTx& tx, const MemberId& member_id) + static bool is_active_member( + ccf::kv::ReadOnlyTx& tx, const MemberId& member_id) { auto member_info = tx.ro(Tables::MEMBER_INFO); auto mi = member_info->get(member_id); @@ -75,15 +76,15 @@ namespace ccf return mi->status == MemberStatus::ACTIVE; } - static std::map get_active_recovery_members( - kv::ReadOnlyTx& tx) + static std::map get_active_recovery_members( + ccf::kv::ReadOnlyTx& tx) { auto member_info = tx.ro(Tables::MEMBER_INFO); auto member_encryption_public_keys = tx.ro( Tables::MEMBER_ENCRYPTION_PUBLIC_KEYS); - std::map active_recovery_members; + std::map active_recovery_members; member_encryption_public_keys->foreach( [&active_recovery_members, @@ -104,7 +105,8 @@ namespace ccf return active_recovery_members; } - static MemberId add_member(kv::Tx& tx, const NewMember& member_pub_info) + static MemberId add_member( + ccf::kv::Tx& tx, const NewMember& member_pub_info) { auto member_certs = tx.rw(Tables::MEMBER_CERTS); auto member_info = tx.rw(Tables::MEMBER_INFO); @@ -112,8 +114,8 @@ namespace ccf auto signatures = tx.ro(Tables::SIGNATURES); auto member_cert_der = - crypto::make_verifier(member_pub_info.cert)->cert_der(); - auto id = crypto::Sha256Hash(member_cert_der).hex_str(); + ccf::crypto::make_verifier(member_pub_info.cert)->cert_der(); + auto id = ccf::crypto::Sha256Hash(member_cert_der).hex_str(); auto member = member_certs->get(id); if (member.has_value()) @@ -147,7 +149,7 @@ namespace ccf return id; } - static bool activate_member(kv::Tx& tx, const MemberId& member_id) + static bool activate_member(ccf::kv::Tx& tx, const MemberId& member_id) { auto member_info = tx.rw(Tables::MEMBER_INFO); @@ -176,7 +178,7 @@ namespace ccf return newly_active; } - static bool remove_member(kv::Tx& tx, const MemberId& member_id) + static bool remove_member(ccf::kv::Tx& tx, const MemberId& member_id) { auto member_certs = tx.rw(Tables::MEMBER_CERTS); auto member_encryption_public_keys = @@ -230,12 +232,13 @@ namespace ccf return true; } - static UserId add_user(kv::Tx& tx, const NewUser& new_user) + static UserId add_user(ccf::kv::Tx& tx, const NewUser& new_user) { auto user_certs = tx.rw(Tables::USER_CERTS); - auto user_cert_der = crypto::make_verifier(new_user.cert)->cert_der(); - auto id = crypto::Sha256Hash(user_cert_der).hex_str(); + auto user_cert_der = + ccf::crypto::make_verifier(new_user.cert)->cert_der(); + auto id = ccf::crypto::Sha256Hash(user_cert_der).hex_str(); auto user_cert = user_certs->get(id); if (user_cert.has_value()) @@ -262,7 +265,7 @@ namespace ccf return id; } - static void remove_user(kv::Tx& tx, const UserId& user_id) + static void remove_user(ccf::kv::Tx& tx, const UserId& user_id) { // Has no effect if the user does not exist auto user_certs = tx.rw(Tables::USER_CERTS); @@ -273,14 +276,15 @@ namespace ccf } static void add_node( - kv::Tx& tx, const NodeId& id, const NodeInfo& node_info) + ccf::kv::Tx& tx, const NodeId& id, const NodeInfo& node_info) { auto node = tx.rw(Tables::NODES); node->put(id, node_info); } static auto get_trusted_nodes( - kv::ReadOnlyTx& tx, std::optional self_to_exclude = std::nullopt) + ccf::kv::ReadOnlyTx& tx, + std::optional self_to_exclude = std::nullopt) { // Returns the list of trusted nodes. If self_to_exclude is set, // self_to_exclude is not included in the list of returned nodes. @@ -314,8 +318,8 @@ namespace ccf // Service status should use a state machine, very much like NodeState. static void create_service( - kv::Tx& tx, - const crypto::Pem& service_cert, + ccf::kv::Tx& tx, + const ccf::crypto::Pem& service_cert, ccf::TxID create_txid, nlohmann::json service_data = nullptr, bool recovering = false) @@ -347,13 +351,13 @@ namespace ccf } static bool is_service_created( - kv::ReadOnlyTx& tx, const crypto::Pem& expected_service_cert) + ccf::kv::ReadOnlyTx& tx, const ccf::crypto::Pem& expected_service_cert) { auto service = tx.ro(Tables::SERVICE)->get(); return service.has_value() && service->cert == expected_service_cert; } - static bool open_service(kv::Tx& tx) + static bool open_service(ccf::kv::Tx& tx) { auto service = tx.rw(Tables::SERVICE); @@ -400,7 +404,8 @@ namespace ccf return true; } - static std::optional get_service_status(kv::ReadOnlyTx& tx) + static std::optional get_service_status( + ccf::kv::ReadOnlyTx& tx) { auto service = tx.ro(Tables::SERVICE); auto active_service = service->get(); @@ -414,7 +419,9 @@ namespace ccf } static void trust_node( - kv::Tx& tx, const NodeId& node_id, kv::Version latest_ledger_secret_seqno) + ccf::kv::Tx& tx, + const NodeId& node_id, + ccf::kv::Version latest_ledger_secret_seqno) { auto nodes = tx.rw(Tables::NODES); auto node_info = nodes->get(node_id); @@ -436,13 +443,14 @@ namespace ccf LOG_INFO_FMT("Node {} is now {}", node_id, node_info->status); } - static void set_constitution(kv::Tx& tx, const std::string& constitution) + static void set_constitution( + ccf::kv::Tx& tx, const std::string& constitution) { tx.rw(Tables::CONSTITUTION)->put(constitution); } static void trust_node_measurement( - kv::Tx& tx, + ccf::kv::Tx& tx, const pal::PlatformAttestationMeasurement& node_measurement, const QuoteFormat& platform) { @@ -476,7 +484,7 @@ namespace ccf } static void trust_node_host_data( - kv::Tx& tx, + ccf::kv::Tx& tx, const HostData& host_data, const std::optional& security_policy = std::nullopt) { @@ -484,7 +492,7 @@ namespace ccf if (security_policy.has_value()) { auto raw_security_policy = - crypto::raw_from_b64(security_policy.value()); + ccf::crypto::raw_from_b64(security_policy.value()); host_data_table->put( host_data, {raw_security_policy.begin(), raw_security_policy.end()}); } @@ -496,7 +504,7 @@ namespace ccf } static void trust_node_uvm_endorsements( - kv::Tx& tx, const std::optional& uvm_endorsements) + ccf::kv::Tx& tx, const std::optional& uvm_endorsements) { if (!uvm_endorsements.has_value()) { @@ -512,7 +520,7 @@ namespace ccf } static void init_configuration( - kv::Tx& tx, const ServiceConfiguration& configuration) + ccf::kv::Tx& tx, const ServiceConfiguration& configuration) { auto config = tx.rw(Tables::CONFIGURATION); if (config->has()) @@ -525,7 +533,7 @@ namespace ccf config->put(configuration); } - static bool set_recovery_threshold(kv::Tx& tx, size_t threshold) + static bool set_recovery_threshold(ccf::kv::Tx& tx, size_t threshold) { auto config = tx.rw(Tables::CONFIGURATION); @@ -578,7 +586,7 @@ namespace ccf return true; } - static size_t get_recovery_threshold(kv::ReadOnlyTx& tx) + static size_t get_recovery_threshold(ccf::kv::ReadOnlyTx& tx) { auto config = tx.ro(Tables::CONFIGURATION); auto current_config = config->get(); diff --git a/src/service/network_tables.h b/src/service/network_tables.h index 57c6894a735e..26a6c48966c3 100644 --- a/src/service/network_tables.h +++ b/src/service/network_tables.h @@ -36,14 +36,14 @@ namespace ccf { - inline std::shared_ptr make_store() + inline std::shared_ptr make_store() { - return std::make_shared(); + return std::make_shared(); } struct NetworkTables { - std::shared_ptr tables; + std::shared_ptr tables; // // Governance tables (public:ccf.gov.*) diff --git a/src/service/tables/backup_signatures.h b/src/service/tables/backup_signatures.h index 668850941c0e..6e5610d25f35 100644 --- a/src/service/tables/backup_signatures.h +++ b/src/service/tables/backup_signatures.h @@ -14,13 +14,13 @@ namespace ccf { ccf::View view = 0; ccf::SeqNo seqno = 0; - crypto::Sha256Hash root; + ccf::crypto::Sha256Hash root; std::vector signatures; BackupSignatures() = default; BackupSignatures( - ccf::View view_, ccf::SeqNo seqno_, const crypto::Sha256Hash root_) : + ccf::View view_, ccf::SeqNo seqno_, const ccf::crypto::Sha256Hash root_) : view(view_), seqno(seqno_), root(root_) diff --git a/src/service/tables/node_signature.h b/src/service/tables/node_signature.h index 69a8deed114e..78a6d6aecb18 100644 --- a/src/service/tables/node_signature.h +++ b/src/service/tables/node_signature.h @@ -9,7 +9,7 @@ namespace ccf { - using Nonce = crypto::Sha256Hash; + using Nonce = ccf::crypto::Sha256Hash; struct NodeSignature { diff --git a/src/service/tables/previous_service_identity.h b/src/service/tables/previous_service_identity.h index ddb117a4e569..f4ae2b7315b4 100644 --- a/src/service/tables/previous_service_identity.h +++ b/src/service/tables/previous_service_identity.h @@ -10,7 +10,7 @@ namespace ccf { - using PreviousServiceIdentity = ServiceValue; + using PreviousServiceIdentity = ServiceValue; namespace Tables { diff --git a/src/service/tables/secrets.h b/src/service/tables/secrets.h index 2f7c2e376dcf..38c5d83aecd2 100644 --- a/src/service/tables/secrets.h +++ b/src/service/tables/secrets.h @@ -13,13 +13,14 @@ namespace ccf { // Version at which the ledger secret is applicable from (recovery // only) - std::optional version; + std::optional version; // Encrypted secret for each backup std::vector encrypted_secret = {}; // Version at which the previous secret is stored at - std::optional previous_secret_stored_version = std::nullopt; + std::optional previous_secret_stored_version = + std::nullopt; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EncryptedLedgerSecret) diff --git a/src/service/tables/shares.h b/src/service/tables/shares.h index 28191f848219..f80738fa5ea5 100644 --- a/src/service/tables/shares.h +++ b/src/service/tables/shares.h @@ -23,7 +23,8 @@ namespace ccf EncryptedSharesMap encrypted_shares; // Version at which the previous ledger secret was written to the store - std::optional previous_secret_stored_version = std::nullopt; + std::optional previous_secret_stored_version = + std::nullopt; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(RecoverySharesInfo) @@ -38,18 +39,19 @@ namespace ccf std::vector encrypted_data = {}; // Version at which the ledger secret is applicable from - kv::Version version = kv::NoVersion; + ccf::kv::Version version = ccf::kv::NoVersion; // Version at which the ledger secret _before_ this one was written to the // store - std::optional previous_secret_stored_version = std::nullopt; + std::optional previous_secret_stored_version = + std::nullopt; PreviousLedgerSecretInfo() = default; PreviousLedgerSecretInfo( std::vector&& encrypted_data_, - kv::Version version_, - std::optional stored_version_) : + ccf::kv::Version version_, + std::optional stored_version_) : encrypted_data(std::move(encrypted_data_)), version(version_), previous_secret_stored_version(stored_version_) @@ -89,7 +91,7 @@ namespace ccf // integrity on the public-only transactions. However, the corresponding // shares are only written at a later version, once the previous ledger // secrets have been restored. - std::optional next_version = std::nullopt; + std::optional next_version = std::nullopt; }; // Note: Both fields are never empty at the same time diff --git a/src/service/tables/signatures.h b/src/service/tables/signatures.h index 3a6e1b55efd4..665fe56cee78 100644 --- a/src/service/tables/signatures.h +++ b/src/service/tables/signatures.h @@ -20,9 +20,9 @@ namespace ccf ccf::SeqNo commit_seqno = 0; ccf::View commit_view = 0; /// Root of the Merkle Tree as of seqno - 1 - crypto::Sha256Hash root; + ccf::crypto::Sha256Hash root; /// Service-endorsed certificate of the node which produced the signature - crypto::Pem cert; + ccf::crypto::Pem cert; PrimarySignature() {} @@ -31,16 +31,16 @@ namespace ccf seqno(seqno_) {} - PrimarySignature(const crypto::Sha256Hash& root_) : root(root_) {} + PrimarySignature(const ccf::crypto::Sha256Hash& root_) : root(root_) {} PrimarySignature( const ccf::NodeId& node_, ccf::SeqNo seqno_, ccf::View view_, - const crypto::Sha256Hash root_, + const ccf::crypto::Sha256Hash root_, Nonce hashed_nonce_, const std::vector& sig_, - const crypto::Pem& cert_) : + const ccf::crypto::Pem& cert_) : NodeSignature(sig_, node_, hashed_nonce_), seqno(seqno_), view(view_), @@ -58,7 +58,8 @@ namespace ccf using Signatures = ServiceValue; // Serialised Merkle tree at most recent signature is a single Value in the KV - using SerialisedMerkleTree = kv::RawCopySerialisedValue>; + using SerialisedMerkleTree = + ccf::kv::RawCopySerialisedValue>; namespace Tables { diff --git a/src/service/tables/snapshot_evidence.h b/src/service/tables/snapshot_evidence.h index 3f22d13f9f7d..a0dcb12c8fa9 100644 --- a/src/service/tables/snapshot_evidence.h +++ b/src/service/tables/snapshot_evidence.h @@ -11,9 +11,9 @@ namespace ccf struct SnapshotHash { /// Snapshot digest - crypto::Sha256Hash hash; + ccf::crypto::Sha256Hash hash; /// Sequence number to which the snapshot corresponds - kv::Version version; + ccf::kv::Version version; }; DECLARE_JSON_TYPE(SnapshotHash) diff --git a/src/tls/ca.h b/src/tls/ca.h index f532c3e81e32..4965e1bb7323 100644 --- a/src/tls/ca.h +++ b/src/tls/ca.h @@ -7,7 +7,7 @@ #include -using namespace crypto::OpenSSL; +using namespace ccf::crypto::OpenSSL; namespace tls { diff --git a/src/tls/cert.h b/src/tls/cert.h index f6745edd0fe9..c2cb28ddf156 100644 --- a/src/tls/cert.h +++ b/src/tls/cert.h @@ -11,7 +11,7 @@ #include #include -using namespace crypto::OpenSSL; +using namespace ccf::crypto::OpenSSL; namespace tls { @@ -29,14 +29,14 @@ namespace tls Unique_X509 own_cert; Unique_STACK_OF_X509 chain; - std::shared_ptr own_pkey; + std::shared_ptr own_pkey; bool has_own_cert = false; public: Cert( std::shared_ptr peer_ca_, - const std::optional& own_cert_ = std::nullopt, - const std::optional& own_pkey_ = std::nullopt, + const std::optional& own_cert_ = std::nullopt, + const std::optional& own_pkey_ = std::nullopt, const std::optional& peer_hostname_ = std::nullopt, bool auth_required_ = true) : peer_ca(peer_ca_), @@ -45,13 +45,14 @@ namespace tls { if (own_cert_.has_value() && own_pkey_.has_value()) { - const auto certs = crypto::split_x509_cert_bundle(own_cert_->str()); + const auto certs = + ccf::crypto::split_x509_cert_bundle(own_cert_->str()); has_own_cert = true; { Unique_BIO certbio(certs[0]); own_cert = Unique_X509(certbio, true); - own_pkey = std::make_shared(*own_pkey_); + own_pkey = std::make_shared(*own_pkey_); } if (certs.size() > 1) diff --git a/src/tls/context.h b/src/tls/context.h index 53c660524d85..471ffb9816b1 100644 --- a/src/tls/context.h +++ b/src/tls/context.h @@ -16,8 +16,8 @@ namespace ccf::tls class Context { protected: - crypto::OpenSSL::Unique_SSL_CTX cfg; - crypto::OpenSSL::Unique_SSL ssl; + ccf::crypto::OpenSSL::Unique_SSL_CTX cfg; + ccf::crypto::OpenSSL::Unique_SSL ssl; public: Context(bool client) : @@ -202,14 +202,14 @@ namespace ccf::tls // need to do that because it's been verified before and we use // SSL_get_peer_certificate just to extract it from the context. - crypto::OpenSSL::Unique_X509 cert( + ccf::crypto::OpenSSL::Unique_X509 cert( SSL_get_peer_certificate(ssl), /*check_null=*/false); if (!cert) { LOG_TRACE_FMT("Empty peer cert"); return {}; } - crypto::OpenSSL::Unique_BIO bio; + ccf::crypto::OpenSSL::Unique_BIO bio; if (!i2d_X509_bio(bio, cert)) { LOG_TRACE_FMT("Can't convert X509 to DER"); diff --git a/src/tls/test/main.cpp b/src/tls/test/main.cpp index 21dcdb212e72..3dbb7f43b618 100644 --- a/src/tls/test/main.cpp +++ b/src/tls/test/main.cpp @@ -21,7 +21,7 @@ #include using namespace std; -using namespace crypto; +using namespace ccf::crypto; using namespace tls; /// Server uses one pipe while client uses the other. @@ -213,35 +213,35 @@ int handshake(ccf::tls::Context* ctx, std::atomic& keep_going) struct NetworkCA { - shared_ptr kp; - crypto::Pem cert; + shared_ptr kp; + ccf::crypto::Pem cert; }; -static crypto::Pem generate_self_signed_cert( - const crypto::KeyPairPtr& kp, const std::string& name) +static ccf::crypto::Pem generate_self_signed_cert( + const ccf::crypto::KeyPairPtr& kp, const std::string& name) { using namespace std::literals; constexpr size_t certificate_validity_period_days = 365; auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); - return crypto::create_self_signed_cert( + return ccf::crypto::create_self_signed_cert( kp, name, {}, valid_from, certificate_validity_period_days); } -static crypto::Pem generate_endorsed_cert( - const crypto::KeyPairPtr& kp, +static ccf::crypto::Pem generate_endorsed_cert( + const ccf::crypto::KeyPairPtr& kp, const std::string& name, - const crypto::KeyPairPtr& issuer_kp, - const crypto::Pem& issuer_cert) + const ccf::crypto::KeyPairPtr& issuer_kp, + const ccf::crypto::Pem& issuer_cert) { constexpr size_t certificate_validity_period_days = 365; using namespace std::literals; auto valid_from = - ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); + ::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h); - return crypto::create_endorsed_cert( + return ccf::crypto::create_endorsed_cert( kp, name, {}, @@ -255,7 +255,7 @@ static crypto::Pem generate_endorsed_cert( NetworkCA get_ca() { // Create a CA with a self-signed certificate - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto crt = generate_self_signed_cert(kp, "CN=issuer"); LOG_DEBUG_FMT("New self-signed CA certificate:\n{}", crt.str()); return {kp, crt}; @@ -269,12 +269,12 @@ unique_ptr<::tls::Cert> get_dummy_cert( auto ca = make_unique<::tls::CA>(net_ca.cert.str()); // Create a signing request and sign with the CA - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto crt = generate_endorsed_cert(kp, "CN=" + name, net_ca.kp, net_ca.cert); LOG_DEBUG_FMT("New CA-signed certificate:\n{}", crt.str()); // Verify node certificate with the CA's certificate - auto v = crypto::make_verifier(crt); + auto v = ccf::crypto::make_verifier(crt); REQUIRE(v->verify_certificate({&net_ca.cert})); // Create a ::tls::Cert with the CA, the signed certificate and the private @@ -497,7 +497,7 @@ TEST_CASE("verified handshake") TEST_CASE("self-signed server certificate") { - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto pk = kp->private_key_pem(); auto crt = generate_self_signed_cert(kp, "CN=server"); auto server_cert = make_unique(nullptr, crt, pk); @@ -545,7 +545,7 @@ TEST_CASE("self-signed client certificate") auto server_ca = get_ca(); auto server_cert = get_dummy_cert(server_ca, "server", false); - auto kp = crypto::make_key_pair(); + auto kp = ccf::crypto::make_key_pair(); auto pk = kp->private_key_pem(); auto crt = generate_self_signed_cert(kp, "CN=server"); @@ -621,7 +621,7 @@ TEST_CASE("large message") // Uninitialised on purpose, we don't care what's in here size_t len = 8192; uint8_t buf[len]; - auto message = crypto::b64_from_raw(buf, len); + auto message = ccf::crypto::b64_from_raw(buf, len); // Create a CA auto ca = get_ca(); @@ -647,7 +647,7 @@ TEST_CASE("very large message") // Uninitialised on purpose, we don't care what's in here size_t len = 16 * 1024; // 16k, base64 will be more uint8_t buf[len]; - auto message = crypto::b64_from_raw(buf, len); + auto message = ccf::crypto::b64_from_raw(buf, len); // Create a CA auto ca = get_ca(); diff --git a/src/tls/tls.h b/src/tls/tls.h index 5401cc297a06..2ac578757cd6 100644 --- a/src/tls/tls.h +++ b/src/tls/tls.h @@ -31,6 +31,6 @@ namespace tls { inline std::string error_string(int ec) { - return crypto::OpenSSL::error_string(ec); + return ccf::crypto::OpenSSL::error_string(ec); } } diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 5c594e604cb9..750c14f7d350 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -35,6 +35,7 @@ from types import MappingProxyType import threading import copy +import programmability from loguru import logger as LOG @@ -1961,7 +1962,7 @@ def run_app_space_js(args): primary, user.service_id, user_data={"isAdmin": True} ) - with primary.client(None, None, user.local_id) as c: + with primary.client() as c: parent_dir = os.path.normpath( os.path.join(os.path.dirname(__file__), os.path.pardir) ) @@ -1973,15 +1974,26 @@ def run_app_space_js(args): "js", ) bundle = network.consortium.read_bundle_from_dir(logging_js_dir) - r = c.put("/app/custom_endpoints", body=bundle) + signed_bundle = programmability.sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle + ) + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Also modify the runtime options to log and return errors, to aid debugging - r = c.call( + options = {"log_exception_details": True, "return_exception_details": True} + signed_options = programmability.sign_payload( + network.identity(user.local_id), "runtime_options", options + ) + r = c.patch( "/app/custom_endpoints/runtime_options", - {"log_exception_details": True, "return_exception_details": True}, - http_verb="PATCH", + signed_options, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.OK.value, r.status_code diff --git a/tests/fuzzing.py b/tests/fuzzing.py index 52f4d9beae69..afb63c1d7007 100644 --- a/tests/fuzzing.py +++ b/tests/fuzzing.py @@ -149,6 +149,14 @@ def fuzz_node_to_node(network, args): ) fuzz_logger.session = session + # Monkey-patch: Remove any Db loggers from the boofuzz session. We never + # use them, and they're reliant on disk IO (for db commits) so sometimes very slow + session._fuzz_data_logger._fuzz_loggers = [ + logger + for logger in session._fuzz_data_logger._fuzz_loggers + if not isinstance(logger, boofuzz.fuzz_logger_db.FuzzLoggerDb) + ] + session.connect(req) LOG.warning("These tests are verbose and run for a long time") diff --git a/tests/historical_query_perf.py b/tests/historical_query_perf.py index 1f6d133cce27..acd3542c6112 100644 --- a/tests/historical_query_perf.py +++ b/tests/historical_query_perf.py @@ -167,6 +167,12 @@ def test_historical_query_range(network, args): entries = {} node = network.find_node_by_role(role=infra.network.NodeRole.BACKUP, log_capture=[]) + + # During this test enclave may get very busy due to increasing amount of stores fetched in + # memory, so tick messages may stack up, resulting in a delayed 'stop' msg processing, therefore + # leading to a delayed shutdown. + node.remote.remote.shutdown_timeout *= 10 + with node.client(common_headers={"authorization": f"Bearer {jwt}"}) as c: # Index is currently built lazily to avoid impacting other perf tests using the same app # So pre-fetch to ensure index is fully constructed diff --git a/tests/infra/clients.py b/tests/infra/clients.py index 2e89480575e2..629cd81f3998 100644 --- a/tests/infra/clients.py +++ b/tests/infra/clients.py @@ -161,7 +161,7 @@ def __str__(self): if self.headers: string += f" {truncate(str(self.headers), max_len=25)}" if self.body is not None: - string += f' {truncate(f"{self.body}")}' + string += escape_loguru_tags(f' {truncate(f"{self.body}")}') return string diff --git a/tests/infra/github.py b/tests/infra/github.py index 8b761419128d..1c65c404a62a 100644 --- a/tests/infra/github.py +++ b/tests/infra/github.py @@ -197,12 +197,12 @@ def local_branch(): if tag_or_none: return tag_or_none - # ADO + # ADO or GHA short = None if "SYSTEM_PULLREQUEST_SOURCEBRANCH" in os.environ: short = os.environ["SYSTEM_PULLREQUEST_SOURCEBRANCH"] else: - ref = os.environ["BUILD_SOURCEBRANCH"] + ref = os.environ.get("BUILD_SOURCEBRANCH") or os.environ.get("GITHUB_REF") for prefix in ["refs/heads/", "refs/tags/", "refs/pull/"]: if ref.startswith(prefix): short = ref[len(prefix) :] diff --git a/tests/infra/interfaces.py b/tests/infra/interfaces.py index 6041ac051f81..7a9086aca05a 100644 --- a/tests/infra/interfaces.py +++ b/tests/infra/interfaces.py @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -from dataclasses import dataclass, asdict -from typing import Optional, Dict -from enum import Enum, auto +from dataclasses import dataclass, asdict, field +from typing import Optional, Dict, Union +from enum import Enum import urllib.parse from loguru import logger as LOG @@ -41,11 +41,11 @@ def make_address(host, port=0): NODE_TO_NODE_INTERFACE_NAME = "node_to_node_interface" -class EndorsementAuthority(Enum): - Service = auto() - Node = auto() - ACME = auto() - Unsecured = auto() +class EndorsementAuthority(str, Enum): + Service = "Service" + Node = "Node" + ACME = "ACME" + Unsecured = "Unsecured" @dataclass @@ -64,8 +64,8 @@ def to_json(endorsement): @staticmethod def from_json(json): endorsement = Endorsement() - endorsement.authority = json["authority"] - endorsement.acme_configuration = json["acme_configuration"] + endorsement.authority = EndorsementAuthority(json["authority"]) + endorsement.acme_configuration = json.get("acme_configuration", None) return endorsement @@ -75,14 +75,28 @@ class Interface: port: int = 0 -class RedirectionResolver: - pass +class NodeRole(str, Enum): + primary = "primary" + backup = "backup" @dataclass -class NodeByRoleResolver(RedirectionResolver): +class TargetRole: + role: NodeRole + + @staticmethod + def to_json(tr): + return asdict(tr) + + @staticmethod + def from_json(json): + return TargetRole(role=NodeRole(json["role"])) + + +@dataclass +class NodeByRoleResolver: + target: TargetRole = TargetRole(role=NodeRole.primary) kind: str = "NodeByRole" - target = {"role": "primary"} @staticmethod def to_json(nbrr): @@ -91,16 +105,14 @@ def to_json(nbrr): @staticmethod def from_json(json): nbrr = NodeByRoleResolver() - nbrr.target = json["target"] + nbrr.target = TargetRole.from_json(json["target"]) return nbrr -class StaticAddressResolver(RedirectionResolver): - kind: str = "StaticAddress" +@dataclass +class StaticAddressResolver: target_address: str - - def __init__(self, address): - self.target_address = address + kind: str = "StaticAddress" @staticmethod def to_json(sar): @@ -111,25 +123,45 @@ def to_json(sar): @staticmethod def from_json(json): - sar = StaticAddressResolver() - sar.target_address = json["target"]["address"] - return sar + return StaticAddressResolver(target_address=json["target"]["address"]) + + +RedirectionResolver = Union[NodeByRoleResolver, StaticAddressResolver] @dataclass class RedirectionConfig: to_primary: RedirectionResolver = NodeByRoleResolver() + to_backup: RedirectionResolver = NodeByRoleResolver( + target=TargetRole(role=NodeRole.backup) + ) @staticmethod def to_json(rc): - return {"to_primary": rc.to_primary.to_json(rc.to_primary)} + return { + "to_primary": rc.to_primary.to_json(rc.to_primary), + "to_backup": rc.to_backup.to_json(rc.to_backup), + } @staticmethod def from_json(json): - if json["kind"] == "NodeByRole": - return NodeByRoleResolver.from_json(json) - elif json["kind"] == "StaticAddress": - return StaticAddressResolver.from_json(json) + def resolver_from_json(obj): + if obj["kind"] == "NodeByRole": + return NodeByRoleResolver.from_json(obj) + elif obj["kind"] == "StaticAddress": + return StaticAddressResolver.from_json(obj) + + rc = RedirectionConfig() + + tp = json.get("to_primary", None) + if tp: + rc.to_primary = resolver_from_json(tp) + + tb = json.get("to_backup", None) + if tb: + rc.to_backup = resolver_from_json(tb) + + return rc @dataclass @@ -153,7 +185,7 @@ class RPCInterface(Interface): endorsement: Optional[Endorsement] = Endorsement() acme_configuration: Optional[str] = None accepted_endpoints: Optional[str] = None - forwarding_timeout_ms: Optional[int] = None + forwarding_timeout_ms: Optional[int] = DEFAULT_FORWARDING_TIMEOUT_MS redirections: Optional[RedirectionConfig] = None app_protocol: str = "HTTP1" @@ -202,12 +234,15 @@ def to_json(interface): "bind_address": make_address(interface.host, interface.port), "protocol": f"{interface.transport}", "app_protocol": interface.app_protocol, - "published_address": f"{interface.public_host}:{interface.public_port or 0}", "max_open_sessions_soft": interface.max_open_sessions_soft, "max_open_sessions_hard": interface.max_open_sessions_hard, "http_configuration": http_config, "endorsement": Endorsement.to_json(interface.endorsement), } + if interface.public_host: + r["published_address"] = ( + f"{interface.public_host}:{interface.public_port or 0}" + ) if interface.accepted_endpoints: r["accepted_endpoints"] = interface.accepted_endpoints if interface.forwarding_timeout_ms: @@ -256,7 +291,9 @@ def make_secondary_interface(transport="tcp", interface_name=SECONDARY_RPC_INTER @dataclass class HostSpec: - rpc_interfaces: Dict[str, RPCInterface] = RPCInterface() + rpc_interfaces: Dict[str, RPCInterface] = field( + default_factory=lambda: {PRIMARY_RPC_INTERFACE: RPCInterface()} + ) acme_challenge_server_interface: Optional[str] = None def get_primary_interface(self): @@ -277,3 +314,23 @@ def from_json(rpc_interfaces_json): for name, rpc_interface in rpc_interfaces_json.items() } ) + + +if __name__ == "__main__": + # Test some roundtrip conversions + def test_roundtrip(before): + j = before.to_json(before) + after = before.from_json(j) + assert before == after, f"Inaccurate JSON roundtrip:\n {before}\n!=\n {after}" + + rc = RedirectionConfig() + test_roundtrip(rc) + + rc.to_primary = StaticAddressResolver("1.2.3.4") + test_roundtrip(rc) + + rc.to_backup = NodeByRoleResolver(target=TargetRole(NodeRole.backup)) + test_roundtrip(rc) + + hc = HostSpec() + test_roundtrip(hc) diff --git a/tests/infra/remote.py b/tests/infra/remote.py index d8b320edabab..4a8f9c698136 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -379,6 +379,15 @@ def __init__( self.name = name self.out = os.path.join(self.root, "out") self.err = os.path.join(self.root, "err") + self._shutdown_timeout = 10 + + @property + def shutdown_timeout(self): + return self._shutdown_timeout + + @shutdown_timeout.setter + def shutdown_timeout(self, value): + self._shutdown_timeout = value @staticmethod def make_host(host): @@ -515,12 +524,11 @@ def stop(self): LOG.info("[{}] closing".format(self.hostname)) if self.proc: self.proc.terminate() - timeout = 10 try: - self.proc.wait(timeout) + self.proc.wait(self._shutdown_timeout) except subprocess.TimeoutExpired: LOG.exception( - f"Process didn't finish within {timeout} seconds. Trying to get stack trace..." + f"Process didn't finish within {self._shutdown_timeout} seconds. Trying to get stack trace..." ) self._print_stack_trace() raise diff --git a/tests/js-custom-authorization/custom_authorization.py b/tests/js-custom-authorization/custom_authorization.py index 0448237ba21c..36b5bf894e23 100644 --- a/tests/js-custom-authorization/custom_authorization.py +++ b/tests/js-custom-authorization/custom_authorization.py @@ -770,10 +770,6 @@ def test_accept_header(network, args): assert r.status_code == http.HTTPStatus.OK.value assert r.headers["content-type"] == "application/json" - r = c.get("/node/commit", headers={"accept": "application/msgpack"}) - assert r.status_code == http.HTTPStatus.OK.value - assert r.headers["content-type"] == "application/msgpack" - r = c.get( "/node/commit", headers={"accept": "text/html;q=0.9,image/jpeg;video/mpeg;q=0.8,*/*;q=0.1"}, @@ -789,15 +785,6 @@ def test_accept_header(network, args): assert r.status_code == http.HTTPStatus.OK.value assert r.headers["content-type"] == "application/json" - r = c.get( - "/node/commit", - headers={ - "accept": "text/html;q=0.9,image/jpeg;video/mpeg;q=0.8,application/msgpack;q=0.1" - }, - ) - assert r.status_code == http.HTTPStatus.OK.value - assert r.headers["content-type"] == "application/msgpack" - return network @@ -1310,28 +1297,28 @@ def run_interpreter_reuse(args): if __name__ == "__main__": cr = ConcurrentRunner() - cr.add( - "authz", - run, - nodes=infra.e2e_args.nodes(cr.args, 1), - js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-custom-authorization"), - ) - - cr.add( - "limits", - run_limits, - nodes=infra.e2e_args.nodes(cr.args, 1), - js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-limits"), - ) - - cr.add( - "authn", - run_authn, - nodes=infra.e2e_args.nodes(cr.args, 1), - js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-authentication"), - initial_user_count=4, - initial_member_count=2, - ) + # cr.add( + # "authz", + # run, + # nodes=infra.e2e_args.nodes(cr.args, 1), + # js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-custom-authorization"), + # ) + + # cr.add( + # "limits", + # run_limits, + # nodes=infra.e2e_args.nodes(cr.args, 1), + # js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-limits"), + # ) + + # cr.add( + # "authn", + # run_authn, + # nodes=infra.e2e_args.nodes(cr.args, 1), + # js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-authentication"), + # initial_user_count=4, + # initial_member_count=2, + # ) cr.add( "content_types", @@ -1340,18 +1327,18 @@ def run_interpreter_reuse(args): js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-content-types"), ) - cr.add( - "api", - run_api, - nodes=infra.e2e_args.nodes(cr.args, 1), - js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-api"), - ) - - cr.add( - "interpreter_reuse", - run_interpreter_reuse, - nodes=infra.e2e_args.min_nodes(cr.args, f=1), - js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-interpreter-reuse"), - ) + # cr.add( + # "api", + # run_api, + # nodes=infra.e2e_args.nodes(cr.args, 1), + # js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-api"), + # ) + + # cr.add( + # "interpreter_reuse", + # run_interpreter_reuse, + # nodes=infra.e2e_args.min_nodes(cr.args, f=1), + # js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-interpreter-reuse"), + # ) cr.run() diff --git a/tests/perf-system/submitter/submit.cpp b/tests/perf-system/submitter/submit.cpp index 96e51a01fdd5..1d73a5b60ced 100644 --- a/tests/perf-system/submitter/submit.cpp +++ b/tests/perf-system/submitter/submit.cpp @@ -25,7 +25,7 @@ using namespace std; using namespace client; -crypto::Pem key = {}; +ccf::crypto::Pem key = {}; std::string key_id = "Invalid"; std::shared_ptr<::tls::Cert> tls_cert = nullptr; @@ -153,14 +153,14 @@ std::shared_ptr create_connection( const auto raw_key = files::slurp(certificates[1].c_str()); const auto ca = files::slurp_string(certificates[2].c_str()); - key = crypto::Pem(raw_key); + key = ccf::crypto::Pem(raw_key); - const crypto::Pem cert_pem(raw_cert); - auto cert_der = crypto::cert_pem_to_der(cert_pem); - key_id = crypto::Sha256Hash(cert_der).hex_str(); + const ccf::crypto::Pem cert_pem(raw_cert); + auto cert_der = ccf::crypto::cert_pem_to_der(cert_pem); + key_id = ccf::crypto::Sha256Hash(cert_der).hex_str(); - tls_cert = - std::make_shared<::tls::Cert>(std::make_shared<::tls::CA>(ca), cert_pem, key); + tls_cert = std::make_shared<::tls::Cert>( + std::make_shared<::tls::CA>(ca), cert_pem, key); } const auto [host, port] = ccf::split_net_address(server_address); @@ -265,9 +265,9 @@ int main(int argc, char** argv) // Ignore SIGPIPE as it can be raised by write to a socket signal(SIGPIPE, SIG_IGN); - logger::config::default_init(); - logger::config::level() = LoggerLevel::INFO; - crypto::openssl_sha256_init(); + ccf::logger::config::default_init(); + ccf::logger::config::level() = LoggerLevel::INFO; + ccf::crypto::openssl_sha256_init(); CLI::App cli_app{"Perf Tool"}; ArgumentParser args("Perf Tool", cli_app); CLI11_PARSE(cli_app, argc, argv); @@ -381,7 +381,7 @@ int main(int argc, char** argv) } store_parquet_results(args, data_handler); - crypto::openssl_sha256_shutdown(); + ccf::crypto::openssl_sha256_shutdown(); return 0; } diff --git a/tests/programmability.py b/tests/programmability.py index 7f40b8d2d05c..d9e4f62f4363 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -10,6 +10,8 @@ import json from infra.runner import ConcurrentRunner from governance_js import action, proposal, ballot_yes +import ccf.cose +import infra.clients import npm_tests @@ -36,25 +38,19 @@ TESTJS_ROLE = """ export function content(request) { - let raw_id = ccf.strToBuf(request.caller.id); - let user_info = ccf.kv["public:ccf.gov.users.info"].get(raw_id); - if (user_info !== undefined) { - user_info = ccf.bufToJsonCompatible(user_info); - let roles = user_info?.user_data?.roles || []; - - for (const [_, role] of roles.entries()) { - let role_map = ccf.kv[`public:ccf.gov.roles.${role}`]; - let endpoint_name = request.url.split("/")[2]; - if (role_map?.has(ccf.strToBuf(`/${endpoint_name}/read`))) - { - return { - statusCode: 200, - body: { - payload: "Test content", - }, - }; - } - } + console.log(`my_constant is: ${my_object.my_constant}`); + + const endpoint_name = request.url.split("/")[2]; + const action_name = `/${endpoint_name}/read`; + const permitted = my_object.hasRole(request.caller.id, action_name); + if (permitted) + { + return { + statusCode: 200, + body: { + payload: "Test content", + }, + }; } return { @@ -84,6 +80,17 @@ def endpoint_properties( } +def sign_payload(identity, msg_type, json_payload): + serialised_payload = json.dumps(json_payload).encode() + key = open(identity.key, "r").read() + cert = open(identity.cert, "r").read() + phdr = { + "app.msg.type": msg_type, + "app.msg.created_at": int(infra.clients.get_clock().moment().timestamp()), + } + return ccf.cose.create_cose_sign1(serialised_payload, key, cert, phdr) + + def test_custom_endpoints(network, args): primary, _ = network.find_primary() user = network.users[0] @@ -141,10 +148,16 @@ def test_getters(c, expected_body): r.body.text() == module_def["module"] ), f"Expected:\n{module_def['module']}\n\n\nActual:\n{r.body.text()}" - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code - test_getters(c, bundle_with_content) # Install also works with cert authentication, at the expense of potential offline @@ -161,8 +174,15 @@ def test_getters(c, expected_body): assert r.status_code == http.HTTPStatus.OK.value, r.status_code assert r.body.json()["payload"] == "Test content", r.body.json() - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_other_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_other_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code test_getters(c, bundle_with_other_content) @@ -216,8 +236,15 @@ def test_custom_endpoints_circular_includes(network, args): "modules": modules, } - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=recursive_bundle) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", recursive_bundle + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code with primary.client() as c: @@ -265,8 +292,15 @@ def test_custom_endpoints_kv_restrictions(network, args): "modules": modules, } - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_content) + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_content + ) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code with primary.client() as c: @@ -333,8 +367,14 @@ def test_custom_endpoints_js_options(network, args): ) def test_options_patch(c, **kwargs): - r = c.call( - "/app/custom_endpoints/runtime_options", {**kwargs}, http_verb="PATCH" + signed_bundle = sign_payload( + network.identity(user.local_id), "runtime_options", {**kwargs} + ) + + r = c.patch( + "/app/custom_endpoints/runtime_options", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.OK.value, r.status_code new_options = r.body.json() @@ -347,7 +387,7 @@ def test_options_patch(c, **kwargs): assert new_options == get_options, f"{new_options} != {get_options}" return new_options - with primary.client(None, None, user.local_id) as c: + with primary.client(None, None) as c: r = c.get("/app/custom_endpoints/runtime_options") assert r.status_code == http.HTTPStatus.OK.value, r.status_code @@ -419,9 +459,16 @@ def test_custom_role_definitions(network, args): "modules": [{"name": "test.js", "module": TESTJS_ROLE}], } + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_auth + ) # Install app with auth/role support - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_auth) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Add role definition @@ -489,9 +536,16 @@ def test_custom_role_definitions(network, args): "modules": [{"name": "test.js", "module": TESTJS_ROLE}], } + signed_bundle = sign_payload( + network.identity(user.local_id), "custom_endpoints", bundle_with_auth_both + ) # Install two endpoints with role auth - with primary.client(None, None, user.local_id) as c: - r = c.put("/app/custom_endpoints", body=bundle_with_auth_both) + with primary.client() as c: + r = c.put( + "/app/custom_endpoints", + body=signed_bundle, + headers={"Content-Type": "application/cose"}, + ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code # Assign the new role to user0 @@ -525,10 +579,16 @@ def deploy_npm_app_custom(network, args): app_dir, "dist", "bundle.json" ) # Produced by build_npm_app - with primary.client(None, None, user.local_id) as c: + signed_bundle = sign_payload( + network.identity(user.local_id), + "custom_endpoints", + json.load(open(bundle_path)), + ) + with primary.client() as c: r = c.put( "/app/custom_endpoints", - body=json.load(open(bundle_path)), + body=signed_bundle, + headers={"Content-Type": "application/cose"}, ) assert r.status_code == http.HTTPStatus.NO_CONTENT.value, r.status_code @@ -556,6 +616,7 @@ def run(args): network = test_custom_endpoints_circular_includes(network, args) network = test_custom_endpoints_kv_restrictions(network, args) network = test_custom_role_definitions(network, args) + network = test_custom_endpoints_js_options(network, args) network = npm_tests.build_npm_app(network, args) network = deploy_npm_app_custom(network, args) diff --git a/tests/redirects.py b/tests/redirects.py index 624889fc0161..dc58d6001c7f 100644 --- a/tests/redirects.py +++ b/tests/redirects.py @@ -16,7 +16,7 @@ def test_redirects_with_node_role_config(network, args): msg = "Redirect test" req = {"id": 42, "msg": msg} - def test_redirect_to_node(talk_to, redirect_to): + def test_redirect_to_primary(talk_to, redirect_to): interface = redirect_to.host.rpc_interfaces[ infra.interfaces.PRIMARY_RPC_INTERFACE ] @@ -33,6 +33,34 @@ def test_redirect_to_node(talk_to, redirect_to): r = c.get(f"{path}?id={req['id']}", allow_redirects=False) assert r.status_code == http.HTTPStatus.OK + r = c.get(f"{path}/backup?id={req['id']}", allow_redirects=False) + assert r.status_code == http.HTTPStatus.OK + + def assert_redirect_to_backup(response, path): + primary, _ = network.find_nodes() + assert response.status_code == http.HTTPStatus.TEMPORARY_REDIRECT.value + assert "location" in response.headers + loc = response.headers["location"] + assert loc.endswith(path), response.headers + assert ( + primary.host.rpc_interfaces[ + infra.interfaces.PRIMARY_RPC_INTERFACE + ].public_host + not in loc + ), response.headers + # Don't use find_backups, because it will only return non-stopped nodes. + # The service doesn't know about that, so may redirect to a dead (still trusted) node! + backups = [n for n in network.nodes if n != primary] + for backup in backups: + interface = backup.host.rpc_interfaces[ + infra.interfaces.PRIMARY_RPC_INTERFACE + ] + b_loc = f"https://{interface.public_host}:{interface.public_port}" + if loc.startswith(b_loc): + break + else: + assert False, "Redirect header doesn't point to a backup?" + primary, orig_backups = network.find_nodes() LOG.info("Write initial values") @@ -46,14 +74,28 @@ def test_redirect_to_node(talk_to, redirect_to): LOG.info("Redirect to original primary") for backup in orig_backups: - test_redirect_to_node(backup, primary) + test_redirect_to_primary(backup, primary) + + LOG.info("Redirect from primary to backup") + with primary.client("user0") as c: + for path in paths: + full_path = f"{path}/backup?id={req['id']}" + r = c.get(full_path, allow_redirects=False) + assert_redirect_to_backup(r, full_path) LOG.info("Redirect to subsequent primary") primary.stop() network.wait_for_new_primary(primary) new_primary, new_backups = network.find_nodes() for backup in new_backups: - test_redirect_to_node(backup, new_primary) + test_redirect_to_primary(backup, new_primary) + + LOG.info("Redirect from subsequent primary to backup") + with new_primary.client("user0") as c: + for path in paths: + full_path = f"{path}/backup?id={req['id']}" + r = c.get(full_path, allow_redirects=False) + assert_redirect_to_backup(r, full_path) LOG.info("Subsequent primary no longer redirects") assert new_primary in orig_backups # Check it WAS a backup @@ -62,7 +104,7 @@ def test_redirect_to_node(talk_to, redirect_to): r = c.post(path, req, allow_redirects=False) assert r.status_code == http.HTTPStatus.OK.value - LOG.info("Redirects fail when no primary available") + LOG.info("to_primary redirects fail when no primary available") new_primary.stop() backup = new_backups[0] start_time = time.time() @@ -83,34 +125,73 @@ def test_redirect_to_node(talk_to, redirect_to): assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value assert r.body.json()["error"]["code"] == "PrimaryNotFound" + # to_backup redirects continue to execute locally + r = c.get(f"{path}/backup?id={req['id']}", allow_redirects=False) + assert r.status_code == http.HTTPStatus.OK.value + def test_redirects_with_static_name_config(network, args): - hostname = "primary.my.ccf.service.example.test" + primary_hostname = "primary.my.ccf.service.example.test" + backup_hostname = "backup.my.ccf.service.example.test" paths = ("/app/log/private", "/app/log/public") msg = "Redirect test" - new_node = network.create_node( - infra.interfaces.HostSpec( - rpc_interfaces={ - infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface( - host=infra.net.expand_localhost(), - redirections=infra.interfaces.RedirectionConfig( - to_primary=infra.interfaces.StaticAddressResolver(hostname) - ), - ) - } - ) + host_spec = infra.interfaces.HostSpec( + rpc_interfaces={ + infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface( + host=infra.net.expand_localhost(), + redirections=infra.interfaces.RedirectionConfig( + to_primary=infra.interfaces.StaticAddressResolver(primary_hostname), + to_backup=infra.interfaces.StaticAddressResolver(backup_hostname), + ), + ) + } ) + + original, _ = network.find_primary() + + new_node = network.create_node(host_spec) network.join_node(new_node, args.package, args) network.trust_node(new_node, args) + req = {"id": 42, "msg": msg} + + LOG.info( + "Add a node with static address redirect config, check its redirects to primary" + ) with new_node.client("user0") as c: for path in paths: - r = c.post(path, {"id": 42, "msg": msg}, allow_redirects=False) + r = c.post(path, req, allow_redirects=False) + assert r.status_code == http.HTTPStatus.TEMPORARY_REDIRECT.value + assert "location" in r.headers + assert ( + r.headers["location"] == f"https://{primary_hostname}{path}" + ), r.headers + + LOG.info("Add 2 more nodes with static address redirect config") + for _ in range(2): + other_node = network.create_node(host_spec) + network.join_node(other_node, args.package, args) + network.trust_node(other_node, args) + + LOG.info( + "Remove original node, so remaining network all have static address redirect config" + ) + network.retire_node(original, original) + original.stop() + + LOG.info("Test to_backup config with static address") + primary, _ = network.find_primary() + with primary.client("user0") as c: + for path in paths: + full_path = f"{path}/backup?id={req['id']}" + r = c.get(full_path, allow_redirects=False) assert r.status_code == http.HTTPStatus.TEMPORARY_REDIRECT.value assert "location" in r.headers - assert r.headers["location"] == f"https://{hostname}{path}", r.headers + assert ( + r.headers["location"] == f"https://{backup_hostname}{full_path}" + ), r.headers def run_redirect_tests_role(args):