Upgrade to CCF 4.0.7 (#218)

* Bump ccf versions to 4.0.7

* Add more to dockerignore

* Fix shadowed variables

* Updated some nix config

* Fix hidden files

* Use right compiler in sgx build

* Fix compiler in makefile

* Update clang-format

* Use clang-format 11

* Update checks container

* Use normal nixpkgs

* Enable ccf patches

* Build virtual with llvm 16

* Add ccf logger no deprecate

* Fix build of protobuf

* Enable mkWrapper again

* Fix formatting

* Fix prettier fix command in nix

* Format with prettier

* Remove unused arrow dep

* Back to openssl 1.1 for az-dcap

* Fix lskv cluster startup

* Add date to python lib cose governance

* Remove proposal replays

* Update ci image

* Format lskv_cluster script

* Fix tempfile import

---------

Co-authored-by: Andrew Jeffery <[email protected]>

.devcontainer.json

@@ -1,7 +1,7 @@
 {
   "name": "Development Container for CCF C++ Apps",
   "context": "..",
-  "image": "mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual",
+  "image": "mcr.microsoft.com/ccf/app/dev:4.0.7-virtual",
   "runArgs": [],
   "extensions": ["ms-vscode.cpptools"]
 }

.dockerignore

@@ -36,3 +36,5 @@
 /scripts/env/
 /workspace/
 requirements.txt
+*.zip
+*.bak

.github/workflows/benchmark.yml

@@ -11,7 +11,7 @@ concurrency:
 jobs:
   benchmark:
     runs-on: ubuntu-20.04
-    container: mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual
+    container: mcr.microsoft.com/ccf/app/dev:4.0.7-virtual
 
     steps:
       - name: Checkout repository

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ env:
 jobs:
   build-and-test:
     runs-on: ubuntu-20.04
-    container: mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual
+    container: mcr.microsoft.com/ccf/app/dev:4.0.7-virtual
 
     steps:
       - name: Checkout repository
@@ -72,7 +72,7 @@ jobs:
 
   checks:
     runs-on: ubuntu-latest
-    container: ccfmsrc.azurecr.io/ccf/ci/sgx:oe-0.18.2-protoc
+    container: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15
 
     steps:
       - name: Make sure github workspace is git safe

CMakeLists.txt

@@ -41,6 +41,10 @@ option(PUBLIC_LEASES
 option(VERBOSE_LOGGING "enable verbose logging" OFF)
 
 add_compile_definitions(LSKV_VERSION="${LSKV_VERSION}")
+add_compile_definitions(CCF_LOGGER_NO_DEPRECATE)
+# work around an issue in outdated protobuf from CCF
+# https://github.com/protocolbuffers/protobuf/issues/10108
+add_compile_definitions(GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL_INLINE=0)
 
 add_ccf_app(
   lskv

Dockerfile.sgx

@@ -2,14 +2,14 @@
 # Licensed under the MIT License.
 
 # Build
-FROM mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-sgx as builder
+FROM mcr.microsoft.com/ccf/app/dev:4.0.7-sgx as builder
 
 COPY . /src/
 WORKDIR /build
-RUN CC="/opt/oe_lvi/clang-10" CXX="/opt/oe_lvi/clang++-10" cmake -GNinja -DCOMPILE_TARGET=sgx /src && ninja
+RUN CC=$(command -v clang-11) CXX=$(command -v clang++-11) cmake -GNinja -DCOMPILE_TARGET=sgx /src && ninja
 
 # Run
-FROM mcr.microsoft.com/ccf/app/run:4.0.0-dev3-sgx
+FROM mcr.microsoft.com/ccf/app/run:4.0.7-sgx
 
 LABEL org.opencontainers.image.source=https://github.com/microsoft/lskv
 LABEL org.opencontainers.image.description="LSKV SGX node"

Dockerfile.virtual

@@ -2,14 +2,14 @@
 # Licensed under the MIT License.
 
 # Build
-FROM mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual as builder
+FROM mcr.microsoft.com/ccf/app/dev:4.0.7-virtual as builder
 
 COPY . /src/
 WORKDIR /build
-RUN CC=$(command -v clang-10) CXX=$(command -v clang++-10) cmake -GNinja -DCOMPILE_TARGET=virtual /src && ninja
+RUN CC=$(command -v clang-15) CXX=$(command -v clang++-15) cmake -GNinja -DCOMPILE_TARGET=virtual /src && ninja
 
 # Run
-FROM mcr.microsoft.com/ccf/app/run:4.0.0-dev3-virtual
+FROM mcr.microsoft.com/ccf/app/run:4.0.7-virtual
 
 LABEL org.opencontainers.image.source=https://github.com/microsoft/lskv
 LABEL org.opencontainers.image.description="LSKV virtual node"

Makefile

@@ -4,11 +4,11 @@ BUILD=build
 CCF_PREFIX_VIRTUAL=/opt/ccf_virtual
 CCF_PREFIX_SGX=/opt/ccf_sgx
 
-CC!=which clang-10
-CXX!=which clang++-10
+CC!=which clang-15
+CXX!=which clang++-15
 
-OE_CC=/opt/oe_lvi/clang-10
-OE_CXX=/opt/oe_lvi/clang++-10
+OE_CC!=which clang-11
+OE_CXX!=which clang++-11
 
 ETCD_VER=v3.5.4
 ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
@@ -18,10 +18,10 @@ H_FILES=$(wildcard src/**/*.h)
 
 BIN_DIR=bin
 
-CCF_VER=ccf-4.0.0-dev3
-CCF_VER_LOWER=ccf_virtual_4.0.0_dev3
-CCF_SGX_VER_LOWER=ccf_sgx_4.0.0_dev3
-CCF_SGX_UNSAFE_VER_LOWER=ccf_sgx_unsafe_4.0.0_dev3
+CCF_VER=ccf-4.0.7
+CCF_VER_LOWER=ccf_virtual_4.0.7
+CCF_SGX_VER_LOWER=ccf_sgx_4.0.7
+CCF_SGX_UNSAFE_VER_LOWER=ccf_sgx_unsafe_4.0.7
 
 .PHONY: install-ccf-virtual
 install-ccf-virtual:

README.md

@@ -31,10 +31,10 @@ make install-ccf-virtual
 Or
 
 ```bash
-wget https://github.com/microsoft/CCF/releases/download/ccf-4.0.0-dev3/ccf_virtual_4.0.0_dev3_amd64.deb
-sudo dpkg -i ccf_virtual_4.0.0_dev3_amd64.deb # Installs CCF under /opt/ccf_virtual
+wget https://github.com/microsoft/CCF/releases/download/ccf-4.0.7/ccf_virtual_4.0.7_amd64.deb
+sudo dpkg -i ccf_virtual_4.0.7_amd64.deb # Installs CCF under /opt/ccf_virtual
 cat /opt/ccf_virtual/share/VERSION_LONG
-# ccf-4.0.0-dev3
+# ccf-4.0.7
 /opt/ccf_virtual/getting_started/setup_vm/run.sh /opt/ccf_virtual/getting_started/setup_vm/app-dev.yml  # Install dependencies
 ```
 

benchmark/distributed/values.yaml

@@ -4,5 +4,5 @@
 # values for the distributed benchmarking setup
 vm_user: "apj39"
 
-py_ccf_ver: "4.0.0dev3"
-ccf_ver: "4.0.0-dev3"
+py_ccf_ver: "4.0.7"
+ccf_ver: "4.0.7"

benchmark/lskv_cluster.py

@@ -11,6 +11,7 @@
 import os
 import signal
 import subprocess
+import tempfile
 import time
 from dataclasses import dataclass
 from typing import Any, Dict, List
@@ -48,7 +49,7 @@ def __init__(self, address: str, cacert: str, cert: str, key: str):
         self.cert = cert
         self.key = key
 
-    def run(self, method: str, path: str) -> Any:
+    def run(self, method: str, path: str, data=None, content_type=None) -> Any:
         """
         Run a curl invocation.
         """
@@ -65,49 +66,57 @@ def run(self, method: str, path: str) -> Any:
             "--cert",
             self.cert,
         ]
+        if data:
+            cmd += ["--data-binary", data]
+        if content_type:
+            cmd += ["--header", f"content-type: {content_type}"]
         proc = run(cmd)
         out = proc.stdout.decode("utf-8")
         if out:
             return json.loads(out)
         return ""
 
+    def sign_and_send(
+        self, path: str, message_type: str, data: Any, proposal_id=None
+    ) -> Any:
+        """
+        Sign some data and post it.
+        """
+        date_proc = run(["date", "-Is"])
+        date = date_proc.stdout.decode("utf-8").strip()
+
+        with tempfile.NamedTemporaryFile(mode="w+") as data_file:
+            json.dump(data, data_file)
+            data_file.flush()
+
+            cmd = [
+                "ccf_cose_sign1",
+                "--ccf-gov-msg-type",
+                message_type,
+                "--ccf-gov-msg-created_at",
+                date,
+                "--signing-cert",
+                self.cert,
+                "--signing-key",
+                self.key,
+                "--content",
+                data_file.name,
+            ]
+            if proposal_id:
+                cmd += ["--ccf-gov-msg-proposal_id", proposal_id]
+            signed_proc = run(cmd)
 
-# pylint: disable=too-few-public-methods
-class SCurl:
-    """
-    Run SCurl commands.
-    """
+        with tempfile.NamedTemporaryFile(mode="wb+") as signed_data_file:
+            signed_data_file.write(signed_proc.stdout)
+            signed_data_file.flush()
 
-    def __init__(self, address: str, cacert: str, cert: str, key: str):
-        self.address = address
-        self.cacert = cacert
-        self.cert = cert
-        self.key = key
-
-    def run(self, path: str, json_data: Dict[str, Any]) -> Any:
-        """
-        Run an scurl invocation.
-        """
-        json_str = json.dumps(json_data)
-        cmd = [
-            "scurl.sh",
-            f"{self.address}{path}",
-            "--cacert",
-            self.cacert,
-            "--signing-key",
-            self.key,
-            "--signing-cert",
-            self.cert,
-            "--header",
-            "content-type: application/json",
-            "--data-binary",
-            json_str,
-        ]
-        proc = run(cmd)
-        out = proc.stdout.decode("utf-8")
-        if out:
-            return json.loads(out)
-        return ""
+            logger.info("Returning the signed data")
+            return self.run(
+                "POST",
+                path,
+                data=f"@{signed_data_file.name}",
+                content_type="application/cose",
+            )
 
 
 # pylint: disable=too-many-instance-attributes
@@ -509,17 +518,13 @@ class Member:
     def __init__(self, workspace: str, name: str):
         self.workspace = workspace
         self.name = name
+        self.public_key = f"{self.workspace}/sandbox_common/{name}_cert.pem"
+        self.private_key = f"{self.workspace}/sandbox_common/{name}_privk.pem"
         self.curl = Curl(
             "https://127.0.0.1:8000",
             f"{self.workspace}/sandbox_common/service_cert.pem",
-            f"{self.workspace}/sandbox_common/{name}_cert.pem",
-            f"{self.workspace}/sandbox_common/{name}_privk.pem",
-        )
-        self.scurl = SCurl(
-            "https://127.0.0.1:8000",
-            f"{self.workspace}/sandbox_common/service_cert.pem",
-            f"{self.workspace}/sandbox_common/{name}_cert.pem",
-            f"{self.workspace}/sandbox_common/{name}_privk.pem",
+            self.public_key,
+            self.private_key,
         )
 
     def activate_member(self):
@@ -534,8 +539,9 @@ def activate_member(self):
         logger.info("Getting latest state digest")
         state_digest = self.curl.run("POST", "/gov/ack/update_state_digest")
 
-        logger.info("Signing and returning the state digest")
-        self.scurl.run("/gov/ack", state_digest)
+        logger.info("Signing the state digest")
+        logger.info(state_digest)
+        self.curl.sign_and_send("/gov/ack", "ack", state_digest)
 
         logger.info("Listing members")
         self.curl.run("GET", "/gov/members")
@@ -555,14 +561,19 @@ def set_user(self, cert: str):
             ]
         }
         logger.info("Creating set_user proposal")
-        proposal = self.scurl.run("/gov/proposals", set_user)
+        proposal = self.curl.sign_and_send("/gov/proposals", "proposal", set_user)
         proposal_id = proposal["proposal_id"]
 
         logger.info("Accepting the proposal")
         vote_accept = {
             "ballot": "export function vote (proposal, proposerId) { return true }"
         }
-        self.scurl.run(f"/gov/proposals/{proposal_id}/ballots", vote_accept)
+        self.curl.sign_and_send(
+            f"/gov/proposals/{proposal_id}/ballots",
+            "ballot",
+            vote_accept,
+            proposal_id=proposal_id,
+        )
 
     def open_network(self):
         """
@@ -587,14 +598,21 @@ def open_network(self):
                 }
             ]
         }
-        proposal = self.scurl.run("/gov/proposals", transition_service_to_open)
+        proposal = self.curl.sign_and_send(
+            "/gov/proposals", "proposal", transition_service_to_open
+        )
         proposal_id = proposal["proposal_id"]
 
         logger.info("Accepting the proposal")
         vote_accept = {
             "ballot": "export function vote (proposal, proposerId) { return true }"
         }
-        self.scurl.run(f"/gov/proposals/{proposal_id}/ballots", vote_accept)
+        self.curl.sign_and_send(
+            f"/gov/proposals/{proposal_id}/ballots",
+            "ballot",
+            vote_accept,
+            proposal_id=proposal_id,
+        )
 
         logger.info("Network is now open to users!")