diff --git a/.devcontainer.json b/.devcontainer.json index d491478e..ae079868 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,7 +1,7 @@ { "name": "Development Container for CCF C++ Apps", "context": "..", - "image": "mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual", + "image": "mcr.microsoft.com/ccf/app/dev:4.0.7-virtual", "runArgs": [], "extensions": ["ms-vscode.cpptools"] } diff --git a/.dockerignore b/.dockerignore index 307b58e4..74f641ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -36,3 +36,5 @@ /scripts/env/ /workspace/ requirements.txt +*.zip +*.bak diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d5132572..8c03b9e4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,7 +11,7 @@ concurrency: jobs: benchmark: runs-on: ubuntu-20.04 - container: mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual + container: mcr.microsoft.com/ccf/app/dev:4.0.7-virtual steps: - name: Checkout repository diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2b38bbb..277772dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ env: jobs: build-and-test: runs-on: ubuntu-20.04 - container: mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual + container: mcr.microsoft.com/ccf/app/dev:4.0.7-virtual steps: - name: Checkout repository @@ -72,7 +72,7 @@ jobs: checks: runs-on: ubuntu-latest - container: ccfmsrc.azurecr.io/ccf/ci/sgx:oe-0.18.2-protoc + container: ccfmsrc.azurecr.io/ccf/ci:16-08-2023-1-virtual-clang15 steps: - name: Make sure github workspace is git safe diff --git a/CMakeLists.txt b/CMakeLists.txt index 77089a69..3262f227 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,10 @@ option(PUBLIC_LEASES option(VERBOSE_LOGGING "enable verbose logging" OFF) add_compile_definitions(LSKV_VERSION="${LSKV_VERSION}") +add_compile_definitions(CCF_LOGGER_NO_DEPRECATE) +# work around an issue in outdated protobuf from CCF +# https://github.com/protocolbuffers/protobuf/issues/10108 +add_compile_definitions(GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL_INLINE=0) add_ccf_app( lskv diff --git a/Dockerfile.sgx b/Dockerfile.sgx index e3cc6ea0..e4109473 100644 --- a/Dockerfile.sgx +++ b/Dockerfile.sgx @@ -2,14 +2,14 @@ # Licensed under the MIT License. # Build -FROM mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-sgx as builder +FROM mcr.microsoft.com/ccf/app/dev:4.0.7-sgx as builder COPY . /src/ WORKDIR /build -RUN CC="/opt/oe_lvi/clang-10" CXX="/opt/oe_lvi/clang++-10" cmake -GNinja -DCOMPILE_TARGET=sgx /src && ninja +RUN CC=$(command -v clang-11) CXX=$(command -v clang++-11) cmake -GNinja -DCOMPILE_TARGET=sgx /src && ninja # Run -FROM mcr.microsoft.com/ccf/app/run:4.0.0-dev3-sgx +FROM mcr.microsoft.com/ccf/app/run:4.0.7-sgx LABEL org.opencontainers.image.source=https://github.com/microsoft/lskv LABEL org.opencontainers.image.description="LSKV SGX node" diff --git a/Dockerfile.virtual b/Dockerfile.virtual index 06c28248..a1b27d8a 100644 --- a/Dockerfile.virtual +++ b/Dockerfile.virtual @@ -2,14 +2,14 @@ # Licensed under the MIT License. # Build -FROM mcr.microsoft.com/ccf/app/dev:4.0.0-dev3-virtual as builder +FROM mcr.microsoft.com/ccf/app/dev:4.0.7-virtual as builder COPY . /src/ WORKDIR /build -RUN CC=$(command -v clang-10) CXX=$(command -v clang++-10) cmake -GNinja -DCOMPILE_TARGET=virtual /src && ninja +RUN CC=$(command -v clang-15) CXX=$(command -v clang++-15) cmake -GNinja -DCOMPILE_TARGET=virtual /src && ninja # Run -FROM mcr.microsoft.com/ccf/app/run:4.0.0-dev3-virtual +FROM mcr.microsoft.com/ccf/app/run:4.0.7-virtual LABEL org.opencontainers.image.source=https://github.com/microsoft/lskv LABEL org.opencontainers.image.description="LSKV virtual node" diff --git a/Makefile b/Makefile index 09e7872a..1e5cee51 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,11 @@ BUILD=build CCF_PREFIX_VIRTUAL=/opt/ccf_virtual CCF_PREFIX_SGX=/opt/ccf_sgx -CC!=which clang-10 -CXX!=which clang++-10 +CC!=which clang-15 +CXX!=which clang++-15 -OE_CC=/opt/oe_lvi/clang-10 -OE_CXX=/opt/oe_lvi/clang++-10 +OE_CC!=which clang-11 +OE_CXX!=which clang++-11 ETCD_VER=v3.5.4 ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download @@ -18,10 +18,10 @@ H_FILES=$(wildcard src/**/*.h) BIN_DIR=bin -CCF_VER=ccf-4.0.0-dev3 -CCF_VER_LOWER=ccf_virtual_4.0.0_dev3 -CCF_SGX_VER_LOWER=ccf_sgx_4.0.0_dev3 -CCF_SGX_UNSAFE_VER_LOWER=ccf_sgx_unsafe_4.0.0_dev3 +CCF_VER=ccf-4.0.7 +CCF_VER_LOWER=ccf_virtual_4.0.7 +CCF_SGX_VER_LOWER=ccf_sgx_4.0.7 +CCF_SGX_UNSAFE_VER_LOWER=ccf_sgx_unsafe_4.0.7 .PHONY: install-ccf-virtual install-ccf-virtual: diff --git a/README.md b/README.md index c998e581..0ae5072c 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ make install-ccf-virtual Or ```bash -wget https://github.com/microsoft/CCF/releases/download/ccf-4.0.0-dev3/ccf_virtual_4.0.0_dev3_amd64.deb -sudo dpkg -i ccf_virtual_4.0.0_dev3_amd64.deb # Installs CCF under /opt/ccf_virtual +wget https://github.com/microsoft/CCF/releases/download/ccf-4.0.7/ccf_virtual_4.0.7_amd64.deb +sudo dpkg -i ccf_virtual_4.0.7_amd64.deb # Installs CCF under /opt/ccf_virtual cat /opt/ccf_virtual/share/VERSION_LONG -# ccf-4.0.0-dev3 +# ccf-4.0.7 /opt/ccf_virtual/getting_started/setup_vm/run.sh /opt/ccf_virtual/getting_started/setup_vm/app-dev.yml # Install dependencies ``` diff --git a/benchmark/distributed/values.yaml b/benchmark/distributed/values.yaml index 26c73664..e228da3d 100644 --- a/benchmark/distributed/values.yaml +++ b/benchmark/distributed/values.yaml @@ -4,5 +4,5 @@ # values for the distributed benchmarking setup vm_user: "apj39" -py_ccf_ver: "4.0.0dev3" -ccf_ver: "4.0.0-dev3" +py_ccf_ver: "4.0.7" +ccf_ver: "4.0.7" diff --git a/benchmark/lskv_cluster.py b/benchmark/lskv_cluster.py index 371faf19..8cfb448a 100755 --- a/benchmark/lskv_cluster.py +++ b/benchmark/lskv_cluster.py @@ -11,6 +11,7 @@ import os import signal import subprocess +import tempfile import time from dataclasses import dataclass from typing import Any, Dict, List @@ -48,7 +49,7 @@ def __init__(self, address: str, cacert: str, cert: str, key: str): self.cert = cert self.key = key - def run(self, method: str, path: str) -> Any: + def run(self, method: str, path: str, data=None, content_type=None) -> Any: """ Run a curl invocation. """ @@ -65,49 +66,57 @@ def run(self, method: str, path: str) -> Any: "--cert", self.cert, ] + if data: + cmd += ["--data-binary", data] + if content_type: + cmd += ["--header", f"content-type: {content_type}"] proc = run(cmd) out = proc.stdout.decode("utf-8") if out: return json.loads(out) return "" + def sign_and_send( + self, path: str, message_type: str, data: Any, proposal_id=None + ) -> Any: + """ + Sign some data and post it. + """ + date_proc = run(["date", "-Is"]) + date = date_proc.stdout.decode("utf-8").strip() + + with tempfile.NamedTemporaryFile(mode="w+") as data_file: + json.dump(data, data_file) + data_file.flush() + + cmd = [ + "ccf_cose_sign1", + "--ccf-gov-msg-type", + message_type, + "--ccf-gov-msg-created_at", + date, + "--signing-cert", + self.cert, + "--signing-key", + self.key, + "--content", + data_file.name, + ] + if proposal_id: + cmd += ["--ccf-gov-msg-proposal_id", proposal_id] + signed_proc = run(cmd) -# pylint: disable=too-few-public-methods -class SCurl: - """ - Run SCurl commands. - """ + with tempfile.NamedTemporaryFile(mode="wb+") as signed_data_file: + signed_data_file.write(signed_proc.stdout) + signed_data_file.flush() - def __init__(self, address: str, cacert: str, cert: str, key: str): - self.address = address - self.cacert = cacert - self.cert = cert - self.key = key - - def run(self, path: str, json_data: Dict[str, Any]) -> Any: - """ - Run an scurl invocation. - """ - json_str = json.dumps(json_data) - cmd = [ - "scurl.sh", - f"{self.address}{path}", - "--cacert", - self.cacert, - "--signing-key", - self.key, - "--signing-cert", - self.cert, - "--header", - "content-type: application/json", - "--data-binary", - json_str, - ] - proc = run(cmd) - out = proc.stdout.decode("utf-8") - if out: - return json.loads(out) - return "" + logger.info("Returning the signed data") + return self.run( + "POST", + path, + data=f"@{signed_data_file.name}", + content_type="application/cose", + ) # pylint: disable=too-many-instance-attributes @@ -509,17 +518,13 @@ class Member: def __init__(self, workspace: str, name: str): self.workspace = workspace self.name = name + self.public_key = f"{self.workspace}/sandbox_common/{name}_cert.pem" + self.private_key = f"{self.workspace}/sandbox_common/{name}_privk.pem" self.curl = Curl( "https://127.0.0.1:8000", f"{self.workspace}/sandbox_common/service_cert.pem", - f"{self.workspace}/sandbox_common/{name}_cert.pem", - f"{self.workspace}/sandbox_common/{name}_privk.pem", - ) - self.scurl = SCurl( - "https://127.0.0.1:8000", - f"{self.workspace}/sandbox_common/service_cert.pem", - f"{self.workspace}/sandbox_common/{name}_cert.pem", - f"{self.workspace}/sandbox_common/{name}_privk.pem", + self.public_key, + self.private_key, ) def activate_member(self): @@ -534,8 +539,9 @@ def activate_member(self): logger.info("Getting latest state digest") state_digest = self.curl.run("POST", "/gov/ack/update_state_digest") - logger.info("Signing and returning the state digest") - self.scurl.run("/gov/ack", state_digest) + logger.info("Signing the state digest") + logger.info(state_digest) + self.curl.sign_and_send("/gov/ack", "ack", state_digest) logger.info("Listing members") self.curl.run("GET", "/gov/members") @@ -555,14 +561,19 @@ def set_user(self, cert: str): ] } logger.info("Creating set_user proposal") - proposal = self.scurl.run("/gov/proposals", set_user) + proposal = self.curl.sign_and_send("/gov/proposals", "proposal", set_user) proposal_id = proposal["proposal_id"] logger.info("Accepting the proposal") vote_accept = { "ballot": "export function vote (proposal, proposerId) { return true }" } - self.scurl.run(f"/gov/proposals/{proposal_id}/ballots", vote_accept) + self.curl.sign_and_send( + f"/gov/proposals/{proposal_id}/ballots", + "ballot", + vote_accept, + proposal_id=proposal_id, + ) def open_network(self): """ @@ -587,14 +598,21 @@ def open_network(self): } ] } - proposal = self.scurl.run("/gov/proposals", transition_service_to_open) + proposal = self.curl.sign_and_send( + "/gov/proposals", "proposal", transition_service_to_open + ) proposal_id = proposal["proposal_id"] logger.info("Accepting the proposal") vote_accept = { "ballot": "export function vote (proposal, proposerId) { return true }" } - self.scurl.run(f"/gov/proposals/{proposal_id}/ballots", vote_accept) + self.curl.sign_and_send( + f"/gov/proposals/{proposal_id}/ballots", + "ballot", + vote_accept, + proposal_id=proposal_id, + ) logger.info("Network is now open to users!") diff --git a/constitution/actions.js b/constitution/actions.js index 199fdc24..d20b9db4 100644 --- a/constitution/actions.js +++ b/constitution/actions.js @@ -11,7 +11,7 @@ class Action { function parseUrl(url) { // From https://tools.ietf.org/html/rfc3986#appendix-B const re = new RegExp( - "^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?" + "^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?", ); const groups = url.match(re); if (!groups) { @@ -149,7 +149,7 @@ function checkJwks(value, field) { function checkX509CertBundle(value, field) { if (!ccf.isValidX509CertBundle(value)) { throw new Error( - `${field} must be a valid X509 certificate (bundle) in PEM format` + `${field} must be a valid X509 certificate (bundle) in PEM format`, ); } } @@ -170,7 +170,7 @@ function invalidateOtherOpenProposals(proposalIdToRetain) { function setServiceCertificateValidityPeriod(validFrom, validityPeriodDays) { const rawConfig = ccf.kv["public:ccf.gov.service.config"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawConfig === undefined) { throw new Error("Service configuration could not be found"); @@ -187,13 +187,13 @@ function setServiceCertificateValidityPeriod(validFrom, validityPeriodDays) { validityPeriodDays > max_allowed_cert_validity_period_days ) { throw new Error( - `Validity period ${validityPeriodDays} (days) is not allowed: service max allowed is ${max_allowed_cert_validity_period_days} (days)` + `Validity period ${validityPeriodDays} (days) is not allowed: service max allowed is ${max_allowed_cert_validity_period_days} (days)`, ); } const renewed_service_certificate = ccf.network.generateNetworkCertificate( validFrom, - validityPeriodDays ?? max_allowed_cert_validity_period_days + validityPeriodDays ?? max_allowed_cert_validity_period_days, ); const serviceInfoTable = "public:ccf.gov.service.info"; @@ -206,7 +206,7 @@ function setServiceCertificateValidityPeriod(validFrom, validityPeriodDays) { serviceInfo.cert = renewed_service_certificate; ccf.kv[serviceInfoTable].set( getSingletonKvKey(), - ccf.jsonCompatibleToBuf(serviceInfo) + ccf.jsonCompatibleToBuf(serviceInfo), ); } @@ -214,14 +214,14 @@ function setNodeCertificateValidityPeriod( nodeId, nodeInfo, validFrom, - validityPeriodDays + validityPeriodDays, ) { if (nodeInfo.certificate_signing_request === undefined) { throw new Error(`Node ${nodeId} has no certificate signing request`); } const rawConfig = ccf.kv["public:ccf.gov.service.config"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawConfig === undefined) { throw new Error("Service configuration could not be found"); @@ -238,18 +238,18 @@ function setNodeCertificateValidityPeriod( validityPeriodDays > max_allowed_cert_validity_period_days ) { throw new Error( - `Validity period ${validityPeriodDays} (days) is not allowed: service max allowed is ${max_allowed_cert_validity_period_days} (days)` + `Validity period ${validityPeriodDays} (days) is not allowed: service max allowed is ${max_allowed_cert_validity_period_days} (days)`, ); } const endorsed_node_cert = ccf.network.generateEndorsedCertificate( nodeInfo.certificate_signing_request, validFrom, - validityPeriodDays ?? max_allowed_cert_validity_period_days + validityPeriodDays ?? max_allowed_cert_validity_period_days, ); ccf.kv["public:ccf.gov.nodes.endorsed_certificates"].set( ccf.strToBuf(nodeId), - ccf.strToBuf(endorsed_node_cert) + ccf.strToBuf(endorsed_node_cert), ); } @@ -270,13 +270,13 @@ function checkRecoveryThreshold(config, new_config) { if (service.status === "WaitingForRecoveryShares") { throw new Error( - `Cannot set recovery threshold if service is ${service.status}` + `Cannot set recovery threshold if service is ${service.status}`, ); } else if (service.status === "Open") { let activeRecoveryMembersCount = getActiveRecoveryMembersCount(); if (new_config.recovery_threshold > activeRecoveryMembersCount) { throw new Error( - `Cannot set recovery threshold to ${new_config.recovery_threshold}: recovery threshold would be greater than the number of recovery members ${activeRecoveryMembersCount}` + `Cannot set recovery threshold to ${new_config.recovery_threshold}: recovery threshold would be greater than the number of recovery members ${activeRecoveryMembersCount}`, ); } } @@ -293,7 +293,7 @@ function checkReconfigurationType(config, new_config) { ) ) { throw new Error( - `Cannot change reconfiguration type from ${from} to ${to}.` + `Cannot change reconfiguration type from ${from} to ${to}.`, ); } } @@ -327,7 +327,7 @@ function updateServiceConfig(new_config) { ccf.kv[service_config_table].set( getSingletonKvKey(), - ccf.jsonCompatibleToBuf(config) + ccf.jsonCompatibleToBuf(config), ); if (need_recovery_threshold_refresh) { @@ -360,12 +360,12 @@ const actions = new Map([ function (args, proposalId) { ccf.kv["public:ccf.gov.constitution"].set( getSingletonKvKey(), - ccf.jsonCompatibleToBuf(args.constitution) + ccf.jsonCompatibleToBuf(args.constitution), ); // Changing the constitution changes the semantics of any other open proposals, so invalidate them to avoid confusion or malicious vote modification invalidateOtherOpenProposals(proposalId); - } + }, ), ], [ @@ -383,17 +383,17 @@ const actions = new Map([ ccf.kv["public:ccf.gov.members.certs"].set( rawMemberId, - ccf.strToBuf(args.cert) + ccf.strToBuf(args.cert), ); if (args.encryption_pub_key == null) { ccf.kv["public:ccf.gov.members.encryption_public_keys"].delete( - rawMemberId + rawMemberId, ); } else { ccf.kv["public:ccf.gov.members.encryption_public_keys"].set( rawMemberId, - ccf.strToBuf(args.encryption_pub_key) + ccf.strToBuf(args.encryption_pub_key), ); } @@ -402,11 +402,11 @@ const actions = new Map([ member_info.status = "Accepted"; ccf.kv["public:ccf.gov.members.info"].set( rawMemberId, - ccf.jsonCompatibleToBuf(member_info) + ccf.jsonCompatibleToBuf(member_info), ); const rawSignature = ccf.kv["public:ccf.internal.signatures"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawSignature === undefined) { ccf.kv["public:ccf.gov.members.acks"].set(rawMemberId); @@ -416,10 +416,10 @@ const actions = new Map([ ack.state_digest = signature.root; ccf.kv["public:ccf.gov.members.acks"].set( rawMemberId, - ccf.jsonCompatibleToBuf(ack) + ccf.jsonCompatibleToBuf(ack), ); } - } + }, ), ], [ @@ -450,7 +450,7 @@ const actions = new Map([ // to recover the service if (isActiveMember && isRecoveryMember) { const rawConfig = ccf.kv["public:ccf.gov.service.config"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawConfig === undefined) { throw new Error("Service configuration could not be found"); @@ -461,14 +461,14 @@ const actions = new Map([ getActiveRecoveryMembersCount() - 1; if (activeRecoveryMembersCountAfter < config.recovery_threshold) { throw new Error( - `Number of active recovery members (${activeRecoveryMembersCountAfter}) would be less than recovery threshold (${config.recovery_threshold})` + `Number of active recovery members (${activeRecoveryMembersCountAfter}) would be less than recovery threshold (${config.recovery_threshold})`, ); } } ccf.kv["public:ccf.gov.members.info"].delete(rawMemberId); ccf.kv["public:ccf.gov.members.encryption_public_keys"].delete( - rawMemberId + rawMemberId, ); ccf.kv["public:ccf.gov.members.certs"].delete(rawMemberId); ccf.kv["public:ccf.gov.members.acks"].delete(rawMemberId); @@ -480,7 +480,7 @@ const actions = new Map([ // remaining active recovery members ccf.node.triggerLedgerRekey(); } - } + }, ), ], [ @@ -501,7 +501,7 @@ const actions = new Map([ let mi = ccf.bufToJsonCompatible(member_info); mi.member_data = args.member_data; members_info.set(member_id, ccf.jsonCompatibleToBuf(mi)); - } + }, ), ], [ @@ -517,7 +517,7 @@ const actions = new Map([ ccf.kv["public:ccf.gov.users.certs"].set( rawUserId, - ccf.strToBuf(args.cert) + ccf.strToBuf(args.cert), ); if (args.user_data !== null && args.user_data !== undefined) { @@ -525,12 +525,12 @@ const actions = new Map([ userInfo.user_data = args.user_data; ccf.kv["public:ccf.gov.users.info"].set( rawUserId, - ccf.jsonCompatibleToBuf(userInfo) + ccf.jsonCompatibleToBuf(userInfo), ); } else { ccf.kv["public:ccf.gov.users.info"].delete(rawUserId); } - } + }, ), ], [ @@ -543,7 +543,7 @@ const actions = new Map([ const user_id = ccf.strToBuf(args.user_id); ccf.kv["public:ccf.gov.users.certs"].delete(user_id); ccf.kv["public:ccf.gov.users.info"].delete(user_id); - } + }, ), ], [ @@ -561,12 +561,12 @@ const actions = new Map([ userInfo.user_data = args.user_data; ccf.kv["public:ccf.gov.users.info"].set( userId, - ccf.jsonCompatibleToBuf(userInfo) + ccf.jsonCompatibleToBuf(userInfo), ); } else { ccf.kv["public:ccf.gov.users.info"].delete(userId); } - } + }, ), ], [ @@ -578,7 +578,7 @@ const actions = new Map([ }, function (args) { updateServiceConfig(args); - } + }, ), ], [ @@ -589,7 +589,7 @@ const actions = new Map([ }, function (args) { ccf.node.triggerRecoverySharesRefresh(); - } + }, ), ], [ @@ -601,7 +601,7 @@ const actions = new Map([ function (args) { ccf.node.triggerLedgerRekey(); - } + }, ), ], [ @@ -611,22 +611,22 @@ const actions = new Map([ checkType( args.next_service_identity, "string", - "next service identity (PEM certificate)" + "next service identity (PEM certificate)", ); checkX509CertBundle( args.next_service_identity, - "next_service_identity" + "next_service_identity", ); checkType( args.previous_service_identity, "string?", - "previous service identity (PEM certificate)" + "previous service identity (PEM certificate)", ); if (args.previous_service_identity !== undefined) { checkX509CertBundle( args.previous_service_identity, - "previous_service_identity" + "previous_service_identity", ); } }, @@ -646,7 +646,7 @@ const actions = new Map([ args.next_service_identity === undefined) ) { throw new Error( - `Opening a recovering network requires both, the previous and the next service identity` + `Opening a recovering network requires both, the previous and the next service identity`, ); } @@ -656,7 +656,7 @@ const actions = new Map([ : undefined; const next_identity = ccf.strToBuf(args.next_service_identity); ccf.node.transitionServiceToOpen(previous_identity, next_identity); - } + }, ), ], [ @@ -672,7 +672,7 @@ const actions = new Map([ const nameBuf = ccf.strToBuf(name); const bundleBuf = ccf.jsonCompatibleToBuf(bundle); ccf.kv["public:ccf.gov.tls.ca_cert_bundles"].set(nameBuf, bundleBuf); - } + }, ), ], [ @@ -685,7 +685,7 @@ const actions = new Map([ const name = args.name; const nameBuf = ccf.strToBuf(name); ccf.kv["public:ccf.gov.tls.ca_cert_bundles"].delete(nameBuf); - } + }, ), ], [ @@ -701,11 +701,11 @@ const actions = new Map([ checkType( args.key_policy.sgx_claims, "object?", - "key_policy.sgx_claims" + "key_policy.sgx_claims", ); if (args.key_policy.sgx_claims) { for (const [name, value] of Object.entries( - args.key_policy.sgx_claims + args.key_policy.sgx_claims, )) { checkType(value, "string", `key_policy.sgx_claims["${name}"]`); } @@ -718,7 +718,7 @@ const actions = new Map([ if (args.auto_refresh) { if (!args.ca_cert_bundle_name) { throw new Error( - "ca_cert_bundle_name is missing but required if auto_refresh is true" + "ca_cert_bundle_name is missing but required if auto_refresh is true", ); } let url; @@ -729,12 +729,12 @@ const actions = new Map([ } if (url.scheme != "https") { throw new Error( - "issuer must be a URL starting with https:// if auto_refresh is true" + "issuer must be a URL starting with https:// if auto_refresh is true", ); } if (url.query || url.fragment) { throw new Error( - "issuer must be a URL without query/fragment if auto_refresh is true" + "issuer must be a URL without query/fragment if auto_refresh is true", ); } } @@ -745,11 +745,11 @@ const actions = new Map([ const caCertBundleNameBuf = ccf.strToBuf(args.ca_cert_bundle_name); if ( !ccf.kv["public:ccf.gov.tls.ca_cert_bundles"].has( - caCertBundleNameBuf + caCertBundleNameBuf, ) ) { throw new Error( - `No CA cert bundle found with name '${caCertBundleName}'` + `No CA cert bundle found with name '${caCertBundleName}'`, ); } } @@ -763,7 +763,7 @@ const actions = new Map([ const issuerBuf = ccf.strToBuf(issuer); const metadataBuf = ccf.jsonCompatibleToBuf(metadata); ccf.kv["public:ccf.gov.jwt.issuers"].set(issuerBuf, metadataBuf); - } + }, ), ], [ @@ -783,7 +783,7 @@ const actions = new Map([ const metadata = ccf.bufToJsonCompatible(metadataBuf); const jwks = args.jwks; ccf.setJwtPublicSigningKeys(issuer, metadata, jwks); - } + }, ), ], [ @@ -799,7 +799,7 @@ const actions = new Map([ } ccf.kv["public:ccf.gov.jwt.issuers"].delete(issuerBuf); ccf.removeJwtPublicSigningKeys(args.issuer); - } + }, ), ], [ @@ -815,7 +815,7 @@ const actions = new Map([ // Adding a new allowed code ID changes the semantics of any other open proposals, so invalidate them to avoid confusion or malicious vote modification invalidateOtherOpenProposals(proposalId); - } + }, ), ], [ @@ -828,7 +828,7 @@ const actions = new Map([ const codeId = ccf.strToBuf(args.executor_code_id); const ALLOWED = ccf.jsonCompatibleToBuf("AllowedToExecute"); ccf.kv["public:ccf.gov.nodes.executor_code_ids"].set(codeId, ALLOWED); - } + }, ), ], [ @@ -839,7 +839,7 @@ const actions = new Map([ checkType( args.security_policy_digest, "string", - "security_policy_digest" + "security_policy_digest", ); }, function (args, proposalId) { @@ -848,15 +848,15 @@ const actions = new Map([ if (args.security_policy_raw != "") { const redigested_raw = ccf.bufToStr( - ccf.digest("SHA-256", ccf.strToBuf(args.security_policy_raw)) + ccf.digest("SHA-256", ccf.strToBuf(args.security_policy_raw)), ); const quoted_digest = ccf.bufToStr( - hexStrToBuf(args.security_policy_digest) + hexStrToBuf(args.security_policy_digest), ); if (redigested_raw != quoted_digest) { throw new Error( - `The hash of raw policy ${raw} does not match digest ${digest}` + `The hash of raw policy ${raw} does not match digest ${digest}`, ); } } @@ -865,7 +865,7 @@ const actions = new Map([ // Adding a new allowed security policy changes the semantics of any other open proposals, so invalidate them to avoid confusion or malicious vote modification invalidateOtherOpenProposals(proposalId); - } + }, ), ], [ @@ -875,13 +875,13 @@ const actions = new Map([ checkType( args.security_policy_digest, "string", - "security_policy_digest" + "security_policy_digest", ); }, function (args) { const digest = ccf.strToBuf(args.security_policy_digest); ccf.kv["public:ccf.gov.nodes.security_policies"].delete(digest); - } + }, ), ], [ @@ -900,7 +900,7 @@ const actions = new Map([ let ni = ccf.bufToJsonCompatible(node_info); ni.node_data = args.node_data; nodes_info.set(node_id, ccf.jsonCompatibleToBuf(ni)); - } + }, ), ], [ @@ -913,26 +913,26 @@ const actions = new Map([ checkType( args.validity_period_days, "integer", - "validity_period_days" + "validity_period_days", ); checkBounds( args.validity_period_days, 1, null, - "validity_period_days" + "validity_period_days", ); } }, function (args) { const rawConfig = ccf.kv["public:ccf.gov.service.config"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawConfig === undefined) { throw new Error("Service configuration could not be found"); } const serviceConfig = ccf.bufToJsonCompatible(rawConfig); const node = ccf.kv["public:ccf.gov.nodes.info"].get( - ccf.strToBuf(args.node_id) + ccf.strToBuf(args.node_id), ); if (node === undefined) { throw new Error(`No such node: ${args.node_id}`); @@ -947,7 +947,7 @@ const actions = new Map([ ccf.network.getLatestLedgerSecretSeqno(); ccf.kv["public:ccf.gov.nodes.info"].set( ccf.strToBuf(args.node_id), - ccf.jsonCompatibleToBuf(nodeInfo) + ccf.jsonCompatibleToBuf(nodeInfo), ); // Also generate and record service-endorsed node certificate from node CSR @@ -965,22 +965,23 @@ const actions = new Map([ args.validity_period_days > max_allowed_cert_validity_period_days ) { throw new Error( - `Validity period ${args.validity_period_days} is not allowed: max allowed is ${max_allowed_cert_validity_period_days}` + `Validity period ${args.validity_period_days} is not allowed: max allowed is ${max_allowed_cert_validity_period_days}`, ); } const endorsed_node_cert = ccf.network.generateEndorsedCertificate( nodeInfo.certificate_signing_request, args.valid_from, - args.validity_period_days ?? max_allowed_cert_validity_period_days + args.validity_period_days ?? + max_allowed_cert_validity_period_days, ); ccf.kv["public:ccf.gov.nodes.endorsed_certificates"].set( ccf.strToBuf(args.node_id), - ccf.strToBuf(endorsed_node_cert) + ccf.strToBuf(endorsed_node_cert), ); } } - } + }, ), ], [ @@ -992,7 +993,7 @@ const actions = new Map([ function (args) { const codeId = ccf.strToBuf(args.code_id); ccf.kv["public:ccf.gov.nodes.code_ids"].delete(codeId); - } + }, ), ], [ @@ -1004,7 +1005,7 @@ const actions = new Map([ function (args) { const codeId = ccf.strToBuf(args.executor_code_id); ccf.kv["public:ccf.gov.nodes.executor_code_ids"].delete(codeId); - } + }, ), ], [ @@ -1015,14 +1016,14 @@ const actions = new Map([ }, function (args) { const rawConfig = ccf.kv["public:ccf.gov.service.config"].get( - getSingletonKvKey() + getSingletonKvKey(), ); if (rawConfig === undefined) { throw new Error("Service configuration could not be found"); } const serviceConfig = ccf.bufToJsonCompatible(rawConfig); const node = ccf.kv["public:ccf.gov.nodes.info"].get( - ccf.strToBuf(args.node_id) + ccf.strToBuf(args.node_id), ); if (node === undefined) { return; @@ -1030,7 +1031,7 @@ const actions = new Map([ const node_obj = ccf.bufToJsonCompatible(node); if (node_obj.status === "Pending") { ccf.kv["public:ccf.gov.nodes.info"].delete( - ccf.strToBuf(args.node_id) + ccf.strToBuf(args.node_id), ); } else { node_obj.status = @@ -1039,10 +1040,10 @@ const actions = new Map([ : "Retired"; ccf.kv["public:ccf.gov.nodes.info"].set( ccf.strToBuf(args.node_id), - ccf.jsonCompatibleToBuf(node_obj) + ccf.jsonCompatibleToBuf(node_obj), ); } - } + }, ), ], [ @@ -1055,19 +1056,19 @@ const actions = new Map([ checkType( args.validity_period_days, "integer", - "validity_period_days" + "validity_period_days", ); checkBounds( args.validity_period_days, 1, null, - "validity_period_days" + "validity_period_days", ); } }, function (args) { const node = ccf.kv["public:ccf.gov.nodes.info"].get( - ccf.strToBuf(args.node_id) + ccf.strToBuf(args.node_id), ); if (node === undefined) { throw new Error(`No such node: ${args.node_id}`); @@ -1081,9 +1082,9 @@ const actions = new Map([ args.node_id, nodeInfo, args.valid_from, - args.validity_period_days + args.validity_period_days, ); - } + }, ), ], [ @@ -1095,13 +1096,13 @@ const actions = new Map([ checkType( args.validity_period_days, "integer", - "validity_period_days" + "validity_period_days", ); checkBounds( args.validity_period_days, 1, null, - "validity_period_days" + "validity_period_days", ); } }, @@ -1114,11 +1115,11 @@ const actions = new Map([ nodeId, nodeInfo, args.valid_from, - args.validity_period_days + args.validity_period_days, ); } }); - } + }, ), ], [ @@ -1130,22 +1131,22 @@ const actions = new Map([ checkType( args.validity_period_days, "integer", - "validity_period_days" + "validity_period_days", ); checkBounds( args.validity_period_days, 1, null, - "validity_period_days" + "validity_period_days", ); } }, function (args) { setServiceCertificateValidityPeriod( args.valid_from, - args.validity_period_days + args.validity_period_days, ); - } + }, ), ], [ @@ -1155,7 +1156,7 @@ const actions = new Map([ for (var key in args) { if (key !== "reconfiguration_type" && key !== "recovery_threshold") { throw new Error( - `Cannot change ${key} via set_service_configuration.` + `Cannot change ${key} via set_service_configuration.`, ); } } @@ -1165,7 +1166,7 @@ const actions = new Map([ }, function (args) { updateServiceConfig(args); - } + }, ), ], [ @@ -1174,7 +1175,7 @@ const actions = new Map([ function (args) {}, function (args, proposalId) { ccf.node.triggerLedgerChunk(); - } + }, ), ], [ @@ -1183,7 +1184,7 @@ const actions = new Map([ function (args) {}, function (args, proposalId) { ccf.node.triggerSnapshot(); - } + }, ), ], [ @@ -1193,12 +1194,12 @@ const actions = new Map([ checkType( args.interfaces, "array?", - "interfaces to refresh the certificates for" + "interfaces to refresh the certificates for", ); }, function (args, proposalId) { ccf.node.triggerACMERefresh(args.interfaces); - } + }, ), ], [ @@ -1216,7 +1217,7 @@ const actions = new Map([ throw new Error("Service identity certificate mismatch"); } }, - function (args) {} + function (args) {}, ), ], [ @@ -1230,7 +1231,7 @@ const actions = new Map([ // apply function (args) { setPublicPrefix(args.public_prefix); - } + }, ), ], [ @@ -1244,7 +1245,7 @@ const actions = new Map([ // apply function (args) { removePublicPrefix(args.public_prefix); - } + }, ), ], ]); diff --git a/constitution/validate.js b/constitution/validate.js index a4c0332d..0ae1937a 100644 --- a/constitution/validate.js +++ b/constitution/validate.js @@ -12,7 +12,7 @@ export function validate(input) { definition.validate(action.args); } catch (e) { errors.push( - `${action.name} at position ${position} failed validation: ${e}\n${e.stack}` + `${action.name} at position ${position} failed validation: ${e}\n${e.stack}`, ); } } else { diff --git a/flake.lock b/flake.lock index f203e435..446279e8 100644 --- a/flake.lock +++ b/flake.lock @@ -1,12 +1,15 @@ { "nodes": { "flake-utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", "type": "github" }, "original": { @@ -17,11 +20,11 @@ }, "nix-filter": { "locked": { - "lastModified": 1666547822, - "narHash": "sha256-razwnAybPHyoAyhkKCwXdxihIqJi1G6e1XP4FQOJTEs=", + "lastModified": 1687178632, + "narHash": "sha256-HS7YR5erss0JCaUijPeyg2XrisEb959FIct3n2TMGbE=", "owner": "numtide", "repo": "nix-filter", - "rev": "1a3b735e13e90a8d2fd5629f2f8363bd7ffbbec7", + "rev": "d90c75e8319d0dd9be67d933d8eb9d0894ec9174", "type": "github" }, "original": { @@ -32,11 +35,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1666629043, - "narHash": "sha256-Yoq6Ut2F3Ol73yO9hG93x6ts5c4F5BhKTbcF3DtBEAw=", + "lastModified": 1691368598, + "narHash": "sha256-ia7li22keBBbj02tEdqjVeLtc7ZlSBuhUk+7XTUFr14=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b39fd6e4edef83cb4a135ebef98751ce23becc33", + "rev": "5a8e9243812ba528000995b294292d3b5e120947", "type": "github" }, "original": { @@ -52,6 +55,21 @@ "nix-filter": "nix-filter", "nixpkgs": "nixpkgs" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/nix/ccf.nix b/nix/ccf.nix index 6d843ac7..3bdb00aa 100644 --- a/nix/ccf.nix +++ b/nix/ccf.nix @@ -1,17 +1,17 @@ { fetchFromGitHub, + pkg-config, cmake, ninja, stdenv, openenclave, libuv, - az-dcap, - sgx-dcap, - sgx-psw, + # az-dcap, + # sgx-dcap, + # sgx-psw, makeWrapper, protobuf, openssl, - arrow-cpp, platform ? "virtual", }: let toRemove = @@ -25,13 +25,13 @@ in stdenv.mkDerivation rec { pname = "ccf-${platform}"; - version = "4.0.0-dev3"; + version = "4.0.7"; src = fetchFromGitHub { owner = "microsoft"; repo = "CCF"; name = "ccf-${version}"; rev = "ccf-${version}"; - hash = "sha256-IG5lgtiq/VPTJm9hMYcj9JTxsapD7+H1aw5+b6pY0ko="; + hash = "sha256-CofADLExBTo3CH7iACKKNxMsSpy/ZBWBRaXc3ELHAd4="; }; patches = [ patches/ccf-no-python.diff @@ -43,10 +43,11 @@ in nativeBuildInputs = [ cmake ninja + pkg-config libuv protobuf - arrow-cpp - sgx-dcap + # arrow-cpp + # sgx-dcap openenclave makeWrapper ]; @@ -62,8 +63,8 @@ in NIX_NO_SELF_RPATH = "1"; postInstall = '' - wrapProgram $out/bin/cchost \ - --suffix LD_LIBRARY_PATH ':' "${az-dcap}/lib:${sgx-psw}/lib:${sgx-dcap}/lib" + # wrapProgram $out/bin/cchost \ + # --suffix LD_LIBRARY_PATH ':' "''${az-dcap}/lib:''${sgx-psw}/lib:''${sgx-dcap}/lib" wrapProgram $out/bin/keygenerator.sh \ --prefix PATH ':' "${openssl}/bin" diff --git a/nix/ci-checks.nix b/nix/ci-checks.nix index 61f9365a..89ea8631 100644 --- a/nix/ci-checks.nix +++ b/nix/ci-checks.nix @@ -130,7 +130,7 @@ in { prettier = writeShellScriptBin "prettier" '' - git ls-files -- . ':!:3rdparty/' | grep -e '\.ts$' -e '\.js$' -e '\.md$' -e '\.yaml$' -e '\.yml$' -e '\.json$' | xargs npx ${nodePackages.prettier}/bin/prettier --write + git ls-files -- . ':!:3rdparty/' | grep -e '\.ts$' -e '\.js$' -e '\.md$' -e '\.yaml$' -e '\.yml$' -e '\.json$' | xargs ${nodePackages.prettier}/bin/prettier --write ''; black = diff --git a/nix/default.nix b/nix/default.nix index d98b2c19..2b767714 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -28,12 +28,12 @@ pkgs.lib.makeScope pkgs.newScope ( }; ccf = self.callPackage ./ccf.nix { - stdenv = pkgs.llvmPackages_10.libcxxStdenv; + stdenv = pkgs.llvmPackages_16.libcxxStdenv; }; ccf-sandbox = self.callPackage ./ccf-sandbox.nix {inherit ccf;}; lskv = self.callPackage ./lskv.nix { inherit ccf; - stdenv = pkgs.llvmPackages_10.libcxxStdenv; + stdenv = pkgs.llvmPackages_16.libcxxStdenv; }; lskv-sandbox = self.callPackage ./lskv-sandbox.nix {inherit ccf-sandbox lskv;}; packages = lskvlib.forAllPlatforms { @@ -53,25 +53,25 @@ pkgs.lib.makeScope pkgs.newScope ( az-dcap = self.callPackage ./az-dcap.nix {}; sgx-dcap = self.callPackage ./sgx-dcap.nix {}; - openenclave-version = "0.18.4"; + openenclave-version = "0.19.3"; openenclave-src = pkgs.fetchFromGitHub { owner = "openenclave"; repo = "openenclave"; rev = "v${openenclave-version}"; - hash = "sha256-65LHXKfDWUvLCMupJkF7o7d6ljsO7nwcmQxRU8H2Xls="; + hash = "sha256-RN7Mq6RO09CZOEoi/nYpPfa7TT1I5FYKqET8wRXnIxU="; fetchSubmodules = true; }; lvi-mitigation = self.callPackage ./lvi-mitigation.nix {}; openenclave = self.callPackage ./openenclave.nix { - # Openenclave doesn't build with libcxx, for some reason. - inherit (pkgs.llvmPackages_10) stdenv; + stdenv = pkgs.llvmPackages_11.libcxxStdenv; + openssl = pkgs.openssl_1_1; }; k6 = self.callPackage ./k6.nix {}; mkShell = args: (pkgs.mkShell.override { - stdenv = pkgs.llvmPackages_10.libcxxStdenv; + stdenv = pkgs.llvmPackages_16.libcxxStdenv; }) ({ NIX_CFLAGS_COMPILE = "-Wno-unused-command-line-argument"; NIX_NO_SELF_RPATH = "1"; diff --git a/nix/lskv.nix b/nix/lskv.nix index 035ec42c..3ade6b08 100644 --- a/nix/lskv.nix +++ b/nix/lskv.nix @@ -1,7 +1,7 @@ { stdenv, cmake, - sgx-dcap, + # sgx-dcap, openenclave, ninja, protobuf, @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { cmake ninja protobuf - sgx-dcap + # sgx-dcap (ccf.override {inherit platform;}) openenclave ]; diff --git a/nix/openenclave.nix b/nix/openenclave.nix index 2eece81d..687faaa5 100644 --- a/nix/openenclave.nix +++ b/nix/openenclave.nix @@ -4,27 +4,26 @@ fetchzip, openenclave-version, openenclave-src, - lvi-mitigation, cmake, ninja, perl, - openssl_1_1, + openssl, }: let sgx-h = fetchurl { url = "https://raw.githubusercontent.com/torvalds/linux/v5.13/arch/x86/include/uapi/asm/sgx.h"; sha256 = "4764b8ce858579d99f1b66bb1e5f04ba149a38aea15649fff19f65f8d9113fd0"; }; compiler-rt = fetchzip { - url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/compiler-rt-10.0.1.src.tar.xz"; - hash = "sha256-OErVbpYasfvBK0793ujshuHK4tbqq3grQHjYDpebmT4="; + url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/compiler-rt-11.1.0.src.tar.xz"; + hash = "sha256-jycaXF3wGF85B2cwe+1q5fVPhR+/JnaZ+4A8y/qyBag="; }; libcxx = fetchzip { - url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/libcxx-10.0.1.src.tar.xz"; - sha256 = "sha256-/OhdYPlbNHMxX2VxlurkOspC1OyPDmyUqXvZKxzwkTg="; + url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/libcxx-11.1.0.src.tar.xz"; + sha256 = "sha256-UoRPugdPj0FtKp79V1nljehWyhChxgUo3mb/Wyq/RIA="; }; symcrypt = fetchzip { - url = "https://github.com/microsoft/SymCrypt/releases/download/v101.3.0/symcrypt_AMD64_oe_full_v101.3.0-31e06ae.tgz"; - sha256 = "sha256-diA653HZ4Mn4JbeT6+U0anhP3ySVWZWjcXH7KVVkqkY="; + url = "https://github.com/microsoft/SymCrypt/releases/download/v103.0.1/symcrypt-linux-oe_full-AMD64-103.0.1-69dbff3.tar.gz"; + sha256 = "sha256-VCJlAOnbY2kYlnNv6SxumD4BinntAvpBFkUs9hBxCY4="; stripRoot = false; }; in @@ -32,13 +31,16 @@ in pname = "openenclave"; version = openenclave-version; src = openenclave-src; - patches = [patches/openenclave.diff]; + patches = [ + # patches/openenclave.diff + patches/openenclave-pkgconfig.diff + ]; cmakeFlags = [ "-DCMAKE_BUILD_TYPE=RelWithDebInfo" "-DFETCHCONTENT_SOURCE_DIR_COMPILER-RT-SOURCES=${compiler-rt}" "-DFETCHCONTENT_SOURCE_DIR_LIBCXX_SOURCES=${libcxx}" "-DFETCHCONTENT_SOURCE_DIR_SYMCRYPT_PACKAGE=${symcrypt}" - "-DCLANG_INTRINSIC_HEADERS_DIR=${toString stdenv.cc.cc.lib}/lib/clang/10.0.1/include" + "-DCLANG_INTRINSIC_HEADERS_DIR=${toString stdenv.cc.cc.lib}/lib/clang/${stdenv.cc.version}/include" "-DENABLE_REFMAN=OFF" "-DBUILD_TESTS=OFF" @@ -46,34 +48,31 @@ in # This breaks reproducible builds. "-DBUILD_OEUTIL_TOOL=OFF" - "-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON" - "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=ON" - - "-DLVI_MITIGATION=ControlFlow" + # "-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON" + # "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=ON" ]; preConfigure = '' - mkdir -p build/host build/3rdparty/symcrypt_engine + mkdir -p build/host + mkdir -p build/3rdparty/symcrypt_engine cp ${sgx-h} build/host/sgx.h ln -s ${compiler-rt} 3rdparty/compiler-rt/compiler-rt ln -s ${libcxx} 3rdparty/libcxx/libcxx ln -s ${symcrypt} build/3rdparty/symcrypt_engine/SymCrypt - ln -s ${lvi-mitigation}/bin build/lvi_mitigation_bin patchShebangs tools/oeutil/gen_pubkey_header.sh - patchShebangs tools/oeapkman/oeapkman + substituteInPlace tools/oeutil/gen_pubkey_header.sh --replace '/var/tmp/oeutil_lock' '.oeutil_lock' patchShebangs 3rdparty/openssl/append-unsupported patchShebangs 3rdparty/musl/append-deprecations + ''; - substituteInPlace pkgconfig/*.pc --replace \''${prefix}/@CMAKE_INSTALL_LIBDIR@ @CMAKE_INSTALL_LIBDIR@ - substituteInPlace pkgconfig/*.pc --replace \''${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ @CMAKE_INSTALL_INCLUDEDIR@ - - # since expansion isn't possible in cmakeFlags - cmakeFlags="$cmakeFlags -DLVI_MITIGATION_BINDIR=$PWD/build/lvi_mitigation_bin" + postFixup = '' + substituteInPlace $out/lib/${pname}/cmake/${pname}-*.cmake \ + --replace 'set(_IMPORT_PREFIX' '#set(_IMPORT_PREFIX' ''; nativeBuildInputs = [cmake ninja perl]; - propagatedBuildInputs = [openssl_1_1]; + propagatedBuildInputs = [openssl]; # Not sure if we want to keep this dontAutoPatchelf = true; diff --git a/nix/patches/ccf-no-python-pb2.diff b/nix/patches/ccf-no-python-pb2.diff index b580a8df..09e9c9b1 100644 --- a/nix/patches/ccf-no-python-pb2.diff +++ b/nix/patches/ccf-no-python-pb2.diff @@ -2,10 +2,10 @@ diff --git a/src/apps/external_executor/protobuf/CMakeLists.txt b/src/apps/exter index d35fe3537..3418d7be5 100644 --- a/src/apps/external_executor/protobuf/CMakeLists.txt +++ b/src/apps/external_executor/protobuf/CMakeLists.txt -@@ -35,24 +35,6 @@ foreach(proto_file ${PROTO_FILES}) +@@ -38,24 +38,6 @@ foreach(proto_file ${PROTO_FILES}) ${CMAKE_CURRENT_BINARY_DIR}/${PROTO_NAME_WE}.pb.cc ) - + - add_custom_command( - OUTPUT - ${CMAKE_SOURCE_DIR}/tests/external_executor/${PROTO_NAME_WE}_pb2.py @@ -24,6 +24,6 @@ index d35fe3537..3418d7be5 100644 - JOB_POOL one_job - ) - - if(COMPILE_TARGET STREQUAL "sgx") - add_enclave_library( - ${PROTO_NAME_WE}.proto.enclave + # For now, copy .proto files across to external executor test + configure_file( + ${proto_file} diff --git a/nix/patches/openenclave-pkgconfig.diff b/nix/patches/openenclave-pkgconfig.diff new file mode 100644 index 00000000..1963200f --- /dev/null +++ b/nix/patches/openenclave-pkgconfig.diff @@ -0,0 +1,308 @@ +diff --git a/pkgconfig/oeenclave-clang++-lvi-cfg.pc b/pkgconfig/oeenclave-clang++-lvi-cfg.pc +index 6ac9d6ec7..380ae3ba2 100644 +--- a/pkgconfig/oeenclave-clang++-lvi-cfg.pc ++++ b/pkgconfig/oeenclave-clang++-lvi-cfg.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibslvicfg=@LVI_MITIGATION_MBEDTLS_CRYPTO_LIBS@ + openssllibslvicfg=@LVI_MITIGATION_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-clang++.pc b/pkgconfig/oeenclave-clang++.pc +index f588fa1fd..2108e0657 100644 +--- a/pkgconfig/oeenclave-clang++.pc ++++ b/pkgconfig/oeenclave-clang++.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibs=@ENCLAVE_MBEDTLS_CRYPTO_LIBS@ + openssllibs=@ENCLAVE_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-clang-lvi-cfg.pc b/pkgconfig/oeenclave-clang-lvi-cfg.pc +index a7a8c4e7c..ebb4bafc3 100644 +--- a/pkgconfig/oeenclave-clang-lvi-cfg.pc ++++ b/pkgconfig/oeenclave-clang-lvi-cfg.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibslvicfg=@LVI_MITIGATION_MBEDTLS_CRYPTO_LIBS@ + openssllibslvicfg=@LVI_MITIGATION_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-clang.pc b/pkgconfig/oeenclave-clang.pc +index 583a474d3..01d179059 100644 +--- a/pkgconfig/oeenclave-clang.pc ++++ b/pkgconfig/oeenclave-clang.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibs=@ENCLAVE_MBEDTLS_CRYPTO_LIBS@ + openssllibs=@ENCLAVE_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-g++-lvi-cfg.pc b/pkgconfig/oeenclave-g++-lvi-cfg.pc +index 44b6fe8fd..1f06128db 100644 +--- a/pkgconfig/oeenclave-g++-lvi-cfg.pc ++++ b/pkgconfig/oeenclave-g++-lvi-cfg.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibslvicfg=@LVI_MITIGATION_MBEDTLS_CRYPTO_LIBS@ + openssllibslvicfg=@LVI_MITIGATION_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-g++.pc b/pkgconfig/oeenclave-g++.pc +index 80ae85bd0..a1a45bd0b 100644 +--- a/pkgconfig/oeenclave-g++.pc ++++ b/pkgconfig/oeenclave-g++.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibs=@ENCLAVE_MBEDTLS_CRYPTO_LIBS@ + openssllibs=@ENCLAVE_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-gcc-lvi-cfg.pc b/pkgconfig/oeenclave-gcc-lvi-cfg.pc +index bae4563ab..4aa5843de 100644 +--- a/pkgconfig/oeenclave-gcc-lvi-cfg.pc ++++ b/pkgconfig/oeenclave-gcc-lvi-cfg.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibslvicfg=@LVI_MITIGATION_MBEDTLS_CRYPTO_LIBS@ + openssllibslvicfg=@LVI_MITIGATION_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oeenclave-gcc.pc b/pkgconfig/oeenclave-gcc.pc +index 068c32f05..dd585c395 100644 +--- a/pkgconfig/oeenclave-gcc.pc ++++ b/pkgconfig/oeenclave-gcc.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + mbedtlslibs=@ENCLAVE_MBEDTLS_CRYPTO_LIBS@ + openssllibs=@ENCLAVE_OPENSSL_CRYPTO_LIBS@ +diff --git a/pkgconfig/oehost-clang++.pc b/pkgconfig/oehost-clang++.pc +index 89b5c91e0..114630754 100644 +--- a/pkgconfig/oehost-clang++.pc ++++ b/pkgconfig/oehost-clang++.pc +@@ -1,14 +1,13 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave clang++ flags and libraries for building hosts. + Version: @OE_VERSION@ + Requires: openssl +-Cflags: @HOST_CXXFLAGS_CLANG@ @HOST_INCLUDES@ ++Cflags: @HOST_CXXFLAGS_CLANG@ @HOST_INCLUDES@ + Libs: @HOST_CXXLIBS@ +diff --git a/pkgconfig/oehost-clang.pc b/pkgconfig/oehost-clang.pc +index 702ac7afb..3efa2d388 100644 +--- a/pkgconfig/oehost-clang.pc ++++ b/pkgconfig/oehost-clang.pc +@@ -1,14 +1,13 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave clang flags and libraries for building hosts. + Version: @OE_VERSION@ + Requires: openssl +-Cflags: @HOST_CFLAGS_CLANG@ @HOST_INCLUDES@ ++Cflags: @HOST_CFLAGS_CLANG@ @HOST_INCLUDES@ + Libs: @HOST_CLIBS@ +diff --git a/pkgconfig/oehost-g++.pc b/pkgconfig/oehost-g++.pc +index 944f85875..da928c4b0 100644 +--- a/pkgconfig/oehost-g++.pc ++++ b/pkgconfig/oehost-g++.pc +@@ -1,14 +1,13 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave g++ flags and libraries for building hosts. + Version: @OE_VERSION@ + Requires: openssl +-Cflags: @HOST_CXXFLAGS_GCC@ @HOST_INCLUDES@ ++Cflags: @HOST_CXXFLAGS_GCC@ @HOST_INCLUDES@ + Libs: @HOST_CXXLIBS@ +diff --git a/pkgconfig/oehost-gcc.pc b/pkgconfig/oehost-gcc.pc +index 6c86ba006..9b550b434 100644 +--- a/pkgconfig/oehost-gcc.pc ++++ b/pkgconfig/oehost-gcc.pc +@@ -1,14 +1,13 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=${pcfiledir}/../.. +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave gcc flags and libraries for building hosts. + Version: @OE_VERSION@ + Requires: openssl +-Cflags: @HOST_CFLAGS_GCC@ @HOST_INCLUDES@ ++Cflags: @HOST_CFLAGS_GCC@ @HOST_INCLUDES@ + Libs: @HOST_CLIBS@ +diff --git a/pkgconfig/oehostverify-clang++.pc b/pkgconfig/oehostverify-clang++.pc +index 33c01b43f..3c1e0fc4f 100644 +--- a/pkgconfig/oehostverify-clang++.pc ++++ b/pkgconfig/oehostverify-clang++.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=@PREFIX@ +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave clang++ flags and libraries verifying remote reports on hosts. +diff --git a/pkgconfig/oehostverify-clang.pc b/pkgconfig/oehostverify-clang.pc +index ed5b67a28..d643115d7 100644 +--- a/pkgconfig/oehostverify-clang.pc ++++ b/pkgconfig/oehostverify-clang.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=@PREFIX@ +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave clang flags and libraries verifying remote reports on hosts. +diff --git a/pkgconfig/oehostverify-g++.pc b/pkgconfig/oehostverify-g++.pc +index da30cf65b..0d4535cf4 100644 +--- a/pkgconfig/oehostverify-g++.pc ++++ b/pkgconfig/oehostverify-g++.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=@PREFIX@ +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave g++ flags and libraries verifying remote reports on hosts. +diff --git a/pkgconfig/oehostverify-gcc.pc b/pkgconfig/oehostverify-gcc.pc +index 0559e1f58..5a0cd10ed 100644 +--- a/pkgconfig/oehostverify-gcc.pc ++++ b/pkgconfig/oehostverify-gcc.pc +@@ -1,10 +1,9 @@ + # Copyright (c) Open Enclave SDK contributors. + # Licensed under the MIT License. + +-prefix=@PREFIX@ +-exec_prefix=${prefix} +-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ ++exec_prefix=@CMAKE_INSTALL_PREFIX@ ++includedir=@CMAKE_INSTALL_INCLUDEDIR@ ++libdir=@CMAKE_INSTALL_LIBDIR@ + + Name: Open Enclave + Description: Open Enclave gcc flags and libraries verifying remote reports on hosts. diff --git a/python/lskv/governance.py b/python/lskv/governance.py index f416378d..4d40ecd1 100644 --- a/python/lskv/governance.py +++ b/python/lskv/governance.py @@ -124,6 +124,8 @@ def run( "ccf_cose_sign1", "--ccf-gov-msg-type", gov_msg_type, + "--ccf-gov-msg-created_at", + "$(date -Is)", "--signing-key", self.signing_key, "--signing-cert", diff --git a/requirements.txt b/requirements.txt index 386d1a86..226df003 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -ccf==4.0.0dev3 +ccf==4.0.7 cimetrics==0.3.14 pandas==1.5.0 notebook==6.4.12 diff --git a/scripts/check-format.sh b/scripts/check-format.sh index 5e057eb0..bff447c5 100755 --- a/scripts/check-format.sh +++ b/scripts/check-format.sh @@ -32,7 +32,7 @@ fi file_name_regex="^[[:lower:]0-9_]+$" unformatted_files="" badly_named_files="" -clang_fmt=clang-format-10 +clang_fmt=clang-format-11 if [[ ! $(command -v ${clang_fmt}) ]]; then clang_fmt=clang-format fi diff --git a/src/app/app.cpp b/src/app/app.cpp index 18b3e04e..311e7ee0 100644 --- a/src/app/app.cpp +++ b/src/app/app.cpp @@ -1132,8 +1132,8 @@ namespace app lstore.foreach([&response, &now_s](auto id, auto lease) { if (!lease.has_expired(now_s)) { - auto* lease = response.add_leases(); - lease->set_id(id); + auto* added_lease = response.add_leases(); + added_lease->set_id(id); } return true; }); diff --git a/src/app/index.cpp b/src/app/index.cpp index 8d15bfce..72a3bd39 100644 --- a/src/app/index.cpp +++ b/src/app/index.cpp @@ -203,14 +203,16 @@ namespace app::index auto& values = keys_to_values.at(key); // TODO(#204): Should be able to remove multiple values from a key's // vector at once to avoid multiple copyings of the elements. - auto it = values.begin(); - while (it != values.end()) + auto values_it = values.begin(); + while (values_it != values.end()) { - if (it->mod_revision < at) + if (values_it->mod_revision < at) { CCF_APP_DEBUG( - "Removing compacted value for key {} at {}", key, it->mod_revision); - it = values.erase(it); + "Removing compacted value for key {} at {}", + key, + values_it->mod_revision); + values_it = values.erase(values_it); } else { diff --git a/tests/test_single.py b/tests/test_single.py index 3551cba4..a1e2cdbd 100644 --- a/tests/test_single.py +++ b/tests/test_single.py @@ -294,13 +294,6 @@ def test_public_prefix(governance_client, http1_client, sandbox): public_domain = txn.get_public_domain() assert len(public_domain.get_tables()) == 1 - # setting an existing prefix is ok - proposal = governance.Proposal() - proposal.set_public_prefix(prefix) - res = governance_client.propose(proposal) - proposal_id = res.proposal_id - governance_client.accept(proposal_id) - # removing an existing prefix is ok proposal = governance.Proposal() proposal.remove_public_prefix(prefix) @@ -308,13 +301,6 @@ def test_public_prefix(governance_client, http1_client, sandbox): proposal_id = res.proposal_id governance_client.accept(proposal_id) - # and removing one that doesn't exist is ok too - proposal = governance.Proposal() - proposal.remove_public_prefix(prefix) - res = governance_client.propose(proposal) - proposal_id = res.proposal_id - governance_client.accept(proposal_id) - # setting a new key now doesn't end up public res = http1_client.put(f"{prefix}/test", "my secret") term = int(res.json()["header"]["raftTerm"])