Skip to content

Commit

Permalink
Merge branch 'main' of github.com:microsoft/CCF into try_optional_fie…
Browse files Browse the repository at this point in the history
…lds_are_always_omitted
  • Loading branch information
eddyashton committed Jul 2, 2024
2 parents d9c862d + 329288c commit ab1fff3
Show file tree
Hide file tree
Showing 420 changed files with 5,938 additions and 4,561 deletions.
2 changes: 1 addition & 1 deletion .azure-pipelines-gh-pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
variables:
Codeql.SkipTaskAutoInjection: true
skipComponentGovernanceDetection: true
container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
container: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
pool:
vmImage: ubuntu-20.04

Expand Down
2 changes: 1 addition & 1 deletion .azure-pipelines-model-checking.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ schedules:
resources:
containers:
- container: virtual
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

jobs:
Expand Down
6 changes: 3 additions & 3 deletions .azure-pipelines-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ pr: none
resources:
containers:
- container: virtual
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

- container: snp
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

- container: sgx
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro

variables:
Expand Down
4 changes: 2 additions & 2 deletions .azure-pipelines-templates/deploy_aci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
- script: |
set -ex
docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY
docker pull $ACR_REGISTRY/ccf/ci:oe-0.19.6-0-snp-clang15
docker pull $ACR_REGISTRY/ccf/ci:2024-06-26-snp-clang15
docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
name: build_ci_image
Expand All @@ -63,7 +63,7 @@ jobs:
ACR_TOKEN_NAME: ci-push-token
ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD)
ACR_REGISTRY: ccfmsrc.azurecr.io
BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15
BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
- script: |
set -ex
Expand Down
65 changes: 0 additions & 65 deletions .azure-pipelines-templates/matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,20 +69,6 @@ parameters:
ctest_args: '-L "raft_scenario"'

jobs:
# Debug
- ${{ each target in parameters.target }}:
- template: common.yml
parameters:
target: ${{ target }}
env: ${{ parameters.env[target] }}
cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build[target].cmake_args }}"
cmake_env: "${{ parameters.build[target].cmake_env }}"
suffix: "Debug"
artifact_name: "${{ target }}_Debug"
ctest_filter: "${{ parameters.test[target].ctest_args }}"
depends_on: configure
installExtendedTestingTools: false

# Tracing
- template: common.yml
parameters:
Expand All @@ -97,57 +83,6 @@ jobs:
installExtendedTestingTools: false
ninja_targets: "${{ parameters.build.tracing.ninja_targets }}"

# Performance
- ${{ if eq(parameters.perf_tests, 'run') }}:
- template: common.yml
parameters:
target: SGX
env: ${{ parameters.env.SGX }}
cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }}"
suffix: "Perf"
artifact_name: "SGX_Perf"
ctest_filter: "${{ parameters.test.perf.ctest_args }}"
depends_on: configure
installExtendedTestingTools: false

- ${{ if eq(parameters.perf_tests, 'run') }}:
- template: common.yml
parameters:
target: Virtual
env: ${{ parameters.env.Virtual }}
cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}"
cmake_env: "${{ parameters.build.Virtual.cmake_env }}"
suffix: "Perf"
artifact_name: "Virtual_Perf"
ctest_filter: "${{ parameters.test.virtual_perf.ctest_args }}"
depends_on: configure
installExtendedTestingTools: false

- ${{ if eq(parameters.perf_tests, 'run') }}:
- template: common.yml
parameters:
target: Virtual
env: ${{ parameters.env.Virtual }}
cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DWORKER_THREADS=2"
cmake_env: "${{ parameters.build.Virtual.cmake_env }}"
suffix: "Perf_MultiThreaded"
artifact_name: "Virtual_Perf_MultiThreaded"
ctest_filter: "-R pi_basic_mt"
depends_on: configure
installExtendedTestingTools: false

- ${{ if eq(parameters.perf_tests, 'run') }}:
- template: common.yml
parameters:
target: SGX
env: ${{ parameters.env.SGX }}
cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DWORKER_THREADS=2"
suffix: "Perf_MultiThreaded"
artifact_name: "SGX_Perf_MultiThreaded"
ctest_filter: "-R pi_basic_mt"
depends_on: configure
installExtendedTestingTools: false

- ${{ if eq(parameters.perf_tests, 'run') }}:
- template: simulation.yml
parameters:
Expand Down
6 changes: 3 additions & 3 deletions .azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ schedules:
resources:
containers:
- container: virtual
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

- container: snp
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

- container: sgx
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro

variables:
Expand Down
2 changes: 1 addition & 1 deletion .azure_pipelines_snp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ schedules:
resources:
containers:
- container: virtual
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

jobs:
Expand Down
6 changes: 3 additions & 3 deletions .daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ schedules:
resources:
containers:
- container: virtual
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE

- container: snp
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-snp-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro

- container: sgx
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-sgx
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx

jobs:
Expand Down
2 changes: 1 addition & 1 deletion .daily_canary
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
( V ) / . \ | +---=---'
/--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
2024__
!..!
!..!
2 changes: 1 addition & 1 deletion .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "CCF Development Environment",
"image": "ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15",
"image": "ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15",
"runArgs": [],
"extensions": [
"eamodio.gitlens",
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/bencher.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
name: Continuous Benchmarking with Bencher
runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
container:
image: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
steps:
- uses: actions/checkout@v4
with:
Expand Down
91 changes: 91 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
name: CI

on:
push:
branches: [main]
pull_request:
workflow_dispatch:

permissions:
contents: read
actions: read
checks: write

jobs:
build_and_test:
name: CI
strategy:
matrix:
platform:
- name: virtual
image: default
nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
- name: snp
image: default
nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
- name: sgx
image: sgx
nodes: [self-hosted, 1ES.Pool=gha-sgx-ccf-sub]
options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
runs-on: ${{ matrix.platform.nodes }}
container:
image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-26-06-2024
options: ${{ matrix.platform.options }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: "Build Debug ${{ matrix.platform.name }}"
run: |
set -ex
git config --global --add safe.directory /__w/CCF/CCF
mkdir build
cd build
cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DVERBOSE_LOGGING=ON ..
ninja
shell: bash

- name: "Platform SGX"
run: |
sudo groupadd -fg $(/usr/bin/stat -Lc '%g' /dev/sgx/provision) sgx_prv
sudo usermod -a -G sgx_prv $(whoami)
samples/scripts/sgxinfo.sh
cat /proc/cpuinfo | grep flags | uniq
shell: bash
if: "${{ matrix.platform.name == 'sgx' }}"

- name: "Platform SNP"
run: |
samples/scripts/snpinfo.sh
shell: bash
if: "${{ matrix.platform.name == 'snp' }}"

- name: "Test ${{ matrix.platform.name }}"
run: |
set -ex
cd build
rm -rf /github/home/.cache
mkdir -p /github/home/.cache
export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
# Unit tests
./tests.sh --output-on-failure -L unit -j$(nproc --all)
# All other acceptably fast tests, mostly end-to-end
./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite|unit"
# Partitions tests
./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite"
shell: bash
if: "${{ matrix.platform.name != 'snp' }}" # Needs 1ES Pool support

- name: "Upload logs for ${{ matrix.platform.name }}"
uses: actions/upload-artifact@v4
with:
name: logs-${{ matrix.platform.name }}
path: |
build/workspace/*/*.config.json
build/workspace/*/out
build/workspace/*/err
if-no-files-found: ignore
if: success() || failure()
2 changes: 1 addition & 1 deletion .github/workflows/ci-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ permissions: read-all
jobs:
checks:
runs-on: ubuntu-latest
container: ccfmsrc.azurecr.io/ccf/ci:oe-0.19.6-0-virtual-clang15
container: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15

steps:
- run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
Expand Down
43 changes: 10 additions & 33 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,67 +21,44 @@ jobs:
name: Analyze
# Insufficient space to run on public runner, so use custom pool
runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
container:
image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
options: --user root

permissions:
security-events: write

strategy:
fail-fast: false
matrix:
# Override automatic language detection by changing the below list
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
language: ["cpp"]
# Learn more...
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0

# Initializes the CodeQL tools for scanning.
# Done before CodeQL init to let it find the commit successfully
- name: Work around git warning
run: git config --global --add safe.directory /__w/CCF/CCF

- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
queries: security-extended
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main

# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
#- name: Autobuild
# uses: github/codeql-action/autobuild@v2

# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl

# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language

#- run: |
# make bootstrap
# make release
- run: |
cd getting_started/setup_vm
sudo apt update
sudo apt install -y ansible software-properties-common bsdmainutils dnsutils
sudo ansible-playbook ccf-dev.yml --extra-vars "platform=virtual" --extra-vars "require_open_enclave=false"
name: Install dependencies

- run: |
set -ex
mkdir build
cd build
cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-11` -DCMAKE_CXX_COMPILER=`which clang++-11` ..
cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF ..
name: Run CMake
- run: |
cd build
make
make -j16
name: Run Make
- name: Perform CodeQL Analysis
Expand Down
Loading

0 comments on commit ab1fff3

Please sign in to comment.