Merge branch 'master' into eero/upgrade-2

.bazelversion

@@ -1 +1 @@
-7.3.1
+7.4.1

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "ghcr.io/dfinity/ic-build@sha256:0fe4b6d8d22100dddea4c26a0baca52c190141658a66542d589164bcc7611ded",
+  "image": "ghcr.io/dfinity/ic-build@sha256:1d775e6d161dee883d10f082ab7fdde3e3d26061e1209255fb6514f8b62b206b",
   "remoteUser": "ubuntu",
   "privileged": true,
   "runArgs": [
@@ -14,7 +14,10 @@
   ],
   "workspaceMount": "source=${localWorkspaceFolder},target=/ic,type=bind",
   "workspaceFolder": "/ic",
-  "initializeCommand": "mkdir -p ~/.aws ~/.ssh ~/.cache ~/.local/share/fish && touch ~/.zsh_history ~/.bash_history",
+  "initializeCommand": "mkdir -p ~/.aws ~/.ssh ~/.cache/cargo ~/.local/share/fish && touch ~/.zsh_history ~/.bash_history",
+  "containerEnv": {
+    "CARGO_TARGET_DIR": "/home/ubuntu/.cache/cargo"
+  },
   "mounts": [
     {
       "source": "${localEnv:HOME}/.cache",

.github/CI_README.md

@@ -17,3 +17,9 @@ Due to some limitations of GitHub Actions CI, we need to generate our own CI yam
 1. Add your new workflow to `workflow-source`. Include any anchors you would like to use under the block `anchors`. If you name it something else, it will break.
 1. Push your changes to GitHub which will trigger CI. This will automatically run a custom script (`generate-ci.py`) which will generate the full yaml file from your anchors, as well as delete the `anchors` block, as this will not work for github actions. This new yaml file will automatically be placed in the `workflows` directory.
 1. Check that this new workflow file is correct.
+
+## Using GitHub Apps in CI
+Our CI contains several automated steps to either generate commits or PRs, usually for linting or version updates. Previously we used Personal Access Tokens generated by a service account to create a commit or PR with the correct permissions but we have now moved to GitHub apps. These have several benefits:
+- they generate a token which is only valid while the workflow is running, enhancing security and removing the need for regular rotation
+- they can be managed via the organization and do not require a separate service account
+- they can be installed on mutliple repos, however they only have access to the repo they are installed in

.github/CODEOWNERS

@@ -6,7 +6,7 @@
 /.devcontainer/           @dfinity/idx
 /buf.yaml                 @dfinity/ic-message-routing-owners
 /cpp/                     @dfinity/node
-/hs/                      @dfinity/utopia
+/hs/spec_compliance       @dfinity/research
 /licenses/                @dfinity/idx
 /bin/ict                  @dfinity/idx
 /bin/                     @dfinity/idx
@@ -71,6 +71,7 @@ go_deps.bzl               @dfinity/idx
 /.github/CODEOWNERS                                                      @dfinity/ic-owners-owners
 /ci/                                                                     @dfinity/idx
 /ci/src/dependencies/                                                    @dfinity/product-security
+/ci/src/dependencies/resources/container_scanner_finding_failover_ignore_list_guestos.txt @dfinity/node
 /.pre-commit-config.yaml                                                 @dfinity/idx
 /pre-commit/                                                             @dfinity/idx
 
@@ -207,6 +208,7 @@ go_deps.bzl               @dfinity/idx
 /rs/rust_canisters/stable_structures/                   @dfinity/execution
 /rs/rust_canisters/stable_memory_integrity              @dfinity/execution
 /rs/rust_canisters/canister_creator                     @dfinity/execution
+/rs/rust_canisters/load_simulator                       @dfinity/execution
 /rs/rust_canisters/xnet_test/                           @dfinity/ic-message-routing-owners
 /rs/rust_canisters/downstream_calls_test/               @dfinity/ic-message-routing-owners
 /rs/rust_canisters/random_traffic_test/                 @dfinity/ic-message-routing-owners
@@ -225,16 +227,21 @@ go_deps.bzl               @dfinity/idx
 /rs/test_utilities/embedders/                           @dfinity/execution
 /rs/test_utilities/execution_environment/               @dfinity/execution
 /rs/test_utilities/in_memory_logger/                    @dfinity/crypto-team
+/rs/test_utilities/metrics                              @dfinity/networking @dfinity/ic-message-routing-owners
 /rs/test_utilities/src/crypto.rs                        @dfinity/crypto-team
 /rs/test_utilities/src/crypto/                          @dfinity/crypto-team
 /rs/test_utilities/src/cycles_account_manager.rs        @dfinity/execution
+/rs/test_utilities/state/                               @dfinity/execution @dfinity/ic-message-routing-owners
 /rs/test_utilities/types/src/batch/                     @dfinity/consensus
 /rs/tests/                                              @dfinity/idx
+/rs/tests/research                                      @dfinity/research @dfinity/idx
 /rs/tests/dashboards/IC/execution-metrics.json          @dfinity/execution @dfinity/idx
 /rs/tests/dashboards/IC/bitcoin.json                    @dfinity/execution @dfinity/idx
 /rs/tests/driver/src/driver/simulate_network.rs         @dfinity/networking
 /rs/tests/boundary_nodes/                               @dfinity/boundary-node @dfinity/idx
+/rs/tests/ckbtc/                                        @dfinity/cross-chain-team @dfinity/idx
 /rs/tests/consensus/                                    @dfinity/consensus @dfinity/idx
+/rs/tests/cross_chain/                                  @dfinity/cross-chain-team @dfinity/idx
 /rs/tests/crypto/                                       @dfinity/crypto-team @dfinity/idx
 /rs/tests/dre/                                          @dfinity/dre @dfinity/idx
 /rs/tests/execution/                                    @dfinity/execution @dfinity/idx
@@ -245,26 +252,10 @@ go_deps.bzl               @dfinity/idx
 /rs/tests/node/                                         @dfinity/node @dfinity/idx
 /rs/tests/query_stats/                                  @dfinity/execution @dfinity/consensus @dfinity/idx
 /rs/tests/sdk/                                          @dfinity/sdk @dfinity/idx
-/rs/tests/src/basic_health_test.rs                      @dfinity/idx
-/rs/tests/src/btc_integration/                          @dfinity/execution @dfinity/idx
-/rs/tests/src/canister_http/                            @dfinity/networking @dfinity/idx
-/rs/tests/src/canister_sig_verification_cache_test/     @dfinity/crypto-team @dfinity/idx
-/rs/tests/src/certificate_orchestrator.rs               @dfinity/boundary-node @dfinity/idx
-/rs/tests/src/ckbtc/                                    @dfinity/cross-chain-team @dfinity/idx
-/rs/tests/src/consensus/                                @dfinity/consensus @dfinity/idx
-/rs/tests/src/cross_chain/                              @dfinity/cross-chain-team @dfinity/idx
-/rs/tests/src/crypto/                                   @dfinity/crypto-team @dfinity/idx
-/rs/tests/src/custom_domains_integration/               @dfinity/boundary-node @dfinity/idx
-/rs/tests/src/execution/                                @dfinity/execution @dfinity/idx
-/rs/tests/src/ipv4_tests/                               @dfinity/node  @dfinity/idx
 /rs/tests/src/ledger_tests/                             @dfinity/finint  @dfinity/idx
-/rs/tests/src/message_routing/                          @dfinity/ic-message-routing-owners  @dfinity/idx
-/rs/tests/src/networking/                               @dfinity/networking @dfinity/idx
 /rs/tests/src/nns_tests/                                @dfinity/nns-team @dfinity/idx
-/rs/tests/src/orchestrator/                             @dfinity/consensus @dfinity/idx
-/rs/tests/src/query_stats/                              @dfinity/execution @dfinity/consensus @dfinity/idx
 /rs/tests/src/rosetta_test.rs                           @dfinity/finint @dfinity/idx
-/rs/tests/src/tecdsa/                                   @dfinity/consensus @dfinity/idx
+/rs/tests/src/rosetta_tests/                            @dfinity/finint @dfinity/idx
 /rs/tests/k8s/                                          @dfinity/idx @dfinity/node
 /rs/tla_instrumentation/                                @dfinity/research @dfinity/formal-models
 /rs/tools/                                              @dfinity/ic-interface-owners
@@ -294,7 +285,7 @@ go_deps.bzl               @dfinity/idx
 /rs/xnet/                                               @dfinity/ic-message-routing-owners
 
 # [No-Approvals]
-# Ghost is a group with no direct members. GitLab will bypass codeowners for files that match ghost ownership.
+# Ghost is a group with no direct members. GitHub will bypass codeowners for files that match ghost ownership.
 *.lock
 Cargo.toml
 .gitignore

.github/actions/bazel-test-all/action.yaml

@@ -58,6 +58,9 @@ runs:
               rm "$exportout"
           fi
 
+          # output node name to gihub step summary
+          [ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
+
           exit "$BAZEL_EXIT_CODE"
         env:
           BAZEL_COMMAND: ${{ inputs.BAZEL_COMMAND }}

.github/workflows-source/ci-main.yml

@@ -26,14 +26,15 @@ env:
   CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
   CI_PIPELINE_SOURCE: ${{ github.event_name }}
   CI_PROJECT_DIR: ${{ github.workspace }}
+  CI_EVENT_NAME: ${{ github.event_name }}
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   CI_RUN_ID: ${{ github.run_id }}
   RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
   BUILDEVENT_DATASET: "github-ci-dfinity"
 
 anchors:
   image: &image
-    image: ghcr.io/dfinity/ic-build@sha256:0fe4b6d8d22100dddea4c26a0baca52c190141658a66542d589164bcc7611ded
+    image: ghcr.io/dfinity/ic-build@sha256:1d775e6d161dee883d10f082ab7fdde3e3d26061e1209255fb6514f8b62b206b
   dind-large-setup: &dind-large-setup
     runs-on:
       labels: dind-large
@@ -50,12 +51,6 @@ anchors:
     container:
       <<: *image
     timeout-minutes: 30
-  before-script: &before-script
-    name: Before script
-    id: before-script
-    shell: bash
-    run: |
-      [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
   docker-login: &docker-login
     name: Login to Dockerhub
     shell: bash
@@ -106,27 +101,56 @@ jobs:
       labels: dind-large
     steps:
       - <<: *checkout
-      - <<: *before-script
       - <<: *docker-login
       - name: Set BAZEL_EXTRA_ARGS
         shell: bash
         run: |
           set -xeuo pipefail
+          # Determine which tests to skip and append 'long_test' for pull requests, merge groups or push on dev-gh-*
+          EXCLUDED_TEST_TAGS=(
+              system_test_hourly
+              system_test_nightly
+              system_test_nightly_nns
+              system_test_staging
+              system_test_hotfix
+              system_test_benchmark
+              fuzz_test
+              fi_tests_nightly
+              nns_tests_nightly
+          )
+          if [[ "$CI_EVENT_NAME" =~ ^(pull_request|merge_group)$ ]]; then
+              if [[ "$CI_EVENT_NAME" == "merge_group" || "${RUN_ON_DIFF_ONLY:-}" == "true" ]]; then
+                  EXCLUDED_TEST_TAGS+=(long_test)
+              fi
+          elif [[ "$CI_EVENT_NAME" == "push" ]] && [[ "$BRANCH_NAME" =~ ^dev-gh-.* ]]; then
+              EXCLUDED_TEST_TAGS+=(long_test)
+          fi
+          # Export excluded tags as environment variable for ci/bazel-scripts/diff.sh
+          echo "EXCLUDED_TEST_TAGS=${EXCLUDED_TEST_TAGS[*]}" >> $GITHUB_ENV
+          # Prepend tags with '-' and join them with commas for Bazel
+          TEST_TAG_FILTERS=$(IFS=,; echo "${EXCLUDED_TEST_TAGS[*]/#/-}")
+          # Determine BAZEL_EXTRA_ARGS based on event type or branch name
+          BAZEL_EXTRA_ARGS="--test_tag_filters=$TEST_TAG_FILTERS"
           if [[ "${{ github.event_name }}" == 'merge_group' ]]; then
-            echo "BAZEL_EXTRA_ARGS=--test_timeout_filters=short,moderate --flaky_test_attempts=3" >> $GITHUB_ENV
+              BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate --flaky_test_attempts=3"
           elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
-            echo "BAZEL_EXTRA_ARGS=--test_timeout_filters=short,moderate" >> $GITHUB_ENV
+              BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate"
           else
-            echo "BAZEL_EXTRA_ARGS=--keep_going" >> $GITHUB_ENV
+              BAZEL_EXTRA_ARGS+=" --keep_going"
           fi
+          # Export BAZEL_EXTRA_ARGS to environment
+          echo "BAZEL_EXTRA_ARGS=$BAZEL_EXTRA_ARGS" >> $GITHUB_ENV
+        env:
+          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
       - name: Run Bazel Test All
         id: bazel-test-all
-        uses:  ./.github/actions/bazel-test-all/
+        uses: ./.github/actions/bazel-test-all/
         env:
           AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
           # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
           OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
+          CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
+          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
         with:
           BAZEL_COMMAND: "test"
           BAZEL_TARGETS: "//..."
@@ -140,9 +164,9 @@ jobs:
   bazel-build-all-config-check:
     <<: *dind-large-setup
     name: Bazel Build All Config Check
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI_BUILD_CHECK') }}
     steps:
       - <<: *checkout
-      - <<: *before-script
       - <<: *docker-login
       - name: Run bazel build --config=check //rs/...
         id: bazel-build-config-check
@@ -191,7 +215,6 @@ jobs:
     <<: *dind-large-setup
     steps:
       - <<: *checkout
-      - <<: *before-script
       - name: Run Bazel Build Fuzzers
         id: bazel-build-fuzzers
         uses:  ./.github/actions/bazel-test-all/
@@ -207,7 +230,6 @@ jobs:
     <<: *dind-large-setup
     steps:
       - <<: *checkout
-      - <<: *before-script
       - name: Run Bazel Build Fuzzers AFL
         id: bazel-build-fuzzers-afl
         uses:  ./.github/actions/bazel-test-all/
@@ -244,16 +266,20 @@ jobs:
   build-ic:
     name: Build IC
     <<: *dind-large-setup
+    # keep options from dind-large-setup but run on dind-small-setup
+    runs-on:
+      group: ch1
+      labels: dind-small
     if: ${{ github.event_name != 'merge_group' }}
     steps:
       - <<: *checkout
-      - <<: *before-script
       - <<: *docker-login
       - name: Run Build IC
         id: build-ic
         shell: bash
         run: |
           set -eExuo pipefail
+          [ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
           REPO_NAME="${GITHUB_REPOSITORY##*/}"
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
           mkdir -p "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}/artifacts"
@@ -263,10 +289,10 @@ jobs:
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
         env:
           BAZEL_COMMAND: "build"
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
       - name: Upload build-ic.tar
         uses: actions/upload-artifact@v4
         with:
@@ -329,7 +355,9 @@ jobs:
 
   cargo-clippy-linux:
     name: Cargo Clippy Linux
-    <<: *dind-large-setup
+    <<: *dind-small-setup
+    runs-on:
+      group: ch1
     steps:
       - <<: *checkout
       - name: Filter Rust Files [*.{rs,toml,lock}]
@@ -360,7 +388,9 @@ jobs:
 
   cargo-build-release-linux:
     name: Cargo Build Release Linux
-    <<: *dind-large-setup
+    <<: *dind-small-setup
+    runs-on:
+      group: ch1
     steps:
       - <<: *checkout
       - name: Filter Rust Files [*.{rs,toml,lock}]
@@ -388,21 +418,3 @@ jobs:
           set -eExuo pipefail
           buildevents cmd "$CI_RUN_ID" "$CI_JOB_NAME" build-command -- \
               cargo build --release --locked
-
-  # CI job is also executed in Schedule Hourly
-  bazel-test-coverage:
-    name: Bazel Test Coverage
-    <<: *dind-large-setup
-    if: |
-      github.event_name == 'pull_request' &&
-      contains(github.event.pull_request.labels.*.name, 'CI_COVERAGE')
-    steps:
-      - <<: *checkout
-      - <<: *before-script
-      - name: Run Bazel Test Coverage
-        shell: bash
-        run: |
-          ./ci/scripts/bazel-coverage.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}