feat(.github): run QUIC Interop Runner (#1682)

* feat(.github): run QUIC Interop Runner

Run the QUIC Interop Runner testcases on pull requests when entering the merge queue.

* Do not always construct comment

* Test for all pull requests to main

* Don't build arm64 on pull requests

* Use actions/download-artifact

* Provide client, server and test argument

* Fix action args

* Copy action into neqo repo

For now until quic-interop/quic-interop-runner#356 is merged.

* checkout

* yml

* introduce folder

* load image

* Only run interop on pull requests

* Remove pull_request path filter

* Trigger CI

* Cache Python deps

* Trigger CI to test python cache

.github/actions/quic-interop-runner/action.yml

@@ -0,0 +1,111 @@
+name: 'QUIC Interop Runner Action'
+description: 'Run the QUIC Interop Runner tests.'
+author: 'mxinden'
+
+inputs:
+  name:
+    description: 'Name of the QUIC implementation'
+    required: true
+  image:
+    description: 'Docker image to be tested. Needs to reside either locally, or on some registry.'
+    required: true
+  url:
+    description: 'URL of the QUIC implementation'
+    required: true
+  role:
+    description: 'client/server/both'
+    required: false
+    default: 'both'
+  client:
+    description: 'client implementations (comma-separated)'
+    required: false
+    default: ''
+  server:
+    description: 'server implementations (comma-separated)'
+    required: false
+    default: ''
+  test:
+    description: 'test cases (comma-separatated)'
+    required: false
+    default: ''
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout quic-interop/quic-interop-runner repository
+      uses: actions/checkout@v4
+      with:
+        repository: 'quic-interop/quic-interop-runner'
+        path: 'quic-interop-runner'
+
+    - name: Enable IPv6 support
+      run: sudo modprobe ip6table_filter
+      shell: bash
+
+    - name: Install dependencies
+      run: |
+        sudo add-apt-repository ppa:wireshark-dev/stable
+        sudo apt-get update
+        sudo apt-get install -y wireshark tshark jq
+      shell: bash
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.8
+        cache: 'pip'
+        cache-dependency-path: 'quic-interop-runner/requirements.txt'
+
+    - name: Install Python packages
+      run: |
+        cd quic-interop-runner
+        pip install -U pip
+        pip install -r requirements.txt
+      shell: bash
+
+    - name: Run tests
+      id: test-run
+      run: |
+        cd quic-interop-runner
+        jq --arg key "${{ inputs.name }}" --argjson newEntry '{"image": "${{ inputs.image }}", "url": "${{ inputs.url }}", "role": "${{ inputs.role }}"}' '.[$key] = $newEntry' implementations.json > temp.$$ && mv temp.$$ implementations.json
+        cat implementations.json
+        ARGS="--log-dir logs --must-include ${{ inputs.name }}"
+        if [ -n "${{ inputs.client }}" ]; then
+          ARGS="$ARGS --client ${{ inputs.client }}"
+        fi
+        if [ -n "${{ inputs.server }}" ]; then
+          ARGS="$ARGS --server ${{ inputs.server }}"
+        fi
+        if [ -n "${{ inputs.test }}" ]; then
+          ARGS="$ARGS --test ${{ inputs.test }}"
+        fi
+        python run.py $ARGS 2>&1 | tee summary
+      shell: bash
+
+    - uses: actions/upload-artifact@v4
+      id: artifact-upload-step
+      if: always()
+      with:
+        name: logs
+        path: quic-interop-runner/logs
+
+    # This action might be running off of a fork and would thus not have write
+    # permissions on the origin repository. In order to allow a separate
+    # priviledged action to post a comment on a pull request, upload the
+    # necessary metadata.
+    - name: store comment-data
+      shell: bash
+      if: github.event_name == 'pull_request'
+      env:
+        PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      run: |
+        mkdir comment-data
+        mv quic-interop-runner/summary comment-data/summary
+        echo $PULL_REQUEST_NUMBER > comment-data/pr-number
+        echo '${{ steps.artifact-upload-step.outputs.artifact-url }}' > comment-data/logs-url
+
+    - name: Upload comment data
+      uses: actions/upload-artifact@v4
+      if: github.event_name == 'pull_request'
+      with:
+        name: comment-data
+        path: ./comment-data

.github/workflows/qns-comment.yml

@@ -0,0 +1,54 @@
+# Post test results as pull request comment.
+#
+# This is done as a separate workflow as it requires write permissions. The
+# tests itself might run off of a fork, i.e. an untrusted environment and should
+# thus not be granted write permissions.
+
+name: Comment on the pull request
+
+on:
+  workflow_run:
+    workflows: ["QUIC Network Simulator"]
+    types:
+      - completed
+
+jobs:
+  comment:
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download comment-data
+        uses: actions/download-artifact@v4
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          name: comment-data
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Format GitHub comment
+        run: |
+          pwd
+          ls -la
+          echo '[**QUIC Interop Runner**](https://github.com/quic-interop/quic-interop-runner)' >> comment
+          echo '' >> comment
+          echo '```' >> comment
+          cat summary >> comment
+          echo '```' >> comment
+          echo '' >> comment
+          echo 'Download artifacts [here](' >> comment
+          cat logs-url >> comment
+          echo ').' >> comment
+        shell: bash
+
+      - name: Read PR Number
+        id: pr-number
+        run: echo "::set-output name=number::$(cat pr-number)"
+        shell: bash
+
+      - name: Comment PR
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: comment
+          pr_number: ${{ steps.pr-number.outputs.number }}

.github/workflows/qns.yml

@@ -6,12 +6,10 @@ on:
   workflow_dispatch:
   pull_request:
     branches: ["main"]
-    paths:
-      - 'qns/**'
-      - '.github/workflows/qns.yml'
   merge_group:
+
 jobs:
-  docker-image:
+  quic-network-simulator:
     runs-on: ubuntu-latest
     permissions:
       packages: write
@@ -44,6 +42,7 @@ jobs:
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Build and push
+        id: docker_build_and_push
         uses: docker/build-push-action@v5
         with:
           push: ${{ github.event_name != 'pull_request' }}
@@ -53,4 +52,21 @@ jobs:
             RUST_VERSION=stable
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          platforms: linux/amd64, linux/arm64
+          # On pull requests only build amd64 for the sake of CI time.
+          platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64, linux/arm64' }}
+          load: ${{ github.event_name == 'pull_request' }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run QUIC Interop tests
+        if: ${{ github.event_name == 'pull_request' }}
+        # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged.
+        uses: ./.github/actions/quic-interop-runner
+        with:
+          name: 'neqo-latest'
+          image: ${{ steps.docker_build_and_push.outputs.imageID }}
+          url: https://github.com/mozilla/neqo
+          test: handshake
+          client: neqo-latest,quic-go,ngtcp2,neqo,msquic
+          server: neqo-latest,quic-go,ngtcp2,neqo,msquic