CI: coverage report for integration test (#5425)

juicedata · Dec 24, 2024 · c022f41 · c022f41
1 parent 8fbbb9f
commit c022f41
Show file tree

Hide file tree

Showing 7 changed files with 151 additions and 4 deletions.
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -61,7 +61,7 @@ runs:
           fi
         fi
         echo "start to build ${{inputs.target}}"
-        make ${{inputs.target}}
+        make ${{inputs.target}}.cover
         if ${{ inputs.useBeta }}; then
           echo "upload beta version"
           curl --upload-file ${{inputs.target}} https://juicefs-pub.oss-cn-hangzhou.aliyuncs.com/

diff --git a/.github/actions/mount-coverage-dir/action.yml b/.github/actions/mount-coverage-dir/action.yml
@@ -0,0 +1,66 @@
+name: 'mount_coverage_dir'
+description: 'mount coverage directory'
+inputs:
+  mount_point:
+    description: 'mount point'
+    required: true
+    type: string
+  subdir:
+    description: 'subdir'
+    required: false
+    type: string
+  token:
+    description: 'token of jfs'
+    required: true
+    type: string
+  access_key:
+    description: 'access key of object storage service'
+    required: true
+    type: string
+  secret_key:
+    description: 'secret key of object storage service'
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: set subdir
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)
+        job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | select(.status=="in_progress") | .id')
+        echo Job ID is: ${job_id}
+
+        if [ "${{ github.event_name }}" == "pull_request" ]; then
+          branch=${GITHUB_BASE_REF} # 目标分支
+        elif [ "${{ github.event_name }}" == "push" ]; then
+          branch=${GITHUB_REF#refs/heads/} # 当前分支
+        else
+          branch=${GITHUB_REF#refs/heads/} # 对于 schedule 和 workflow_dispatch
+        fi
+        echo input.subdir is ${{inputs.subdir}}
+        if [ -z "${{inputs.subdir}}" ]; then
+          if [[ "${{github.event_name}}" == "schedule" ]]; then
+            subdir=juicefs/schedule/$(date +"%Y%m%d")/${{github.workflow}}
+          else
+            subdir=juicefs/pr/$branch/${{github.workflow}}/${job_id}
+          fi
+        else
+          subdir=${{inputs.subdir}}
+        fi
+        echo "subdir=$subdir"
+        echo "subdir=$subdir" >> $GITHUB_ENV
+
+    - name: mount coverage dir
+      shell: bash
+      run: |
+        sudo mkdir -p /root/.juicefs
+        sudo wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount 
+        sudo chmod +x /root/.juicefs/jfsmount
+        sudo curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs
+        sudo juicefs auth ci-coverage --access-key ${{ inputs.access_key }} --secret-key ${{ inputs.secret_key }} --token ${{inputs.token}} --encrypt-keys
+        sudo juicefs mount ci-coverage --subdir ${subdir} ${{inputs.mount_point}} --allow-other
+        
diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml
@@ -0,0 +1,33 @@
+name: 'upload_coverage_report'
+description: 'upload coverage report to oss'
+inputs:
+  type: 
+    description: 'type of the test'
+    required: true
+    default: 'integration'
+    type: string
+
+  UPLOAD_TOKEN:
+    description: 'upload token'
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: umount juicefs
+      shell: bash
+      run: |
+        sudo umount /tmp/jfs || true
+        sleep 3s
+
+    - name: generate mount coverage report
+      shell: bash
+      run: |
+        echo "generate coverage percentage report"
+        sudo go tool covdata percent -i=cover/ | sudo tee cover/percent.txt
+        echo "generate coverage text report"
+        sudo go tool covdata textfmt -i=cover/ -o cover/cover.txt
+        echo "generate coverage html report"
+        sudo go tool cover -html=cover/cover.txt -o cover/cover.html
+        .github/scripts/upload_coverage_report.sh cover/cover.html juicefs_${{github.workflow}}_${{github.run_id}}.html ${{inputs.UPLOAD_TOKEN}}
diff --git a/.github/scripts/upload_coverage_report.sh b/.github/scripts/upload_coverage_report.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# 参数检查
+if [ "$#" -ne 3 ]; then
+  echo "Usage: $0 <coverage_file> <upload_path> <token>"
+  exit 1
+fi
+
+COVERAGE_FILE=$1
+UPLOAD_PATH=$2
+TOKEN=$3
+attempt=1
+max_attempts=3
+
+while [ $attempt -le $max_attempts ]; do
+  response=$(curl -w '%{http_code}' -s -o /dev/null --form "file=@${COVERAGE_FILE}" "https://juicefs.com/upload-file-u80sdvuke/${UPLOAD_PATH}?token=${TOKEN}")
+  if [ "$response" -eq 200 ]; then
+    echo "Coverage Report: https://i.juicefs.io/ci-coverage/${UPLOAD_PATH}"
+    break
+  else
+    echo "Upload attempt $attempt failed with status code $response. Retrying..."
+    attempt=$((attempt + 1))
+    sleep 5  # 等待5秒钟后重试
+  fi
+done
+
+if [ "$response" -ne 200 ]; then
+  echo "Upload failed after $max_attempts attempts with status code $response"
+  exit 1
+fi
diff --git a/.github/workflows/cancel_outdate_runs.yml b/.github/workflows/cancel_outdate_runs.yml
@@ -21,7 +21,7 @@ jobs:
           sudo chmod +x /root/.juicefs/jfsmount
           sudo curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs
           sudo juicefs auth ci-coverage --access-key ${{ secrets.CI_COVERAGE_AWS_AK }} --secret-key ${{ secrets.CI_COVERAGE_AWS_SK }} --token ${{ secrets.CI_COVERAGE_AWS_TOKEN }} --encrypt-keys
-          sudo juicefs mount ci-coverage --subdir cancel-outdate-runs /jfs --allow-other
+          sudo juicefs mount ci-coverage --subdir juicefs/cancel-outdate-runs /jfs --allow-other
             
       - name: Get previous head_sha
         timeout-minutes: 1

diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml
@@ -67,6 +67,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Set Variable
         id: vars
         run: |
@@ -75,7 +84,7 @@ jobs:
           else
             echo "target=juicefs" >> $GITHUB_OUTPUT
           fi
-
+      
       - name: Build
         uses: ./.github/actions/build
         with: 
@@ -101,7 +110,7 @@ jobs:
           meta_url=$(get_meta_url ${{matrix.meta}})
           # sudo mkdir /var/jfs
           # sudo chmod 777 /var/jfs
-          sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report 
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report 
           stat /tmp/jfs/.accesslog
 
       - name: Pjdfstest
@@ -128,6 +137,12 @@ jobs:
             grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           fi
 
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
       - name: Send Slack Notification
         if: failure()
         uses: juicedata/slack-notify-action@main

diff --git a/Makefile b/Makefile
@@ -37,6 +37,9 @@ juicefs.ceph: Makefile cmd/*.go pkg/*/*.go
 juicefs.fdb: Makefile cmd/*.go pkg/*/*.go
 	go build -tags fdb -ldflags="$(LDFLAGS)"  -o juicefs.fdb .
 
+juicefs.fdb.cover: Makefile cmd/*.go pkg/*/*.go
+	go build -tags fdb -ldflags="$(LDFLAGS)" -cover -o juicefs.fdb .
+
 juicefs.gluster: Makefile cmd/*.go pkg/*/*.go
 	go build -tags gluster -ldflags="$(LDFLAGS)"  -o juicefs.gluster .