coverage for it

juicedata · Dec 24, 2024 · e8cd36b · e8cd36b
1 parent 2c63cbe
commit e8cd36b
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 34 deletions.
diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml
@@ -44,6 +44,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Set Variable
         id: vars
         run: |
@@ -77,13 +86,13 @@ jobs:
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          ./juicefs format $meta_url --enable-acl --trash-days 0 myjfs
+          GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --enable-acl --trash-days 0 myjfs
 
       - name: Juicefs Mount
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr 
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr 
           if [ ! -f /tmp/jfs/.accesslog ]; then
             echo "<FATAL>: mount failed"
             exit 1
@@ -132,6 +141,12 @@ jobs:
             grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           fi
 
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure() 

diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml
@@ -60,6 +60,15 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 1
+
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
 
       - name: Set Variable
         id: vars
@@ -87,13 +96,13 @@ jobs:
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          sudo ./juicefs format $meta_url --trash-days 0 pics
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --trash-days 0 pics
 
       - name: Juicefs Mount
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr
           stat /jfs/.accesslog
 
       - name: Fslock Test
@@ -148,6 +157,12 @@ jobs:
           tail -300 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60

diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
@@ -73,6 +73,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Set Variable
         id: vars
         run: |
@@ -91,7 +100,7 @@ jobs:
       - name: Load and dump with small directory
         timeout-minutes: 30
         run: |
-          sudo AWS_ACCESS_KEY_ID=${{secrets.CI_AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY=${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
+          sudo GOCOVERDIR=$(pwd)/cover AWS_ACCESS_KEY_ID=${{secrets.CI_AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY=${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
 
       - name: Load and dump with big directory
         timeout-minutes: 30
@@ -100,6 +109,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           META: ${{matrix.meta}}
           START_META: false
+          GOCOVERDIR: $(pwd)/cover
         run: |
           sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
       
@@ -111,6 +121,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           META: ${{matrix.meta}} 
           START_META: false 
+          GOCOVERDIR: $(pwd)/cover
         run: |
           sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
           
@@ -121,6 +132,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           META: ${{matrix.meta}} 
           START_META: false 
+          GOCOVERDIR: $(pwd)/cover
         run: | 
           sudo -E .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
 
@@ -132,6 +144,12 @@ jobs:
             tail -300 ~/.juicefs/juicefs.log
             grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
           fi
+
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
 
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)

diff --git a/.github/workflows/ltpfs.yml b/.github/workflows/ltpfs.yml
@@ -27,6 +27,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Build
         timeout-minutes: 10
         uses: ./.github/actions/build
@@ -43,11 +52,11 @@ jobs:
 
       - name: Juicefs Format
         run: |
-          sudo ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics
 
       - name: Juicefs Mount
         run: |
-          sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report
 
       - name: LTP FS
         timeout-minutes: 50
@@ -77,6 +86,12 @@ jobs:
         if: always()
         run: |
           cat /opt/ltp/output/result.log.failed
+          
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
 
       - name: Send Slack Notification
         if: failure()

diff --git a/.github/workflows/ltpsyscalls.yml b/.github/workflows/ltpsyscalls.yml
@@ -39,6 +39,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Set Variable
         id: vars
         run: |
@@ -71,13 +80,13 @@ jobs:
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          sudo ./juicefs format --trash-days 0 $meta_url pics
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 $meta_url pics
 
       - name: Juicefs Mount
         run: |
           source .github/scripts/start_meta_engine.sh
           meta_url=$(get_meta_url ${{matrix.meta}})
-          sudo ./juicefs mount --enable-xattr -d $meta_url /tmp/jfs --no-usage-report
+          sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount --enable-xattr -d $meta_url /tmp/jfs --no-usage-report
 
       - name: Install LTP Syscalls
         run: |
@@ -123,6 +132,12 @@ jobs:
         run: |
           cat /opt/ltp/output/result.log.failed
 
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
       - name: Send Slack Notification
         if: failure()
         uses: juicedata/slack-notify-action@main

diff --git a/.github/workflows/rmfiles.yml b/.github/workflows/rmfiles.yml
@@ -26,7 +26,16 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 1
-
+
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} 
+
       - name: Set Variable
         id: vars
         run: |
@@ -58,12 +67,12 @@ jobs:
           # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz
           # gzip -dk  500K_empty_files.dump.gz
           # ./juicefs load $meta_url 500K_empty_files.dump
-          ./juicefs format $meta_url jfs
-          ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 10 --threads 10 --no-usage-report
-          ./juicefs mount -d $meta_url $mp --no-usage-report
+          GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url jfs
+          GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 10 --threads 10 --no-usage-report
+          GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report
           sleep 3
           ls -l $mp/test
-          ./juicefs rmr $mp/test/
+          GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/
           sleep 3 
           ls -l $mp/test && exit 1 || true
         
@@ -91,16 +100,16 @@ jobs:
           # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz
           # gzip -dk  500K_empty_files.dump.gz
           # ./juicefs load $meta_url 500K_empty_files.dump
-          ./juicefs format $meta_url jfs
-          ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 15 --threads 10 --no-usage-report
-          ./juicefs mount -d $meta_url $mp --no-usage-report
+          GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url jfs
+          GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 15 --threads 10 --no-usage-report
+          GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report
           sleep 3
           declare -a pidlist
-          ./juicefs rmr $mp/test/ || true &
+          GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &
           pidlist+=($!)
-          ./juicefs rmr $mp/test/ || true &
+          GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &
           pidlist+=($!)
-          ./juicefs rmr $mp/test/ || true &
+          GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true &
           pidlist+=($!)
           wait "${pidlist[@]}"
           ls -l $mp/test && exit 1 || true
@@ -111,7 +120,13 @@ jobs:
         run: | 
           tail -300 ~/.juicefs/juicefs.log
           grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
-          
+
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}          
+
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60

diff --git a/.github/workflows/storage.yml b/.github/workflows/storage.yml
@@ -46,6 +46,15 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Set environment variables
         run: |
           if [ "${{ matrix.storage }}" = "gluster" ]; then
@@ -65,7 +74,13 @@ jobs:
 
       - name: Test
         run: |
-          sudo STORAGE=${{matrix.storage}} .github/scripts/storage.sh
+          sudo GOCOVERDIR=$(pwd)/cover STORAGE=${{matrix.storage}} .github/scripts/storage.sh
+
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}       
 
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)