From e8cd36b94ffaee931f489c8fce2f45a4e7d7b30b Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Tue, 24 Dec 2024 16:00:19 +0800 Subject: [PATCH] coverage for it --- .github/workflows/fsrand.yml | 19 +++++++++++-- .github/workflows/integrationtests.yml | 19 +++++++++++-- .github/workflows/load.yml | 20 ++++++++++++- .github/workflows/ltpfs.yml | 19 +++++++++++-- .github/workflows/ltpsyscalls.yml | 19 +++++++++++-- .github/workflows/rmfiles.yml | 39 ++++++++++++++++++-------- .github/workflows/storage.yml | 17 ++++++++++- .github/workflows/sync.yml | 29 ++++++++++++++----- .github/workflows/vdbench.yml | 21 ++++++++++++-- .github/workflows/xattr.yml | 19 +++++++++++-- 10 files changed, 187 insertions(+), 34 deletions(-) diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml index 0c2c37a2adbc..066641d8b774 100644 --- a/.github/workflows/fsrand.yml +++ b/.github/workflows/fsrand.yml @@ -44,6 +44,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -77,13 +86,13 @@ jobs: run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - ./juicefs format $meta_url --enable-acl --trash-days 0 myjfs + GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --enable-acl --trash-days 0 myjfs - name: Juicefs Mount run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr + sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr if [ ! -f /tmp/jfs/.accesslog ]; then echo ": mount failed" exit 1 @@ -132,6 +141,12 @@ jobs: grep ":" /var/log/juicefs.log && exit 1 || true fi + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) # if: failure() diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index 574f83361487..0f75b1d6f280 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -60,6 +60,15 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 1 + + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} - name: Set Variable id: vars @@ -87,13 +96,13 @@ jobs: run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs format $meta_url --trash-days 0 pics + sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --trash-days 0 pics - name: Juicefs Mount run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr + sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr stat /jfs/.accesslog - name: Fslock Test @@ -148,6 +157,12 @@ jobs: tail -300 /var/log/juicefs.log grep ":" /var/log/juicefs.log && exit 1 || true + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) timeout-minutes: 60 diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml index 88002eeb4039..7ccd22a720af 100644 --- a/.github/workflows/load.yml +++ b/.github/workflows/load.yml @@ -73,6 +73,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -91,7 +100,7 @@ jobs: - name: Load and dump with small directory timeout-minutes: 30 run: | - sudo AWS_ACCESS_KEY_ID=${{secrets.CI_AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY=${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir + sudo GOCOVERDIR=$(pwd)/cover AWS_ACCESS_KEY_ID=${{secrets.CI_AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY=${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir - name: Load and dump with big directory timeout-minutes: 30 @@ -100,6 +109,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META: ${{matrix.meta}} START_META: false + GOCOVERDIR: $(pwd)/cover run: | sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir @@ -111,6 +121,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META: ${{matrix.meta}} START_META: false + GOCOVERDIR: $(pwd)/cover run: | sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir @@ -121,6 +132,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META: ${{matrix.meta}} START_META: false + GOCOVERDIR: $(pwd)/cover run: | sudo -E .github/scripts/command/load_dump_bench.sh test_list_with_big_dir @@ -132,6 +144,12 @@ jobs: tail -300 ~/.juicefs/juicefs.log grep ":" ~/.juicefs/juicefs.log && exit 1 || true fi + + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) diff --git a/.github/workflows/ltpfs.yml b/.github/workflows/ltpfs.yml index 6a1bc67902b3..2540aebbcf72 100644 --- a/.github/workflows/ltpfs.yml +++ b/.github/workflows/ltpfs.yml @@ -27,6 +27,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Build timeout-minutes: 10 uses: ./.github/actions/build @@ -43,11 +52,11 @@ jobs: - name: Juicefs Format run: | - sudo ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics + sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics - name: Juicefs Mount run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report + sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report - name: LTP FS timeout-minutes: 50 @@ -77,6 +86,12 @@ jobs: if: always() run: | cat /opt/ltp/output/result.log.failed + + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} - name: Send Slack Notification if: failure() diff --git a/.github/workflows/ltpsyscalls.yml b/.github/workflows/ltpsyscalls.yml index 63494583c532..427f4dee9b0a 100644 --- a/.github/workflows/ltpsyscalls.yml +++ b/.github/workflows/ltpsyscalls.yml @@ -39,6 +39,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -71,13 +80,13 @@ jobs: run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs format --trash-days 0 $meta_url pics + sudo GOCOVERDIR=$(pwd)/cover ./juicefs format --trash-days 0 $meta_url pics - name: Juicefs Mount run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs mount --enable-xattr -d $meta_url /tmp/jfs --no-usage-report + sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount --enable-xattr -d $meta_url /tmp/jfs --no-usage-report - name: Install LTP Syscalls run: | @@ -123,6 +132,12 @@ jobs: run: | cat /opt/ltp/output/result.log.failed + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Send Slack Notification if: failure() uses: juicedata/slack-notify-action@main diff --git a/.github/workflows/rmfiles.yml b/.github/workflows/rmfiles.yml index 81aabd5843b3..c754221d8b87 100644 --- a/.github/workflows/rmfiles.yml +++ b/.github/workflows/rmfiles.yml @@ -26,7 +26,16 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 1 - + + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -58,12 +67,12 @@ jobs: # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz # gzip -dk 500K_empty_files.dump.gz # ./juicefs load $meta_url 500K_empty_files.dump - ./juicefs format $meta_url jfs - ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 10 --threads 10 --no-usage-report - ./juicefs mount -d $meta_url $mp --no-usage-report + GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url jfs + GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 10 --threads 10 --no-usage-report + GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report sleep 3 ls -l $mp/test - ./juicefs rmr $mp/test/ + GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ sleep 3 ls -l $mp/test && exit 1 || true @@ -91,16 +100,16 @@ jobs: # wget -q https://s.juicefs.com/static/bench/500K_empty_files.dump.gz # gzip -dk 500K_empty_files.dump.gz # ./juicefs load $meta_url 500K_empty_files.dump - ./juicefs format $meta_url jfs - ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 15 --threads 10 --no-usage-report - ./juicefs mount -d $meta_url $mp --no-usage-report + GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url jfs + GOCOVERDIR=$(pwd)/cover ./juicefs mdtest $meta_url test --dirs 10 --depth 3 --files 15 --threads 10 --no-usage-report + GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url $mp --no-usage-report sleep 3 declare -a pidlist - ./juicefs rmr $mp/test/ || true & + GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true & pidlist+=($!) - ./juicefs rmr $mp/test/ || true & + GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true & pidlist+=($!) - ./juicefs rmr $mp/test/ || true & + GOCOVERDIR=$(pwd)/cover ./juicefs rmr $mp/test/ || true & pidlist+=($!) wait "${pidlist[@]}" ls -l $mp/test && exit 1 || true @@ -111,7 +120,13 @@ jobs: run: | tail -300 ~/.juicefs/juicefs.log grep ":" ~/.juicefs/juicefs.log && exit 1 || true - + + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) timeout-minutes: 60 diff --git a/.github/workflows/storage.yml b/.github/workflows/storage.yml index ea3ad7c2a5fb..a9607ba34dcf 100644 --- a/.github/workflows/storage.yml +++ b/.github/workflows/storage.yml @@ -46,6 +46,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set environment variables run: | if [ "${{ matrix.storage }}" = "gluster" ]; then @@ -65,7 +74,13 @@ jobs: - name: Test run: | - sudo STORAGE=${{matrix.storage}} .github/scripts/storage.sh + sudo GOCOVERDIR=$(pwd)/cover STORAGE=${{matrix.storage}} .github/scripts/storage.sh + + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index e4df60911c9b..498e049b737e 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -48,6 +48,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Build uses: ./.github/actions/build @@ -55,27 +64,33 @@ jobs: timeout-minutes: 60 run: | if [[ "${{matrix.type}}" == 'sync' ]]; then - sudo META=redis .github/scripts/sync/sync.sh + sudo GOCOVERDIR=$(pwd)/cover META=redis .github/scripts/sync/sync.sh elif [[ "${{matrix.type}}" == 'sync_fsrand' ]]; then - sudo META=redis .github/scripts/sync/sync_fsrand.sh + sudo GOCOVERDIR=$(pwd)/cover META=redis .github/scripts/sync/sync_fsrand.sh elif [[ "${{matrix.type}}" == 'sync_minio' ]]; then - sudo META=redis .github/scripts/sync/sync_minio.sh + sudo GOCOVERDIR=$(pwd)/cover META=redis .github/scripts/sync/sync_minio.sh elif [[ "${{matrix.type}}" == 'sync_cluster' ]]; then types=("ecdsa" "ed25519" "rsa") random_type=${types[$RANDOM % ${#types[@]}]} - sudo CI=true META=redis KEY_TYPE=$random_type .github/scripts/sync/sync_cluster.sh + sudo CI=true GOCOVERDIR=$(pwd)/cover META=redis KEY_TYPE=$random_type .github/scripts/sync/sync_cluster.sh elif [[ "${{matrix.type}}" == 'sync_exclude' ]]; then - sudo python3 .github/scripts/hypo/sync_test.py + sudo GOCOVERDIR=$(pwd)/cover python3 .github/scripts/hypo/sync_test.py if [ "${{github.event_name}}" == "schedule" ]; then - sudo MAX_EXAMPLE=500 STEP_COUNT=100 PROFILE=ci python3 .github/scripts/hypo/sync.py + sudo GOCOVERDIR=$(pwd)/cover MAX_EXAMPLE=500 STEP_COUNT=100 PROFILE=ci python3 .github/scripts/hypo/sync.py else - sudo MAX_EXAMPLE=200 STEP_COUNT=50 PROFILE=ci python3 .github/scripts/hypo/sync.py + sudo GOCOVERDIR=$(pwd)/cover MAX_EXAMPLE=200 STEP_COUNT=50 PROFILE=ci python3 .github/scripts/hypo/sync.py fi else echo "Unknown type: ${{matrix.type}}" exit 1 fi + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) # if: failure() diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml index fea72a286b2b..9cb2313413c8 100644 --- a/.github/workflows/vdbench.yml +++ b/.github/workflows/vdbench.yml @@ -38,6 +38,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -85,8 +94,8 @@ jobs: run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - ./juicefs format $meta_url vdbench-long-run --trash-days 0 - ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --cache-size 1024 --max-deletes 50 + GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url vdbench-long-run --trash-days 0 + GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --cache-size 1024 --max-deletes 50 vdbench/vdbench -f .github/workflows/resources/vdbench_long_run.conf -jn - uses: actions/upload-artifact@v3 @@ -108,7 +117,13 @@ jobs: run: | tail -300 ~/.juicefs/juicefs.log grep ":" ~/.juicefs/juicefs.log && exit 1 || true - + + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Send Slack Notification if: failure() uses: juicedata/slack-notify-action@main diff --git a/.github/workflows/xattr.yml b/.github/workflows/xattr.yml index 0441a18f2083..8b05c7ab7fa0 100644 --- a/.github/workflows/xattr.yml +++ b/.github/workflows/xattr.yml @@ -62,6 +62,15 @@ jobs: with: fetch-depth: 1 + - name: mount coverage dir + timeout-minutes: 5 + uses: ./.github/actions/mount-coverage-dir + with: + mount_point: cover + access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} + secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} + token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} + - name: Set Variable id: vars run: | @@ -88,7 +97,7 @@ jobs: run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - sudo ./juicefs format $meta_url --trash-days 0 pics + sudo GOCOVERDIR=$(pwd)/cover ./juicefs format $meta_url --trash-days 0 pics - name: Juicefs Mount run: | @@ -96,7 +105,7 @@ jobs: meta_url=$(get_meta_url ${{matrix.meta}}) # sudo mkdir /var/jfs # sudo chmod 777 /var/jfs - sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr + sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr stat /tmp/jfs/.accesslog - name: Test @@ -120,6 +129,12 @@ jobs: grep ":" /var/log/juicefs.log && exit 1 || true fi + - name: upload coverage report + timeout-minutes: 5 + uses: ./.github/actions/upload-coverage + with: + UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} + - name: Setup upterm session if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) timeout-minutes: 60