Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: fix save benchmark #5449

Merged
merged 6 commits into from
Dec 27, 2024
Merged

CI: fix save benchmark #5449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3088dc8
fix save benchmark
zhoucheng361 Dec 26, 2024
1cac1b8
wip
zhoucheng361 Dec 26, 2024
c691c51
wip
zhoucheng361 Dec 26, 2024
0211455
wip
zhoucheng361 Dec 26, 2024
fce4bfb
wip
zhoucheng361 Dec 26, 2024
b40ab4b
wip
zhoucheng361 Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/actions/sysbench/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,18 @@ inputs:
description: 'file test mode'
required: true
default: 'seqrd'
awk_access_key_id:
aws_access_key_id:
description: 'aws access key id'
required: true
default: ''
awk_access_key_secret:
aws_access_key_secret:
description: 'aws access key secret'
required: true
default: ''
aws_access_token:
description: 'aws access token'
required: true
default: ''
meta:
description: 'meta'
required: true
Expand Down Expand Up @@ -57,4 +61,4 @@ runs:
echo "cost $runtime seconds"
cd -
version=$(./juicefs -V|cut -b 17- | sed 's/:/-/g')
AWS_ACCESS_KEY_ID=${{inputs.aws_access_key_id}} AWS_SECRET_ACCESS_KEY=${{inputs.aws_access_key_secret}} .github/scripts/save_benchmark.sh --name ${{inputs.name}} --result $runtime --meta ${{inputs.meta}} --storage ${{inputs.storage}} --extra ${{inputs.compress}}
sudo -E AWS_ACCESS_KEY=${{inputs.aws_access_key_id}} AWS_SECRET_KEY=${{inputs.aws_access_key_secret}} AWS_ACCESS_TOKEN=${{inputs.aws_access_token}} .github/scripts/save_benchmark.sh --name ${{inputs.name}} --result $runtime --meta ${{inputs.meta}} --storage ${{inputs.storage}} --extra ${{inputs.compress}}
15 changes: 13 additions & 2 deletions .github/scripts/save_benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
#/bin/bash -e

mount_jfs(){
mkdir -p /root/.juicefs
wget -q s.juicefs.com/static/Linux/mount -O /root/.juicefs/jfsmount
chmod +x /root/.juicefs/jfsmount
curl -s -L https://juicefs.com/static/juicefs -o /usr/local/bin/juicefs && sudo chmod +x /usr/local/bin/juicefs
juicefs auth ci-coverage --access-key $AWS_ACEESS_KEY --secret-key $AWS_SECRET_KEY --token $AWS_ACCESS_TOKEN --encrypt-keys
juicefs mount ci-coverage --subdir juicefs/ci-benchmark/ --allow-other /ci-benchmark
}

save_benchmark(){
while [[ $# -gt 0 ]]; do
key="$1"
Expand Down Expand Up @@ -56,8 +65,10 @@ save_benchmark(){
EOF
cat result.json
if [[ "$GITHUB_EVENT_NAME" == "schedule" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
echo "save benchmark"
AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/$GITHUB_WORKFLOW/$name/$created_date/$meta-$storage.json
mount_jfs
echo "save result.json to /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/$meta-$storage.json"
mkdir -p /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/
cp result.json /ci-benchmark/$GITHUB_WORKFLOW/$name/$created_date/$meta-$storage.json
fi
}

Expand Down
26 changes: 17 additions & 9 deletions .github/workflows/load.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,20 @@ jobs:

- name: Load and dump with small directory
timeout-minutes: 30
env:
AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}
AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
META: ${{matrix.meta}}
run: |
sudo GOCOVERDIR=$(pwd)/cover AWS_ACCESS_KEY_ID=${{secrets.CI_AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY=${{secrets.CI_AWS_ACCESS_KEY_SECRET}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
sudo GOCOVERDIR=$(pwd)/cover START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir

- name: Load and dump with big directory
timeout-minutes: 30
env:
AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}
AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
META: ${{matrix.meta}}
START_META: false
run: |
Expand All @@ -116,19 +122,21 @@ jobs:
if: false
timeout-minutes: 30
env:
AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
META: ${{matrix.meta}}
AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}
AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
META: ${{matrix.meta}}
START_META: false
run: |
sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir

- name: List big directory
timeout-minutes: 30
env:
AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
META: ${{matrix.meta}}
AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_COVERAGE_AWS_SK}}
AWS_ACCESS_TOKEN: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
META: ${{matrix.meta}}
START_META: false
run: |
sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
Expand Down
32 changes: 19 additions & 13 deletions .github/workflows/sysbench.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
sysbench:
timeout-minutes: 60
strategy:
fail-fast: false
fail-fast: true
matrix:
meta: [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'mariadb', 'badger', 'fdb']
# meta: ['redis']
Expand Down Expand Up @@ -100,8 +100,9 @@ jobs:
- name: seq read 1M
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 1000
file_total_size: '1G'
file_test_mode: 'seqrd'
Expand All @@ -113,8 +114,9 @@ jobs:
- name: seq write 1M
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 1000
file_total_size: '1G'
file_test_mode: 'seqwr'
Expand All @@ -126,8 +128,9 @@ jobs:
- name: seq read 100K
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 10000
file_total_size: '1G'
file_test_mode: 'seqrd'
Expand All @@ -139,8 +142,9 @@ jobs:
- name: seq write 100K
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 10000
file_total_size: '1G'
file_test_mode: 'seqwr'
Expand All @@ -152,8 +156,9 @@ jobs:
- name: random readwrite 1M
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 1000
file_total_size: '1G'
file_test_mode: 'rndrw'
Expand All @@ -165,8 +170,9 @@ jobs:
- name: random readwrite 100K
uses: ./.github/actions/sysbench
with:
aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
aws_access_key_id: ${{secrets.CI_COVERAGE_AWS_AK}}
aws_access_key_secret: ${{secrets.CI_COVERAGE_AWS_SK}}
awk_access_token: ${{secrets.CI_COVERAGE_AWS_TOKEN}}
file_num: 10000
file_total_size: '1G'
file_test_mode: 'rndrw'
Expand Down
Loading