CI: refactor misc test. (#4535)

.github/scripts/command/config.sh

@@ -3,8 +3,9 @@ source .github/scripts/common/common.sh
 
 [[ -z "$META" ]] && META=sqlite3
 source .github/scripts/start_meta_engine.sh
-start_meta_engine $META
+start_meta_engine $META minio
 META_URL=$(get_meta_url $META)
+[ ! -x mc ] && wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc
 
 download_juicefs_client(){
     version=$1
@@ -38,6 +39,26 @@ test_config_max_client_version()
     ./juicefs mount $META_URL /jfs -d
 }
 
+test_confi_secret_key(){
+    # # Consider command as failed when any component of the pipe fails:
+    # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
+    prepare_test
+    set -o pipefail
+    ./mc config host add minio http://127.0.0.1:9000 minioadmin minioadmin
+    ./mc admin user add minio juicedata juicedata
+    ./mc admin policy attach minio consoleAdmin --user juicedata
+    ./juicefs format --storage minio --bucket http://localhost:9000/jfs-test --access-key juicedata --secret-key juicedata $meta_url myjfs
+    ./juicefs mount $META_URL /jfs -d --io-retries 1 --no-usage-report --heartbeat 5
+
+    ./mc admin user remove minio juicedata
+    ./mc admin user add minio juicedata1 juicedata1
+    ./mc admin policy attach minio consoleAdmin --user juicedata1
+    ./juicefs config $META_URL --access-key juicedata1 --secret-key juicedata1
+    sleep 6
+    echo abc | tee /jfs/abc.txt && echo "write success"
+    cat /jfs/abc.txt | grep abc && echo "read success"
+}
+
 
 source .github/scripts/common/run_test.sh && run_test $@
 

.github/scripts/command/format.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+source .github/scripts/common/common.sh
+
+[[ -z "$META" ]] && META=sqlite3
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META
+META_URL=$(get_meta_url $META)
+
+test_mount_process_exit_on_format()
+{
+    prepare_test
+    for i in {1..3}; do 
+        echo "round $i"
+        ./juicefs format $META_URL volume-$i
+        ./juicefs mount -d $META_URL /tmp/myjfs$i_$j --no-usage-report
+        cd /tmp/myjfs$i_$j
+        bash -c 'for k in {1..300}; do echo abc>$k; sleep 0.2; done' || true & 
+        cd -
+        sleep 3
+        uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4) 
+        ./juicefs destroy --force $META_URL $uuid
+        ./juicefs format $META_URL new-volume-$i 
+        sleep 15   
+        ps -ef | grep juicefs
+        # TODO: fix the bug and remove the following line
+        # SEE https://github.com/juicedata/juicefs/issues/4534
+        # pidof juicefs && exit 1
+        uuid=$(./juicefs status $META_URL | grep UUID | cut -d '"' -f 4) 
+        ./juicefs destroy --force $META_URL $uuid
+    done
+}
+
+
+source .github/scripts/common/run_test.sh && run_test $@
+

.github/scripts/command/fsck.sh

@@ -38,7 +38,7 @@ test_sync_dir_stat()
     kill -9 $pid
     ./juicefs info -r /jfs/d
     ./juicefs info -r /jfs/d --strict 
-    ./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r -v
+    ./juicefs fsck $META_URL --path /d --sync-dir-stat --repair -r
     ./juicefs info -r /jfs/d | tee info1.log
     ./juicefs info -r /jfs/d --strict | tee info2.log
     diff info1.log info2.log

.github/scripts/command/gc.sh

@@ -1,15 +1,29 @@
 #!/bin/bash -e
 
-python3 -c "import xattr" || sudo pip install xattr 
-sudo dpkg -s redis-tools || sudo .github/scripts/apt_install.sh redis-tools
-
+python3 -c "import xattr" || pip install xattr 
+dpkg -s redis-tools || .github/scripts/apt_install.sh redis-tools
+dpkg -s fio || .github/scripts/apt_install.sh fio
 source .github/scripts/common/common.sh
 
 [[ -z "$META" ]] && META=sqlite3
 source .github/scripts/start_meta_engine.sh
 start_meta_engine $META
 META_URL=$(get_meta_url $META)
 
+test_delay_delete_slice_after_compaction(){
+    prepare_test
+    ./juicefs format $META_URL myjfs --trash-days 1
+    ./juicefs mount -d $META_URL /jfs --no-usage-report
+    fio --name=abc --rw=randwrite --refill_buffers --size=500M --bs=256k --directory=/jfs
+    redis-cli save
+    # don't skip files when gc compact
+    export JFS_SKIPPED_TIME=1
+    ./juicefs gc --compact --delete $META_URL
+    killall -9 redis-server
+    sleep 3
+    ./juicefs fsck $META_URL
+}
+
 test_gc_trash_slices(){
     prepare_test
     ./juicefs format $META_URL myjfs

.github/scripts/common/run_test.sh

@@ -35,9 +35,13 @@ run_test(){
     else
         # Find and run all test functions
         tests=$(grep -oP '^\s*test_\w+\s*\(\s*\)' "$0")
-        for test in ${tests}; do
-            run_one_test $test
-        done
+        if [[ -z "$tests" ]]; then
+            echo -e "\033[0;31mNo test function found in $0\033[0m"
+        else
+            for test in ${tests}; do
+                run_one_test $test
+            done
+        fi
     fi
     END_TIME_ALL=$(date +%s)
     ELAPSED_TIME_ALL=$((END_TIME_ALL - START_TIME_ALL))

.github/workflows/command.yml

@@ -75,6 +75,15 @@ jobs:
       - name: Build 
         timeout-minutes: 10
         uses: ./.github/actions/build
+
+      - name: Test Gc
+        timeout-minutes: 30
+        run: |
+          sudo META=${{matrix.meta}} .github/scripts/command/gc.sh      
+
+      - name: Test Format
+        run: |
+          sudo META=${{matrix.meta}} .github/scripts/command/format.sh
 
       - name: Test Config
         run: |
@@ -94,11 +103,6 @@ jobs:
         timeout-minutes: 30
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/fsck.sh 
-  
-      - name: Test Gc
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/gc.sh
 
       - name: Test Info
         run: |