From 9789d34f9871b222095efb2eb4a0911638d80dde Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 15:58:17 +0800
Subject: [PATCH 01/17] test sync include/exclude option

---
 .github/scripts/command/sync_fsrand.sh |   4 +-
 .github/scripts/hypo/sync.py           |  59 +++++++++++
 .github/scripts/hypo/syncrand.py       | 138 +++++++++++++++++++++++++
 3 files changed, 199 insertions(+), 2 deletions(-)
 create mode 100644 .github/scripts/hypo/sync.py
 create mode 100644 .github/scripts/hypo/syncrand.py

diff --git a/.github/scripts/command/sync_fsrand.sh b/.github/scripts/command/sync_fsrand.sh
index 17275c4ad19c..bc02e6daa828 100755
--- a/.github/scripts/command/sync_fsrand.sh
+++ b/.github/scripts/command/sync_fsrand.sh
@@ -99,10 +99,11 @@ test_update(){
     sudo -u $USER PROFILE=generate EXCLUDE_RULES=$EXCLUDE_RULES MAX_EXAMPLE=$MAX_EXAMPLE SEED=$SEED ROOT_DIR1=$SOURCE_DIR1 ROOT_DIR2=$SOURCE_DIR2 python3 .github/scripts/hypo/fsrand2.py || true
     # chmod 777 $SOURCE_DIR1
     # chmod 777 $SOURCE_DIR2
+    do_copy $sync_option
     for i in {1..5}; do
         sync_option+=" --update --delete-dst"
-        sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option 2>&1| tee sync.log || true
         echo sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option
+        sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option 2>&1| tee sync.log || true
         if grep -q "Failed to delete" sync.log; then
             echo "failed to delete, retry sync"
         else
@@ -110,7 +111,6 @@ test_update(){
             break
         fi
     done
-    do_copy $sync_option
     check_diff $DEST_DIR1 $DEST_DIR2
 }
 
diff --git a/.github/scripts/hypo/sync.py b/.github/scripts/hypo/sync.py
new file mode 100644
index 000000000000..cb215c25e729
--- /dev/null
+++ b/.github/scripts/hypo/sync.py
@@ -0,0 +1,59 @@
+import os
+import pwd
+import subprocess
+import json
+import unittest
+import common
+try: 
+    __import__('xattr')
+except ImportError:
+    subprocess.check_call(["pip", "install", "xattr"])
+import xattr
+try:
+    __import__("hypothesis")
+except ImportError:
+    subprocess.check_call(["pip", "install", "hypothesis"])
+from hypothesis import HealthCheck, assume, example, given, strategies as st, settings, Verbosity
+from hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes, invariant
+from hypothesis import Phase, seed
+from strategy import *
+from fs_op import FsOperation
+import random
+import time
+
+st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
+st_option = st.fixed_dictionaries({
+    "option": st.just("--include") | st.just("--exclude"),
+    "pattern": st_patterns
+})
+st_options = st.lists(st_option, min_size=1, max_size=5).filter(lambda self: any(item["pattern"].count('***') ==1 and item["pattern"].endswith('/***') for item in self))
+SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
+SRC = '/tmp/src_sync'
+DEST_RSYNC = '/tmp/dst_rsync'
+DEST_JUICESYNC = '/tmp/dst_juicesync'
+class TestEncoding(unittest.TestCase):
+    @seed(SEED)
+    @settings(verbosity=Verbosity.verbose, 
+              max_examples=10000, 
+              deadline=None, 
+              suppress_health_check=(HealthCheck.filter_too_much,),
+              )
+    @given(options=st_options)
+    @example(options=[{'option':'--include' ,'pattern': 'aaa/'}, {'option':'--include', 'pattern':'aaa/a'}, {'option':'--include', 'pattern':'aaa/a/***'}, {'option':'--exclude', 'pattern':'*'}])
+    def test_sync(self, options):
+        subprocess.check_call(['rm', '-rf', DEST_RSYNC])
+        subprocess.check_call(['rm', '-rf', DEST_JUICESYNC])
+        options_str = ' '.join([f'{item["option"]} {item["pattern"]}' for item in options])
+        options_str2 = ' '.join([f'{item["option"]} "{item["pattern"]}"' for item in options])
+        print(f'rm -rf {DEST_RSYNC} && rsync -r -vvv {SRC}/ {DEST_RSYNC}/ {options_str2}')
+        cmd = f'rsync -r -vvv {SRC}/ {DEST_RSYNC}/ {options_str}'
+        subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        
+        print(f'rm -rf {DEST_JUICESYNC} && ./juicefs sync --dirs -v {SRC}/ {DEST_JUICESYNC}/ {options_str2}')
+        cmd = f'./juicefs sync --dirs -v {SRC}/ {DEST_JUICESYNC}/ {options_str}'
+        subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        
+        subprocess.check_call(['diff', '-r', DEST_RSYNC, DEST_JUICESYNC])
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
new file mode 100644
index 000000000000..d2bbf6aaa478
--- /dev/null
+++ b/.github/scripts/hypo/syncrand.py
@@ -0,0 +1,138 @@
+import os
+import pwd
+import subprocess
+import json
+import common
+try: 
+    __import__('xattr')
+except ImportError:
+    subprocess.check_call(["pip", "install", "xattr"])
+import xattr
+try:
+    __import__("hypothesis")
+except ImportError:
+    subprocess.check_call(["pip", "install", "hypothesis"])
+from hypothesis import assume, strategies as st, settings, Verbosity
+from hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes, invariant
+from hypothesis import Phase, seed
+from strategy import *
+from fs_op import FsOperation
+import random
+import time
+
+st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
+SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
+@seed(SEED)
+class SyncMachine(RuleBasedStateMachine):
+    Files = Bundle('files')
+    Folders = Bundle('folders')
+    Entries = Files | Folders
+    start = time.time()
+    ROOT_DIR1 = '/tmp/src_sync'
+    ROOT_DIR2 = '/tmp/src_sync2'
+    log_level = os.environ.get('LOG_LEVEL', 'INFO')
+    logger = common.setup_logger(f'./syncrand.log', 'syncrand_logger', log_level)
+    fsop = FsOperation(logger)
+    
+    @initialize(target=Folders)
+    def init_folders(self):
+        if not os.path.exists(self.ROOT_DIR1):
+            os.makedirs(self.ROOT_DIR1)
+        if not os.path.exists(self.ROOT_DIR2):
+            os.makedirs(self.ROOT_DIR2)
+        if os.environ.get('PROFILE', 'generate') != 'generate':
+            common.clean_dir(self.ROOT_DIR1)
+            common.clean_dir(self.ROOT_DIR2)
+        return ''
+    
+    def create_users(self, users):
+        for user in users:
+            if user != 'root':
+                common.create_user(user)
+
+    def __init__(self):
+        super(SyncMachine, self).__init__()
+        print(f'__init__')
+        MAX_RUNTIME=int(os.environ.get('MAX_RUNTIME', '36000'))
+        duration = time.time() - self.start
+        print(f'duration is {duration}')
+        if duration > MAX_RUNTIME:
+            raise Exception(f'run out of time: {duration}')
+
+    def equal(self, result1, result2):
+        if type(result1) != type(result2):
+            return False
+        if isinstance(result1, Exception):
+            r1 = str(result1).replace(self.ROOT_DIR1, '')
+            r2 = str(result2).replace(self.ROOT_DIR2, '')
+            return r1 == r2
+        elif isinstance(result1, tuple):
+            return result1 == result2
+        elif isinstance(result1, str):
+            r1 = str(result1).replace(self.ROOT_DIR1, '')
+            r2 = str(result2).replace(self.ROOT_DIR2, '')
+            return  r1 == r2
+        else:
+            return result1 == result2
+
+    @rule(target=Files, 
+          parent = Folders.filter(lambda x: x != multiple()), 
+          file_name = st_entry_name, 
+          mode = st_open_mode, 
+          content = st_content, 
+          umask = st_umask, 
+            )
+    def create_file(self, parent, file_name, content, mode='x', user='root', umask=0o022):
+        result1 = self.fsop.do_create_file(self.ROOT_DIR1, parent, file_name, mode, content, user, umask)
+        result2 = self.fsop.do_create_file(self.ROOT_DIR2, parent, file_name, mode, content, user, umask)
+        assert self.equal(result1, result2), f'\033[31mcreate_file:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+        if isinstance(result1, Exception):
+            return multiple()
+        else:
+            return os.path.join(parent, file_name)
+    
+    @rule( target = Folders, 
+          parent = Folders.filter(lambda x: x != multiple()),
+          subdir = st_entry_name,
+          mode = st_entry_mode,
+          umask = st_umask, 
+          )
+    def mkdir(self, parent, subdir, mode, user='root', umask=0o022):
+        result1 = self.fsop.do_mkdir(self.ROOT_DIR1, parent, subdir, mode, user, umask)
+        result2 = self.fsop.do_mkdir(self.ROOT_DIR2, parent, subdir, mode, user, umask)
+        assert self.equal(result1, result2), f'\033[31mmkdir:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+        if isinstance(result1, Exception):
+            return multiple()
+        else:
+            return os.path.join(parent, subdir)
+
+    @rule(target = Files, 
+          dest_file = Files.filter(lambda x: x != multiple()), 
+          parent = Folders.filter(lambda x: x != multiple()), 
+          link_file_name = st_entry_name, 
+          umask = st_umask,
+            )
+    def hardlink(self, dest_file, parent, link_file_name, user='root', umask=0o022):
+        result1 = self.fsop.do_hardlink(self.ROOT_DIR1, dest_file, parent, link_file_name, user, umask)
+        result2 = self.fsop.do_hardlink(self.ROOT_DIR2, dest_file, parent, link_file_name, user, umask)
+        assert self.equal(result1, result2), f'\033[31mhardlink:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+        if isinstance(result1, Exception):
+            return multiple()
+        else:
+            return os.path.join(parent, link_file_name)
+    
+    def teardown(self):
+        pass
+
+if __name__ == '__main__':
+    MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))
+    STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))
+    settings.register_profile("generate", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
+        print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
+        report_multiple_bugs=False, \
+        phases=[Phase.generate, Phase.target])    
+    profile = os.environ.get('PROFILE', 'generate')
+    settings.load_profile(profile)
+    juicefs_machine = SyncMachine.TestCase()
+    juicefs_machine.runTest()
+    print(json.dumps(FsOperation.stats.get(), sort_keys=True, indent=4))

From 99c9eb54752d2fa74456f56f6de108798cd88b2b Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 16:34:15 +0800
Subject: [PATCH 02/17] wip

---
 .github/scripts/hypo/syncrand.py | 49 ++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index d2bbf6aaa478..55c07c6a4a5e 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -21,6 +21,13 @@
 import time
 
 st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
+st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
+st_option = st.fixed_dictionaries({
+    "option": st.just("--include") | st.just("--exclude"),
+    "pattern": st_patterns
+})
+st_options = st.lists(st_option, min_size=1, max_size=10).filter(lambda self: any(item["pattern"].count('***') ==1 and item["pattern"].endswith('/***') for item in self))
+
 SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
 @seed(SEED)
 class SyncMachine(RuleBasedStateMachine):
@@ -30,6 +37,8 @@ class SyncMachine(RuleBasedStateMachine):
     start = time.time()
     ROOT_DIR1 = '/tmp/src_sync'
     ROOT_DIR2 = '/tmp/src_sync2'
+    DEST_RSYNC = '/tmp/dst_rsync'
+    DEST_JUICESYNC = '/tmp/dst_juicesync'
     log_level = os.environ.get('LOG_LEVEL', 'INFO')
     logger = common.setup_logger(f'./syncrand.log', 'syncrand_logger', log_level)
     fsop = FsOperation(logger)
@@ -40,7 +49,7 @@ def init_folders(self):
             os.makedirs(self.ROOT_DIR1)
         if not os.path.exists(self.ROOT_DIR2):
             os.makedirs(self.ROOT_DIR2)
-        if os.environ.get('PROFILE', 'generate') != 'generate':
+        if os.environ.get('PROFILE', 'dev') != 'generate':
             common.clean_dir(self.ROOT_DIR1)
             common.clean_dir(self.ROOT_DIR2)
         return ''
@@ -106,32 +115,36 @@ def mkdir(self, parent, subdir, mode, user='root', umask=0o022):
         else:
             return os.path.join(parent, subdir)
 
-    @rule(target = Files, 
-          dest_file = Files.filter(lambda x: x != multiple()), 
-          parent = Folders.filter(lambda x: x != multiple()), 
-          link_file_name = st_entry_name, 
-          umask = st_umask,
-            )
-    def hardlink(self, dest_file, parent, link_file_name, user='root', umask=0o022):
-        result1 = self.fsop.do_hardlink(self.ROOT_DIR1, dest_file, parent, link_file_name, user, umask)
-        result2 = self.fsop.do_hardlink(self.ROOT_DIR2, dest_file, parent, link_file_name, user, umask)
-        assert self.equal(result1, result2), f'\033[31mhardlink:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
-        if isinstance(result1, Exception):
-            return multiple()
-        else:
-            return os.path.join(parent, link_file_name)
-    
+    @rule(options = st_options
+        )
+    def sync(self, options):
+        subprocess.check_call(['rm', '-rf', self.DEST_RSYNC])
+        subprocess.check_call(['rm', '-rf', self.DEST_JUICESYNC])
+        options = ' '.join([f'{item["option"]} {item["pattern"]}' for item in options])
+        self.logger.info(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options}')
+        subprocess.check_call(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        self.logger.info(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}')
+        subprocess.check_call(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        subprocess.check_call(['diff', '-r', self.DEST_RSYNC, self.DEST_JUICESYNC])
+        self.fsop.stats.success('sync')
+
     def teardown(self):
         pass
 
 if __name__ == '__main__':
     MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))
     STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))
+    settings.register_profile("dev", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
+        print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
+        report_multiple_bugs=False, 
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
     settings.register_profile("generate", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
         print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
         report_multiple_bugs=False, \
-        phases=[Phase.generate, Phase.target])    
-    profile = os.environ.get('PROFILE', 'generate')
+        phases=[Phase.generate, Phase.target])
+
+    # profile = os.environ.get('PROFILE', 'generate')
+    profile = os.environ.get('PROFILE', 'dev')
     settings.load_profile(profile)
     juicefs_machine = SyncMachine.TestCase()
     juicefs_machine.runTest()

From f5f8e8b8ecce603af4d847cdb13b593c35e8b7b0 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 22:25:16 +0800
Subject: [PATCH 03/17] wip

---
 .github/scripts/hypo/sync.py          | 59 ---------------------------
 .github/scripts/hypo/syncrand.py      | 55 ++++++++++---------------
 .github/scripts/hypo/syncrand_test.py | 13 ++++++
 3 files changed, 35 insertions(+), 92 deletions(-)
 delete mode 100644 .github/scripts/hypo/sync.py
 create mode 100644 .github/scripts/hypo/syncrand_test.py

diff --git a/.github/scripts/hypo/sync.py b/.github/scripts/hypo/sync.py
deleted file mode 100644
index cb215c25e729..000000000000
--- a/.github/scripts/hypo/sync.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import os
-import pwd
-import subprocess
-import json
-import unittest
-import common
-try: 
-    __import__('xattr')
-except ImportError:
-    subprocess.check_call(["pip", "install", "xattr"])
-import xattr
-try:
-    __import__("hypothesis")
-except ImportError:
-    subprocess.check_call(["pip", "install", "hypothesis"])
-from hypothesis import HealthCheck, assume, example, given, strategies as st, settings, Verbosity
-from hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes, invariant
-from hypothesis import Phase, seed
-from strategy import *
-from fs_op import FsOperation
-import random
-import time
-
-st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
-st_option = st.fixed_dictionaries({
-    "option": st.just("--include") | st.just("--exclude"),
-    "pattern": st_patterns
-})
-st_options = st.lists(st_option, min_size=1, max_size=5).filter(lambda self: any(item["pattern"].count('***') ==1 and item["pattern"].endswith('/***') for item in self))
-SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
-SRC = '/tmp/src_sync'
-DEST_RSYNC = '/tmp/dst_rsync'
-DEST_JUICESYNC = '/tmp/dst_juicesync'
-class TestEncoding(unittest.TestCase):
-    @seed(SEED)
-    @settings(verbosity=Verbosity.verbose, 
-              max_examples=10000, 
-              deadline=None, 
-              suppress_health_check=(HealthCheck.filter_too_much,),
-              )
-    @given(options=st_options)
-    @example(options=[{'option':'--include' ,'pattern': 'aaa/'}, {'option':'--include', 'pattern':'aaa/a'}, {'option':'--include', 'pattern':'aaa/a/***'}, {'option':'--exclude', 'pattern':'*'}])
-    def test_sync(self, options):
-        subprocess.check_call(['rm', '-rf', DEST_RSYNC])
-        subprocess.check_call(['rm', '-rf', DEST_JUICESYNC])
-        options_str = ' '.join([f'{item["option"]} {item["pattern"]}' for item in options])
-        options_str2 = ' '.join([f'{item["option"]} "{item["pattern"]}"' for item in options])
-        print(f'rm -rf {DEST_RSYNC} && rsync -r -vvv {SRC}/ {DEST_RSYNC}/ {options_str2}')
-        cmd = f'rsync -r -vvv {SRC}/ {DEST_RSYNC}/ {options_str}'
-        subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        print(f'rm -rf {DEST_JUICESYNC} && ./juicefs sync --dirs -v {SRC}/ {DEST_JUICESYNC}/ {options_str2}')
-        cmd = f'./juicefs sync --dirs -v {SRC}/ {DEST_JUICESYNC}/ {options_str}'
-        subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        subprocess.check_call(['diff', '-r', DEST_RSYNC, DEST_JUICESYNC])
-if __name__ == "__main__":
-    unittest.main()
-
diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index 55c07c6a4a5e..1e910bd60100 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -21,24 +21,27 @@
 import time
 
 st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
-st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
+st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).\
+    filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
+
+st_patterns2 = st.lists(st.sampled_from(['a','?','/','*', '/***']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('***') == -1 or (s.count('/***')==1 and s.endswith('a/***')))
 st_option = st.fixed_dictionaries({
     "option": st.just("--include") | st.just("--exclude"),
-    "pattern": st_patterns
+    "pattern": st_patterns2
 })
-st_options = st.lists(st_option, min_size=1, max_size=10).filter(lambda self: any(item["pattern"].count('***') ==1 and item["pattern"].endswith('/***') for item in self))
+st_options = st.lists(st_option, min_size=1, max_size=10).\
+    filter(lambda self: any(item["pattern"].endswith('/***') for item in self))
 
 SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
 @seed(SEED)
 class SyncMachine(RuleBasedStateMachine):
     Files = Bundle('files')
     Folders = Bundle('folders')
-    Entries = Files | Folders
-    start = time.time()
-    ROOT_DIR1 = '/tmp/src_sync'
-    ROOT_DIR2 = '/tmp/src_sync2'
-    DEST_RSYNC = '/tmp/dst_rsync'
-    DEST_JUICESYNC = '/tmp/dst_juicesync'
+    ROOT_DIR1 = '/tmp/sync_src'
+    ROOT_DIR2 = '/tmp/sync_src2'
+    DEST_RSYNC = '/tmp/rsync'
+    DEST_JUICESYNC = '/tmp/juicesync'
     log_level = os.environ.get('LOG_LEVEL', 'INFO')
     logger = common.setup_logger(f'./syncrand.log', 'syncrand_logger', log_level)
     fsop = FsOperation(logger)
@@ -49,25 +52,13 @@ def init_folders(self):
             os.makedirs(self.ROOT_DIR1)
         if not os.path.exists(self.ROOT_DIR2):
             os.makedirs(self.ROOT_DIR2)
-        if os.environ.get('PROFILE', 'dev') != 'generate':
-            common.clean_dir(self.ROOT_DIR1)
-            common.clean_dir(self.ROOT_DIR2)
+        common.clean_dir(self.ROOT_DIR1)
+        common.clean_dir(self.ROOT_DIR2)
         return ''
     
-    def create_users(self, users):
-        for user in users:
-            if user != 'root':
-                common.create_user(user)
-
     def __init__(self):
         super(SyncMachine, self).__init__()
-        print(f'__init__')
-        MAX_RUNTIME=int(os.environ.get('MAX_RUNTIME', '36000'))
-        duration = time.time() - self.start
-        print(f'duration is {duration}')
-        if duration > MAX_RUNTIME:
-            raise Exception(f'run out of time: {duration}')
-
+        
     def equal(self, result1, result2):
         if type(result1) != type(result2):
             return False
@@ -125,25 +116,23 @@ def sync(self, options):
         subprocess.check_call(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         self.logger.info(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}')
         subprocess.check_call(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.check_call(['diff', '-r', self.DEST_RSYNC, self.DEST_JUICESYNC])
-        self.fsop.stats.success('sync')
+        try:
+            subprocess.check_call(['diff', '-r', self.DEST_RSYNC, self.DEST_JUICESYNC])
+        except subprocess.CalledProcessError as e:
+            print(f'\033[31m{e}\033[0m')
+            raise e
+        self.fsop.stats.success('do_sync')
 
     def teardown(self):
         pass
 
 if __name__ == '__main__':
-    MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))
+    MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '1000'))
     STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))
     settings.register_profile("dev", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
         print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
         report_multiple_bugs=False, 
         phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
-    settings.register_profile("generate", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
-        print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
-        report_multiple_bugs=False, \
-        phases=[Phase.generate, Phase.target])
-
-    # profile = os.environ.get('PROFILE', 'generate')
     profile = os.environ.get('PROFILE', 'dev')
     settings.load_profile(profile)
     juicefs_machine = SyncMachine.TestCase()
diff --git a/.github/scripts/hypo/syncrand_test.py b/.github/scripts/hypo/syncrand_test.py
new file mode 100644
index 000000000000..0e278ecbf52e
--- /dev/null
+++ b/.github/scripts/hypo/syncrand_test.py
@@ -0,0 +1,13 @@
+import unittest
+from syncrand import SyncMachine
+
+class TestFsrand2(unittest.TestCase):
+    def test_create_hardlink(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '**/***'}])
+        state.teardown()
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From bcd3caf753f8d547e9aa9c200139218fb71fbdb3 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 22:29:55 +0800
Subject: [PATCH 04/17] wip

---
 .github/scripts/command/sync.sh |   4 +-
 .github/scripts/testSync.py     | 140 --------------------------------
 2 files changed, 1 insertion(+), 143 deletions(-)
 delete mode 100644 .github/scripts/testSync.py

diff --git a/.github/scripts/command/sync.sh b/.github/scripts/command/sync.sh
index efe4018dedea..ac06753361f8 100755
--- a/.github/scripts/command/sync.sh
+++ b/.github/scripts/command/sync.sh
@@ -117,9 +117,7 @@ do_test_sync_fsrand_with_mount_point(){
 }
 
 test_sync_randomly(){
-    prepare_test
-    [[ ! -d jfs_source ]] && git clone https://github.com/juicedata/juicefs.git jfs_source
-    META_URL=$META_URL python3 .github/scripts/testSync.py
+    MAX_EXAMPLE=1000 STEP_COUNT=200 python3 .github/scripts/hypo/syncrand.py
 }
 
 test_sync_include_exclude_option(){
diff --git a/.github/scripts/testSync.py b/.github/scripts/testSync.py
deleted file mode 100644
index a5d876e9ef67..000000000000
--- a/.github/scripts/testSync.py
+++ /dev/null
@@ -1,140 +0,0 @@
-import subprocess
-try:
-    __import__("hypothesis")
-except ImportError:
-    subprocess.check_call(["pip", "install", "hypothesis"])
-import random
-import shutil
-try:
-    __import__("hypothesis")
-except ImportError:
-    subprocess.check_call(["pip", "install", "hypothesis"])
-
-from hypothesis import given, strategies as st, settings, example
-import os
-
-JFS_SOURCE_DIR='/Users/chengzhou/Documents/juicefs/pkg/'
-JFS_SOURCE_DIR='jfs_source/pkg/'
-MOUNT_POINT='/jfs/'
-JFS_BIN='./juicefs-1.0.0-beta1'
-JFS_BIN='./juicefs-1.0.0-beta2'
-JFS_BIN='./juicefs-1.0.0-beta3'
-JFS_BIN='./juicefs'
-MAX_EXAMPLE=100
-
-def setup():
-    meta_url = os.environ.get('META_URL', 'sqlite3://test.db')
-    volume_name='myjfs'
-    if os.path.isfile('test.db'):
-        os.remove('test.db')
-    if os.path.exists(MOUNT_POINT):
-        os.system('umount %s'%MOUNT_POINT)
-    cache_dir = os.path.expanduser('~/.juicefs/local/%s/'%volume_name)
-    if os.path.exists(cache_dir):
-        try:
-            shutil.rmtree(cache_dir)
-        except OSError as e:
-            print("Error: %s : %s" % (cache_dir, e.strerror))
-    subprocess.check_call([JFS_BIN, 'format', meta_url, volume_name])
-    subprocess.check_call([JFS_BIN, 'mount', '-d', meta_url, MOUNT_POINT])
-    subprocess.check_call([JFS_BIN, 'sync', JFS_SOURCE_DIR, MOUNT_POINT+'jfs_source/'])
-
-def generate_all_entries(root_dir):
-    entries = set()
-    for root, dirs, files in os.walk(root_dir):
-        # print(root)
-        for d in dirs:
-            entries.add(d+'/')
-        for file in files:
-            entries.add(file)
-            file_path = os.path.join(root, file)[len(root_dir):]
-            entries.add(file_path)
-    print(len(entries))
-    return entries
-
-def generate_nested_dir(root_dir):
-    result = []
-    for root, dirs, files in os.walk(root_dir):
-        for d in dirs:
-            dir = os.path.join(root, d)[len(root_dir):]
-            li = dir.split('/')
-            entries = []
-            for i in range(0, len(li)):
-                entries.append('/'.join(li[i:])+'/')
-            for i in range(0, len(entries)):
-                result.append(random.sample(entries, random.randint(0, min(len(entries), 5)) ))
-    print(result)
-    return result
-
-def change_entry(entries):
-    # entries = random.sample( entries, random.randint(0, min(len(entries), 5)) )
-    options = []
-    for entry in entries:
-        type = random.choice(['--include', '--exclude'])
-        value = entry.replace(random.choice(entry), random.choice(['*', '?']), random.randint(0,2))
-        # print(type+' '+value)
-        options.append( (type, "'%s'"%value) )
-    # print(options)
-    return options
-
-all_entry = generate_all_entries(JFS_SOURCE_DIR)
-st_all_entry = st.lists(st.sampled_from(list(all_entry))).map(lambda x: change_entry(x)).filter(lambda x: len(x) != 0)
-nested_dir = generate_nested_dir(JFS_SOURCE_DIR)
-st_nested_dir = st.sampled_from(nested_dir).map(lambda x: change_entry(x)).filter(lambda x: len(x) != 0)
-valid_name = st.text(st.characters(max_codepoint=1000, blacklist_categories=('Cc', 'Cs')), min_size=2).map(lambda s: s.strip()).filter(lambda s: len(s) > 0)
-st_random_text = st.lists(valid_name).map(lambda x: change_entry(x)).filter(lambda x: len(x) != 0)
-
-@given(sync_options=st_random_text)
-@example([['--include', '[*'] ])
-@settings(max_examples=MAX_EXAMPLE, deadline=None)
-def test_sync_with_random_text(sync_options):
-    print(sync_options)
-    compare_rsync_and_juicesync(sync_options)
-
-@given(sync_options=st_all_entry)
-@settings(max_examples=MAX_EXAMPLE, deadline=None)
-def test_sync_with_path_entry(sync_options):
-    compare_rsync_and_juicesync(sync_options)
-
-@given(sync_options=st_nested_dir)
-@example([ ['--include', 'chu*/'],  ['--exclude', 'pk*/'],  ['--exclude', '*.go'] ])
-@settings(max_examples=MAX_EXAMPLE, deadline=None)
-def test_sync_with_nested_dir(sync_options):
-    compare_rsync_and_juicesync(sync_options)
-
-def compare_rsync_and_juicesync(sync_options):
-    assert sync_options != 0
-    sync_options = [item for sublist in sync_options for item in sublist]
-    do_rsync(MOUNT_POINT+'jfs_source/', 'rsync_dir/', sync_options)
-    do_juicesync(MOUNT_POINT+'jfs_source/', 'juicesync_dir/', sync_options)
-    diff_result = os.system('diff -ur juicesync_dir rsync_dir')
-    assert diff_result==0
-
-def do_juicesync(source_dir, dest_dir, sync_options):
-    if os.path.exists(dest_dir):
-        shutil.rmtree(dest_dir)
-    os.makedirs(dest_dir)
-    juicesync_cmd = [JFS_BIN , 'sync', '--dirs', source_dir, dest_dir]+sync_options
-    print('juicesync_cmd: '+' '.join(juicesync_cmd))
-    try:
-        subprocess.check_call(juicesync_cmd)
-    except Exception as e:
-        assert False
-
-def do_rsync(source_dir, dest_dir, sync_options):
-    if os.path.exists(dest_dir):
-        shutil.rmtree(dest_dir)
-    os.makedirs(dest_dir)
-    rsync_cmd = ['rsync', '-a', '-r' , source_dir,  dest_dir]+sync_options
-    print('rsync_cmd: '+ ' '.join(rsync_cmd))
-    try:
-        subprocess.check_call(rsync_cmd)
-    except Exception as e:
-        assert False
-
-if __name__ == "__main__":
-    setup()
-    test_sync_with_random_text()
-    test_sync_with_nested_dir()
-    test_sync_with_path_entry()
-    

From 31005d0b781921139e2950c181a8cf0c99a25d18 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 22:44:36 +0800
Subject: [PATCH 05/17] wip

---
 .github/scripts/hypo/syncrand_test.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/hypo/syncrand_test.py b/.github/scripts/hypo/syncrand_test.py
index 0e278ecbf52e..047dd18d62e5 100644
--- a/.github/scripts/hypo/syncrand_test.py
+++ b/.github/scripts/hypo/syncrand_test.py
@@ -2,12 +2,36 @@
 from syncrand import SyncMachine
 
 class TestFsrand2(unittest.TestCase):
-    def test_create_hardlink(self):
+
+    def test_sync1(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)
+        v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)
+        state.sync(options=[{'option': '--include', 'pattern': 'aa/***'},
+        {'option': '--exclude', 'pattern': 'a?**'}])
+        state.teardown()
+
+    def test_sync2(self):
         state = SyncMachine()
         v1 = state.init_folders()
         v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
         state.sync(options=[{'option': '--exclude', 'pattern': '**/***'}])
         state.teardown()
 
+    def test_sync3(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '/***'}])
+        state.teardown()
+
+    def test_sync4(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '*/***'}])
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file

From eb64d937e1163bbc16c439f1fb71f403cba90a37 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 22:59:30 +0800
Subject: [PATCH 06/17] wip

---
 .github/scripts/command/sync.sh |  3 +-
 .github/workflows/command.yml   | 60 +----------------------
 .github/workflows/sync.yml      | 84 +++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+), 60 deletions(-)
 create mode 100644 .github/workflows/sync.yml

diff --git a/.github/scripts/command/sync.sh b/.github/scripts/command/sync.sh
index ac06753361f8..6af17d1d23c5 100755
--- a/.github/scripts/command/sync.sh
+++ b/.github/scripts/command/sync.sh
@@ -117,7 +117,8 @@ do_test_sync_fsrand_with_mount_point(){
 }
 
 test_sync_randomly(){
-    MAX_EXAMPLE=1000 STEP_COUNT=200 python3 .github/scripts/hypo/syncrand.py
+    python3 .github/scripts/hypo/syncrand_test.py
+    LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=200 python3 .github/scripts/hypo/syncrand.py
 }
 
 test_sync_include_exclude_option(){
diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml
index 1a0045305dd2..a51f689b42ce 100644
--- a/.github/workflows/command.yml
+++ b/.github/workflows/command.yml
@@ -150,69 +150,11 @@ jobs:
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
-
-  command_test3:
-    needs: [build-matrix]
-    strategy:
-      fail-fast: true
-      matrix:
-        meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 1
-
-      - name: Build 
-        uses: ./.github/actions/build
-        # with:
-        #   useBeta: true
-
-      - name: Test Sync
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/sync.sh 
-
-      - name: Test Sync with fsrand
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/sync_fsrand.sh 
-
-      - name: Test Sync with mino
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/sync_minio.sh 
-
-      - name: Test Sync with multi nodes
-        timeout-minutes: 30
-        run: |
-          if [ "${{matrix.meta}}" != "redis" ]; then
-            echo "skip sync with multi nodes when meta is not redis"
-            exit 0
-          fi
-          # not supported algo: "dsa" "ecdsa-sk" "ed25519-sk"
-          types=("ecdsa"  "ed25519"  "rsa")
-          random_type=${types[$RANDOM % ${#types[@]}]}
-          sudo CI=true META=${{matrix.meta}} KEY_TYPE=$random_type .github/scripts/command/sync_cluster.sh 
-
-      - name: Log
-        if: always()
-        run: |
-          echo "juicefs log"
-          sudo tail -n 1000 /var/log/juicefs.log
-          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
-          
-      - name: Setup upterm session
-        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
-        # if: failure()
-        timeout-minutes: 60
-        uses: lhotari/action-upterm@v1
       
 
   success-all-test:
     runs-on: ubuntu-latest
-    needs: [command_test1, command_test2, command_test3]
+    needs: [command_test1, command_test2]
     if: always()
     steps:
       - uses: technote-space/workflow-conclusion-action@v3
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
new file mode 100644
index 000000000000..def8f80b79c2
--- /dev/null
+++ b/.github/workflows/sync.yml
@@ -0,0 +1,84 @@
+name: "sync"
+
+on:
+  push:
+    branches:
+      # - 'main'
+      - 'release-**'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    branches:
+      # - 'main'
+      - 'release-**'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  schedule:
+    - cron:  '30 20 * * *'
+
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+
+jobs:
+  sync:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Build 
+        uses: ./.github/actions/build
+        # with:
+        #   useBeta: true
+
+      - name: Test sync include/exclude option
+        timeout-minutes: 30
+        run: |
+          python3 .github/scripts/hypo/syncrand_test.py
+          LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 python3 .github/scripts/hypo/syncrand.py
+          
+      - name: Test Sync
+        timeout-minutes: 30
+        run: |
+          sudo META=redis .github/scripts/command/sync.sh 
+
+      - name: Test Sync with fsrand
+        timeout-minutes: 30
+        run: |
+          sudo META=redis .github/scripts/command/sync_fsrand.sh 
+
+      - name: Test Sync with mino
+        timeout-minutes: 30
+        run: |
+          sudo META=redis .github/scripts/command/sync_minio.sh 
+
+      - name: Test Sync with multi workers
+        timeout-minutes: 30
+        run: |
+          # not supported algo: "dsa" "ecdsa-sk" "ed25519-sk"
+          types=("ecdsa"  "ed25519"  "rsa")
+          random_type=${types[$RANDOM % ${#types[@]}]}
+          sudo CI=true META=redis KEY_TYPE=$random_type .github/scripts/command/sync_cluster.sh
+
+      - name: Log
+        if: always()
+        run: |
+          echo "juicefs log"
+          sudo tail -n 1000 /var/log/juicefs.log
+          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
+          
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        # if: failure()
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+      
\ No newline at end of file

From 0c61e4178370f59ba569ac70cdd48257311bc97a Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Mon, 4 Mar 2024 23:17:35 +0800
Subject: [PATCH 07/17] wip

---
 .github/scripts/hypo/syncrand.py      |  9 ++++++---
 .github/scripts/hypo/syncrand_test.py | 10 ++++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index 1e910bd60100..389d94ad0a4b 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -23,15 +23,18 @@
 st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
 st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).\
     filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
-
-st_patterns2 = st.lists(st.sampled_from(['a','?','/','*', '/***']), min_size=1, max_size=10)\
+st_patterns = st.lists(st.sampled_from(['a','?','/','*', '/***']), min_size=1, max_size=10)\
     .map(''.join).filter(lambda s: s.find('***') == -1 or (s.count('/***')==1 and s.endswith('a/***')))
+st_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('***') == -1 )
+
 st_option = st.fixed_dictionaries({
     "option": st.just("--include") | st.just("--exclude"),
-    "pattern": st_patterns2
+    "pattern": st_patterns
 })
 st_options = st.lists(st_option, min_size=1, max_size=10).\
     filter(lambda self: any(item["pattern"].endswith('/***') for item in self))
+st_options = st.lists(st_option, min_size=1, max_size=10)
 
 SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
 @seed(SEED)
diff --git a/.github/scripts/hypo/syncrand_test.py b/.github/scripts/hypo/syncrand_test.py
index 047dd18d62e5..fc0cd9176bf1 100644
--- a/.github/scripts/hypo/syncrand_test.py
+++ b/.github/scripts/hypo/syncrand_test.py
@@ -33,5 +33,15 @@ def test_sync4(self):
         state.sync(options=[{'option': '--exclude', 'pattern': '*/***'}])
         state.teardown()
 
+    def test_sync5(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        state.sync(options=[{'option': '--include', 'pattern': 'a'}])
+        v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)
+        v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)
+        state.sync(options=[{'option': '--include', 'pattern': 'aa'},
+        {'option': '--exclude', 'pattern': 'a?**'}])
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file

From 8272728614b34ab64b719d523472aefc4cec94a7 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 12:17:37 +0800
Subject: [PATCH 08/17] wip

---
 .github/scripts/hypo/syncrand.py              |  2 ++
 .github/scripts/{command => sync}/sync.sh     |  0
 .../scripts/{command => sync}/sync_cluster.sh |  0
 .../scripts/{command => sync}/sync_fsrand.sh  |  0
 .../scripts/{command => sync}/sync_minio.sh   |  0
 .github/workflows/sync.yml                    | 26 +++++++++++--------
 6 files changed, 17 insertions(+), 11 deletions(-)
 rename .github/scripts/{command => sync}/sync.sh (100%)
 rename .github/scripts/{command => sync}/sync_cluster.sh (100%)
 rename .github/scripts/{command => sync}/sync_fsrand.sh (100%)
 rename .github/scripts/{command => sync}/sync_minio.sh (100%)

diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index 389d94ad0a4b..19c04fc77d8a 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -27,6 +27,8 @@
     .map(''.join).filter(lambda s: s.find('***') == -1 or (s.count('/***')==1 and s.endswith('a/***')))
 st_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\
     .map(''.join).filter(lambda s: s.find('***') == -1 )
+st_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('**') == -1 )
 
 st_option = st.fixed_dictionaries({
     "option": st.just("--include") | st.just("--exclude"),
diff --git a/.github/scripts/command/sync.sh b/.github/scripts/sync/sync.sh
similarity index 100%
rename from .github/scripts/command/sync.sh
rename to .github/scripts/sync/sync.sh
diff --git a/.github/scripts/command/sync_cluster.sh b/.github/scripts/sync/sync_cluster.sh
similarity index 100%
rename from .github/scripts/command/sync_cluster.sh
rename to .github/scripts/sync/sync_cluster.sh
diff --git a/.github/scripts/command/sync_fsrand.sh b/.github/scripts/sync/sync_fsrand.sh
similarity index 100%
rename from .github/scripts/command/sync_fsrand.sh
rename to .github/scripts/sync/sync_fsrand.sh
diff --git a/.github/scripts/command/sync_minio.sh b/.github/scripts/sync/sync_minio.sh
similarity index 100%
rename from .github/scripts/command/sync_minio.sh
rename to .github/scripts/sync/sync_minio.sh
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index def8f80b79c2..edb87bade8c2 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -3,18 +3,22 @@ name: "sync"
 on:
   push:
     branches:
-      # - 'main'
+      - 'main'
       - 'release-**'
     paths-ignore:
-      - 'docs/**'
-      - '**.md'
+      - '**/sync.yml'
+      - '**/syncrand_test.py'
+      - '**/syncrand.py'
+      - '.github/scripts/sync/*.sh'
   pull_request:
     branches:
-      # - 'main'
+      - 'main'
       - 'release-**'
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+    paths:
+      - '**/sync.yml'
+      - '**/syncrand_test.py'
+      - '**/syncrand.py'
+      - '.github/scripts/sync/*.sh'
   schedule:
     - cron:  '30 20 * * *'
 
@@ -49,17 +53,17 @@ jobs:
       - name: Test Sync
         timeout-minutes: 30
         run: |
-          sudo META=redis .github/scripts/command/sync.sh 
+          sudo META=redis .github/scripts/sync/sync.sh 
 
       - name: Test Sync with fsrand
         timeout-minutes: 30
         run: |
-          sudo META=redis .github/scripts/command/sync_fsrand.sh 
+          sudo META=redis .github/scripts/sync/sync_fsrand.sh 
 
       - name: Test Sync with mino
         timeout-minutes: 30
         run: |
-          sudo META=redis .github/scripts/command/sync_minio.sh 
+          sudo META=redis .github/scripts/sync/sync_minio.sh 
 
       - name: Test Sync with multi workers
         timeout-minutes: 30
@@ -67,7 +71,7 @@ jobs:
           # not supported algo: "dsa" "ecdsa-sk" "ed25519-sk"
           types=("ecdsa"  "ed25519"  "rsa")
           random_type=${types[$RANDOM % ${#types[@]}]}
-          sudo CI=true META=redis KEY_TYPE=$random_type .github/scripts/command/sync_cluster.sh
+          sudo CI=true META=redis KEY_TYPE=$random_type .github/scripts/sync/sync_cluster.sh
 
       - name: Log
         if: always()

From e41d3977b047f7d99670f3b5b61d9ca07afe6441 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 12:37:48 +0800
Subject: [PATCH 09/17] wip

---
 .github/scripts/hypo/syncrand.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index 19c04fc77d8a..94b21af5baac 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -1,13 +1,7 @@
 import os
-import pwd
 import subprocess
 import json
 import common
-try: 
-    __import__('xattr')
-except ImportError:
-    subprocess.check_call(["pip", "install", "xattr"])
-import xattr
 try:
     __import__("hypothesis")
 except ImportError:
@@ -18,7 +12,6 @@
 from strategy import *
 from fs_op import FsOperation
 import random
-import time
 
 st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
 st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).\

From a0cdbc96745b692363d79da02c2bbf331ebe752a Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 12:43:34 +0800
Subject: [PATCH 10/17] wip

---
 .github/workflows/sync.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index edb87bade8c2..14154dad5569 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -47,8 +47,8 @@ jobs:
       - name: Test sync include/exclude option
         timeout-minutes: 30
         run: |
-          python3 .github/scripts/hypo/syncrand_test.py
-          LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 python3 .github/scripts/hypo/syncrand.py
+          sudo python3 .github/scripts/hypo/syncrand_test.py
+          LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 sudo python3 .github/scripts/hypo/syncrand.py
           
       - name: Test Sync
         timeout-minutes: 30

From efd03d3ad8c75d8d440a7c48a9a210babd54c376 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 12:46:46 +0800
Subject: [PATCH 11/17] wip

---
 .github/workflows/sync.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index 14154dad5569..2c6a5d01e402 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -47,7 +47,7 @@ jobs:
       - name: Test sync include/exclude option
         timeout-minutes: 30
         run: |
-          sudo python3 .github/scripts/hypo/syncrand_test.py
+          # sudo python3 .github/scripts/hypo/syncrand_test.py
           LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 sudo python3 .github/scripts/hypo/syncrand.py
           
       - name: Test Sync

From 1539980cdac6468ed80c867c938d234e73aed27c Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 12:52:20 +0800
Subject: [PATCH 12/17] wip

---
 .github/workflows/sync.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index 2c6a5d01e402..eb8b96a19a6a 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -48,7 +48,7 @@ jobs:
         timeout-minutes: 30
         run: |
           # sudo python3 .github/scripts/hypo/syncrand_test.py
-          LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 sudo python3 .github/scripts/hypo/syncrand.py
+          sudo LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 python3 .github/scripts/hypo/syncrand.py
           
       - name: Test Sync
         timeout-minutes: 30

From d5c90b80acb1f3a5ad20beb9ec24800d72a1dd7a Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 13:07:53 +0800
Subject: [PATCH 13/17] wip

---
 .github/scripts/hypo/syncrand.py | 4 ++++
 .github/workflows/sync.yml       | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/hypo/syncrand.py b/.github/scripts/hypo/syncrand.py
index 94b21af5baac..ae72586db196 100644
--- a/.github/scripts/hypo/syncrand.py
+++ b/.github/scripts/hypo/syncrand.py
@@ -131,6 +131,10 @@ def teardown(self):
         print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
         report_multiple_bugs=False, 
         phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
+    settings.register_profile("ci", max_examples=MAX_EXAMPLE, verbosity=Verbosity.normal, 
+        print_blob=False, stateful_step_count=STEP_COUNT, deadline=None, \
+        report_multiple_bugs=False, 
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
     profile = os.environ.get('PROFILE', 'dev')
     settings.load_profile(profile)
     juicefs_machine = SyncMachine.TestCase()
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index eb8b96a19a6a..82b66826150f 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -48,7 +48,7 @@ jobs:
         timeout-minutes: 30
         run: |
           # sudo python3 .github/scripts/hypo/syncrand_test.py
-          sudo LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=100 python3 .github/scripts/hypo/syncrand.py
+          sudo LOG_LEVEL=WARNING PROFILE=ci python3 .github/scripts/hypo/syncrand.py
           
       - name: Test Sync
         timeout-minutes: 30
@@ -85,4 +85,3 @@ jobs:
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
-      
\ No newline at end of file

From 2f010a93f74a0ead02495e67a2d1abefb71bac8d Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 13:30:23 +0800
Subject: [PATCH 14/17] wip

---
 .github/scripts/sync/sync.sh | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/scripts/sync/sync.sh b/.github/scripts/sync/sync.sh
index 6af17d1d23c5..8a3f1d07d868 100755
--- a/.github/scripts/sync/sync.sh
+++ b/.github/scripts/sync/sync.sh
@@ -76,8 +76,8 @@ test_sync_with_loop_link(){
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
     ln -s looplink jfs_source/looplink
-    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options > err.log 2>&1 || true
-    grep "Failed to handle 1 objects" err.log
+    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
+    grep -i "Failed to handle 1 objects" err.log
     rm -rf jfs_source/looplink
 }
 
@@ -116,11 +116,6 @@ do_test_sync_fsrand_with_mount_point(){
     diff -ur --no-dereference fsrand/ /jfs/fsrand/
 }
 
-test_sync_randomly(){
-    python3 .github/scripts/hypo/syncrand_test.py
-    LOG_LEVEL=WARNING MAX_EXAMPLE=1000 STEP_COUNT=200 python3 .github/scripts/hypo/syncrand.py
-}
-
 test_sync_include_exclude_option(){
     prepare_test
     ./juicefs format --trash-days 0 $META_URL myjfs

From d4e5be4b7294e8a2244e06022e7a961a0cfcc5dd Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 13:43:28 +0800
Subject: [PATCH 15/17] wip

---
 .github/scripts/sync/sync.sh |  2 +-
 .github/workflows/sync.yml   | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/scripts/sync/sync.sh b/.github/scripts/sync/sync.sh
index 8a3f1d07d868..bca5f2af62a1 100755
--- a/.github/scripts/sync/sync.sh
+++ b/.github/scripts/sync/sync.sh
@@ -77,7 +77,7 @@ test_sync_with_loop_link(){
     ./juicefs mount -d $META_URL /jfs
     ln -s looplink jfs_source/looplink
     ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
-    grep -i "Failed to handle 1 objects" err.log
+    grep -i "failed to handle 1 objects" err.log || (echo "grep failed" && exit 1)
     rm -rf jfs_source/looplink
 }
 
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index 82b66826150f..376b211bc0c0 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -44,12 +44,6 @@ jobs:
         # with:
         #   useBeta: true
 
-      - name: Test sync include/exclude option
-        timeout-minutes: 30
-        run: |
-          # sudo python3 .github/scripts/hypo/syncrand_test.py
-          sudo LOG_LEVEL=WARNING PROFILE=ci python3 .github/scripts/hypo/syncrand.py
-          
       - name: Test Sync
         timeout-minutes: 30
         run: |
@@ -73,6 +67,12 @@ jobs:
           random_type=${types[$RANDOM % ${#types[@]}]}
           sudo CI=true META=redis KEY_TYPE=$random_type .github/scripts/sync/sync_cluster.sh
 
+      - name: Test sync include/exclude option
+        timeout-minutes: 30
+        run: |
+          # sudo python3 .github/scripts/hypo/syncrand_test.py
+          sudo LOG_LEVEL=WARNING PROFILE=ci python3 .github/scripts/hypo/syncrand.py
+            
       - name: Log
         if: always()
         run: |

From e14fe323e77a072daf368f5651dc685e98b3d7be Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 13:55:19 +0800
Subject: [PATCH 16/17] wip

---
 .github/scripts/sync/sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/sync/sync.sh b/.github/scripts/sync/sync.sh
index bca5f2af62a1..0a5fe573568d 100755
--- a/.github/scripts/sync/sync.sh
+++ b/.github/scripts/sync/sync.sh
@@ -91,7 +91,7 @@ test_sync_with_deep_link(){
         ln -s symlink_$i jfs_source/symlink_$((i+1))
     done
     ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
-    grep "Failed to handle 1 objects" err.log
+    grep -i "failed to handle 1 objects" err.log || (echo "grep failed" && exit 1)
     rm -rf jfs_source/symlink_*
 }
 

From 7e0abd11fc0f86eb1ffc101c79da0e4156a45467 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Tue, 5 Mar 2024 14:17:38 +0800
Subject: [PATCH 17/17] remove check client log

---
 .github/workflows/sync.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
index 376b211bc0c0..af3596d5c00c 100644
--- a/.github/workflows/sync.yml
+++ b/.github/workflows/sync.yml
@@ -72,13 +72,6 @@ jobs:
         run: |
           # sudo python3 .github/scripts/hypo/syncrand_test.py
           sudo LOG_LEVEL=WARNING PROFILE=ci python3 .github/scripts/hypo/syncrand.py
-            
-      - name: Log
-        if: always()
-        run: |
-          echo "juicefs log"
-          sudo tail -n 1000 /var/log/juicefs.log
-          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)