CI: test sync --include/--exclude options

.github/scripts/hypo/syncrand.py

@@ -0,0 +1,142 @@
+import os
+import subprocess
+import json
+import common
+try:
+    __import__("hypothesis")
+except ImportError:
+    subprocess.check_call(["pip", "install", "hypothesis"])
+from hypothesis import assume, strategies as st, settings, Verbosity
+from hypothesis.stateful import rule, precondition, RuleBasedStateMachine, Bundle, initialize, multiple, consumes, invariant
+from hypothesis import Phase, seed
+from strategy import *
+from fs_op import FsOperation
+import random
+
+st_entry_name = st.text(alphabet='abc', min_size=1, max_size=3)
+st_patterns = st.text(alphabet='abc?/*', min_size=1, max_size=5).\
+    filter(lambda s: s.find('***') == -1 or s.endswith('/***'))
+st_patterns = st.lists(st.sampled_from(['a','?','/','*', '/***']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('***') == -1 or (s.count('/***')==1 and s.endswith('a/***')))
+st_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('***') == -1 )
+st_patterns = st.lists(st.sampled_from(['a','?','/','*']), min_size=1, max_size=10)\
+    .map(''.join).filter(lambda s: s.find('**') == -1 )
+
+st_option = st.fixed_dictionaries({
+    "option": st.just("--include") | st.just("--exclude"),
+    "pattern": st_patterns
+})
+st_options = st.lists(st_option, min_size=1, max_size=10).\
+    filter(lambda self: any(item["pattern"].endswith('/***') for item in self))
+st_options = st.lists(st_option, min_size=1, max_size=10)
+
+SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
+@seed(SEED)
+class SyncMachine(RuleBasedStateMachine):
+    Files = Bundle('files')
+    Folders = Bundle('folders')
+    ROOT_DIR1 = '/tmp/sync_src'
+    ROOT_DIR2 = '/tmp/sync_src2'
+    DEST_RSYNC = '/tmp/rsync'
+    DEST_JUICESYNC = '/tmp/juicesync'
+    log_level = os.environ.get('LOG_LEVEL', 'INFO')
+    logger = common.setup_logger(f'./syncrand.log', 'syncrand_logger', log_level)
+    fsop = FsOperation(logger)
+
+    @initialize(target=Folders)
+    def init_folders(self):
+        if not os.path.exists(self.ROOT_DIR1):
+            os.makedirs(self.ROOT_DIR1)
+        if not os.path.exists(self.ROOT_DIR2):
+            os.makedirs(self.ROOT_DIR2)
+        common.clean_dir(self.ROOT_DIR1)
+        common.clean_dir(self.ROOT_DIR2)
+        return ''
+
+    def __init__(self):
+        super(SyncMachine, self).__init__()
+
+    def equal(self, result1, result2):
+        if type(result1) != type(result2):
+            return False
+        if isinstance(result1, Exception):
+            r1 = str(result1).replace(self.ROOT_DIR1, '')
+            r2 = str(result2).replace(self.ROOT_DIR2, '')
+            return r1 == r2
+        elif isinstance(result1, tuple):
+            return result1 == result2
+        elif isinstance(result1, str):
+            r1 = str(result1).replace(self.ROOT_DIR1, '')
+            r2 = str(result2).replace(self.ROOT_DIR2, '')
+            return  r1 == r2
+        else:
+            return result1 == result2
+
+    @rule(target=Files, 
+          parent = Folders.filter(lambda x: x != multiple()), 
+          file_name = st_entry_name, 
+          mode = st_open_mode, 
+          content = st_content, 
+          umask = st_umask, 
+            )
+    def create_file(self, parent, file_name, content, mode='x', user='root', umask=0o022):
+        result1 = self.fsop.do_create_file(self.ROOT_DIR1, parent, file_name, mode, content, user, umask)
+        result2 = self.fsop.do_create_file(self.ROOT_DIR2, parent, file_name, mode, content, user, umask)
+        assert self.equal(result1, result2), f'\033[31mcreate_file:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+        if isinstance(result1, Exception):
+            return multiple()
+        else:
+            return os.path.join(parent, file_name)
+
+    @rule( target = Folders, 
+          parent = Folders.filter(lambda x: x != multiple()),
+          subdir = st_entry_name,
+          mode = st_entry_mode,
+          umask = st_umask, 
+          )
+    def mkdir(self, parent, subdir, mode, user='root', umask=0o022):
+        result1 = self.fsop.do_mkdir(self.ROOT_DIR1, parent, subdir, mode, user, umask)
+        result2 = self.fsop.do_mkdir(self.ROOT_DIR2, parent, subdir, mode, user, umask)
+        assert self.equal(result1, result2), f'\033[31mmkdir:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+        if isinstance(result1, Exception):
+            return multiple()
+        else:
+            return os.path.join(parent, subdir)
+
+    @rule(options = st_options
+        )
+    def sync(self, options):
+        subprocess.check_call(['rm', '-rf', self.DEST_RSYNC])
+        subprocess.check_call(['rm', '-rf', self.DEST_JUICESYNC])
+        options = ' '.join([f'{item["option"]} {item["pattern"]}' for item in options])
+        self.logger.info(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options}')
+        subprocess.check_call(f'rsync -r -vvv {self.ROOT_DIR1}/ {self.DEST_RSYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        self.logger.info(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}')
+        subprocess.check_call(f'./juicefs sync --dirs -v {self.ROOT_DIR1}/ {self.DEST_JUICESYNC}/ {options}'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        try:
+            subprocess.check_call(['diff', '-r', self.DEST_RSYNC, self.DEST_JUICESYNC])
+        except subprocess.CalledProcessError as e:
+            print(f'\033[31m{e}\033[0m')
+            raise e
+        self.fsop.stats.success('do_sync')
+
+    def teardown(self):
+        pass
+
+if __name__ == '__main__':
+    MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '1000'))
+    STEP_COUNT=int(os.environ.get('STEP_COUNT', '50'))
+    settings.register_profile("dev", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
+        print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
+        report_multiple_bugs=False, 
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
+    settings.register_profile("ci", max_examples=MAX_EXAMPLE, verbosity=Verbosity.normal, 
+        print_blob=False, stateful_step_count=STEP_COUNT, deadline=None, \
+        report_multiple_bugs=False, 
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
+    profile = os.environ.get('PROFILE', 'dev')
+    settings.load_profile(profile)
+    juicefs_machine = SyncMachine.TestCase()
+    juicefs_machine.runTest()
+    print(json.dumps(FsOperation.stats.get(), sort_keys=True, indent=4))

.github/scripts/hypo/syncrand_test.py

@@ -0,0 +1,47 @@
+import unittest
+from syncrand import SyncMachine
+
+class TestFsrand2(unittest.TestCase):
+
+    def test_sync1(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)
+        v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)
+        state.sync(options=[{'option': '--include', 'pattern': 'aa/***'},
+        {'option': '--exclude', 'pattern': 'a?**'}])
+        state.teardown()
+
+    def test_sync2(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '**/***'}])
+        state.teardown()
+
+    def test_sync3(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '/***'}])
+        state.teardown()
+
+    def test_sync4(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'', file_name='a', mode='w', parent=v1, umask=0)
+        state.sync(options=[{'option': '--exclude', 'pattern': '*/***'}])
+        state.teardown()
+
+    def test_sync5(self):
+        state = SyncMachine()
+        v1 = state.init_folders()
+        state.sync(options=[{'option': '--include', 'pattern': 'a'}])
+        v2 = state.mkdir(mode=0, parent=v1, subdir='a', umask=0)
+        v3 = state.create_file(content=b'', file_name=v2, mode='w', parent=v2, umask=0)
+        state.sync(options=[{'option': '--include', 'pattern': 'aa'},
+        {'option': '--exclude', 'pattern': 'a?**'}])
+        state.teardown()
+
+if __name__ == '__main__':
+    unittest.main()

.github/scripts/command/sync.sh → .github/scripts/sync/sync.sh

@@ -76,8 +76,8 @@ test_sync_with_loop_link(){
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
     ln -s looplink jfs_source/looplink
-    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options > err.log 2>&1 || true
-    grep "Failed to handle 1 objects" err.log
+    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
+    grep -i "failed to handle 1 objects" err.log || (echo "grep failed" && exit 1)
     rm -rf jfs_source/looplink
 }
 
@@ -91,7 +91,7 @@ test_sync_with_deep_link(){
         ln -s symlink_$i jfs_source/symlink_$((i+1))
     done
     ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
-    grep "Failed to handle 1 objects" err.log
+    grep -i "failed to handle 1 objects" err.log || (echo "grep failed" && exit 1)
     rm -rf jfs_source/symlink_*
 }
 
@@ -116,12 +116,6 @@ do_test_sync_fsrand_with_mount_point(){
     diff -ur --no-dereference fsrand/ /jfs/fsrand/
 }
 
-test_sync_randomly(){
-    prepare_test
-    [[ ! -d jfs_source ]] && git clone https://github.com/juicedata/juicefs.git jfs_source
-    META_URL=$META_URL python3 .github/scripts/testSync.py
-}
-
 test_sync_include_exclude_option(){
     prepare_test
     ./juicefs format --trash-days 0 $META_URL myjfs

.github/scripts/command/sync_fsrand.sh → .github/scripts/sync/sync_fsrand.sh

@@ -99,18 +99,18 @@ test_update(){
     sudo -u $USER PROFILE=generate EXCLUDE_RULES=$EXCLUDE_RULES MAX_EXAMPLE=$MAX_EXAMPLE SEED=$SEED ROOT_DIR1=$SOURCE_DIR1 ROOT_DIR2=$SOURCE_DIR2 python3 .github/scripts/hypo/fsrand2.py || true
     # chmod 777 $SOURCE_DIR1
     # chmod 777 $SOURCE_DIR2
+    do_copy $sync_option
     for i in {1..5}; do
         sync_option+=" --update --delete-dst"
-        sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option 2>&1| tee sync.log || true
         echo sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option
+        sudo -u $USER GOCOVERDIR=$GOCOVERDIR meta_url=$META_URL ./juicefs sync $SOURCE_DIR1 jfs://meta_url/fsrand1/ $sync_option 2>&1| tee sync.log || true
         if grep -q "Failed to delete" sync.log; then
             echo "failed to delete, retry sync"
         else
             echo "sync delete success"
             break
         fi
     done
-    do_copy $sync_option
     check_diff $DEST_DIR1 $DEST_DIR2
 }