sync: fix check-all in cluster mode (#5525)

.github/scripts/sync/sync_cluster.sh

@@ -61,7 +61,7 @@ test_sync_without_mount_point(){
 
     sudo -u juicedata meta_url=$META_URL ./juicefs sync -v jfs://meta_url/data/ minio://minioadmin:[email protected]:9000/data1/ \
          --manager-addr 172.20.0.1:8081 --worker [email protected],[email protected] \
-         --list-threads 10 --list-depth 5 \
+         --list-threads 10 --list-depth 5 --check-new \
          2>&1 | tee sync.log
     # diff data/ /jfs/data1/
     check_sync_log $file_count
@@ -86,6 +86,10 @@ test_sync_without_mount_point2(){
          --list-threads 10 --list-depth 5\
          2>&1 | tee sync.log
     check_sync_log $file_count
+    sudo -u juicedata meta_url=$META_URL ./juicefs sync -v  minio://minioadmin:[email protected]:9000/data/ jfs://meta_url/ \
+         --manager-addr 172.20.0.1:8081 --worker [email protected],[email protected] \
+         --list-threads 10 --list-depth 5 --check-all \
+         2>&1 | tee sync.log
     ./juicefs mount -d $META_URL /jfs
     diff data/ /jfs/data/
     ./mc rm -r --force myminio/data

pkg/sync/cluster.go

@@ -330,7 +330,16 @@ func launchWorker(address string, config *Config, wg *sync.WaitGroup) {
 func marshalObjects(objs []object.Object) ([]byte, error) {
 	var arr []map[string]interface{}
 	for _, o := range objs {
-		arr = append(arr, object.MarshalObject(o))
+		obj := object.MarshalObject(o)
+		switch oo := o.(type) {
+		case *withSize:
+			obj["nsize"] = oo.nsize
+			obj["size"] = oo.Object.Size()
+		case *withFSize:
+			obj["fnsize"] = oo.nsize
+			obj["size"] = oo.File.Size()
+		}
+		arr = append(arr, obj)
 	}
 	return json.MarshalIndent(arr, "", " ")
 }
@@ -343,7 +352,13 @@ func unmarshalObjects(d []byte) ([]object.Object, error) {
 	}
 	var objs []object.Object
 	for _, m := range arr {
-		objs = append(objs, object.UnmarshalObject(m))
+		obj := object.UnmarshalObject(m)
+		if nsize, ok := m["nsize"]; ok {
+			obj = &withSize{obj, int64(nsize.(float64))}
+		} else if fnsize, ok := m["fnsize"]; ok {
+			obj = &withFSize{obj.(object.File), int64(fnsize.(float64))}
+		}
+		objs = append(objs, obj)
 	}
 	return objs, nil
 }

pkg/sync/cluster_test.go

@@ -16,6 +16,7 @@
 package sync
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -36,6 +37,9 @@ func (o *obj) Mtime() time.Time     { return o.mtime }
 func (o *obj) IsDir() bool          { return o.isDir }
 func (o *obj) IsSymlink() bool      { return o.isSymlink }
 func (o *obj) StorageClass() string { return "" }
+func (o *obj) Owner() string        { return "" }
+func (o *obj) Group() string        { return "" }
+func (o *obj) Mode() os.FileMode    { return 0 }
 
 func TestCluster(t *testing.T) {
 	// manager
@@ -62,3 +66,28 @@ func TestCluster(t *testing.T) {
 		t.Fatalf("should end")
 	}
 }
+
+func TestMarshal(t *testing.T) {
+	var objs = []object.Object{
+		&obj{key: "test"},
+		&withSize{&obj{key: "test1", size: 100}, -4},
+		&withFSize{&obj{key: "test2", size: 200}, -1},
+	}
+	d, err := marshalObjects(objs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	objs2, e := unmarshalObjects(d)
+	if e != nil {
+		t.Fatal(e)
+	}
+	if objs2[0].Key() != "test" {
+		t.Fatalf("expect test but got %s", objs2[0].Key())
+	}
+	if objs2[1].Key() != "test1" || objs2[1].Size() != -4 || objs2[1].(*withSize).Object.Size() != 100 {
+		t.Fatalf("expect withSize but got %s", objs2[0].Key())
+	}
+	if objs2[2].Key() != "test2" || objs2[2].Size() != -1 || objs2[2].(*withFSize).File.Size() != 200 {
+		t.Fatalf("expect withFSize but got %s", objs2[0].Key())
+	}
+}