From 166b29c072fdc2ff6dbe8856db54fcb0c3b3e350 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Tue, 30 May 2023 11:09:45 +0800 Subject: [PATCH 01/26] use NFS as object store --- go.mod | 8 +- go.sum | 4 + pkg/object/nfs.go | 276 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 pkg/object/nfs.go diff --git a/go.mod b/go.mod index 1fd800fce001..992ee4d5e141 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/urfave/cli/v2 v2.25.3 github.com/vbauerster/mpb/v7 v7.0.3 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 + github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.3 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.etcd.io/etcd v3.3.27+incompatible @@ -79,7 +80,10 @@ require ( xorm.io/xorm v1.0.7 ) -require github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect +require ( + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect +) require ( cloud.google.com/go v0.102.1 // indirect @@ -258,3 +262,5 @@ replace xorm.io/xorm v1.0.7 => gitea.com/davies/xorm v1.0.8-0.20220528043536-552 replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible replace github.com/urfave/cli/v2 v2.25.3 => github.com/juicedata/cli/v2 v2.25.4-0.20230526070816-8aff66437fa8 + +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13 diff --git a/go.sum b/go.sum index 04962312322c..6be63939768f 100644 --- a/go.sum +++ b/go.sum @@ -625,6 +625,8 @@ github.com/juicedata/cli/v2 v2.25.4-0.20230526070816-8aff66437fa8 h1:P+Qaebmynxe github.com/juicedata/cli/v2 v2.25.4-0.20230526070816-8aff66437fa8/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5u5iRL/e1yr/zXzplhdlaIGwKcOmGRSOfasQ= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= +github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13 h1:/mwt/cUF+mj3+G7qneLsqwAvtAPvKlLbnsPCDn2rW0k= +github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= @@ -895,6 +897,8 @@ github.com/qiniu/go-sdk/v7 v7.15.0 h1:vkxZZHM2Ed0qHeIx7NF3unXav+guaVIXlEsCCkpQAw github.com/qiniu/go-sdk/v7 v7.15.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go new file mode 100644 index 000000000000..9e84f1ec081f --- /dev/null +++ b/pkg/object/nfs.go @@ -0,0 +1,276 @@ +//go:build !nonfs +// +build !nonfs + +/* + * JuiceFS, Copyright 2023 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "os/user" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/vmware/go-nfs-client/nfs" + "github.com/vmware/go-nfs-client/nfs/rpc" +) + +type nfsStore struct { + DefaultObjectStorage + username string + host string + root string + + target *nfs.Target +} + +func (n *nfsStore) String() string { + return fmt.Sprintf("nfs://%s@%s:%s", n.username, n.host, n.root) +} + +func (n *nfsStore) path(key string) string { + return n.root + key +} + +func (n *nfsStore) Head(key string) (Object, error) { + fi, _, err := n.target.Lookup(n.path(key)) + if err != nil { + return nil, err + } + return n.fileInfo(key, fi), nil +} + +func (n *nfsStore) Get(key string, off, limit int64) (io.ReadCloser, error) { + p := n.path(key) + ff, err := n.target.Open(p) + if err != nil { + return nil, err + } + if strings.HasSuffix(p, "/") { + _ = ff.Close() + return io.NopCloser(bytes.NewBuffer([]byte{})), nil + } + + if limit > 0 { + return &SectionReaderCloser{ + SectionReader: io.NewSectionReader(ff, off, limit), + Closer: ff, + }, nil + } + return ff, err +} + +func (n *nfsStore) mkdirAll(path string, perm fs.FileMode) error { + path = strings.TrimSuffix(path, "/") + fi, _, err := n.target.Lookup(path) + if err == nil { + if fi.IsDir() { + logger.Tracef("nfs mkdir: path %s already exists", path) + return nil + } else { + return syscall.ENOTDIR + } + } else if !os.IsNotExist(err) { + return err + } + + dir, _ := filepath.Split(path) + if dir != "." { + if err = n.mkdirAll(dir, perm); err != nil { + return err + } + } + _, err = n.target.Mkdir(path, perm) + return err +} + +func (n *nfsStore) Put(key string, in io.Reader) error { + p := n.path(key) + if strings.HasSuffix(p, dirSuffix) { + return n.mkdirAll(p, 0777) + } + tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp") + _, err := n.target.Create(tmp, 0777) + if os.IsNotExist(err) { + _ = n.mkdirAll(filepath.Dir(p), 0777) + _, err = n.target.Create(tmp, 0777) + } + if err != nil { + return err + } + ff, err := n.target.Open(tmp) + if err != nil { + return err + } + defer func() { _ = n.target.Remove(tmp) }() + buf := bufPool.Get().(*[]byte) + defer bufPool.Put(buf) + _, err = io.CopyBuffer(ff, in, *buf) + if err != nil { + _ = ff.Close() + return err + } + err = ff.Close() + if err != nil { + return err + } + // _ = n.target.Remove(p) + return n.target.Rename(tmp, p) +} + +func (n *nfsStore) Delete(key string) error { + err := n.target.Remove(strings.TrimRight(n.path(key), dirSuffix)) + if err != nil && os.IsNotExist(err) { + err = nil + } + return err +} + +func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object { + owner, group := getOwnerGroup(fi) + isSymlink := !fi.Mode().IsDir() && !fi.Mode().IsRegular() + ff := &file{ + obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""}, + owner, + group, + fi.Mode(), + isSymlink, + } + if fi.IsDir() { + if key != "" && !strings.HasSuffix(key, "/") { + ff.key += "/" + } + ff.size = 0 + } + return ff +} + +func (n *nfsStore) sortByName(path string, fis []*nfs.EntryPlus) []Object { + var obs = make([]Object, 0, len(fis)) + for _, fi := range fis { + p := path + fi.Name() + if strings.HasPrefix(p, n.root) { + key := p[len(n.root):] + obs = append(obs, n.fileInfo(key, fi)) + } + } + sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() }) + return obs +} + +func (n *nfsStore) doFind(path, marker string, out chan Object) { + infos, err := n.target.ReadDirPlus(path) + if err != nil { + logger.Errorf("readdir %s: %s", path, err) + return + } + + obs := n.sortByName(path, infos) + for _, o := range obs { + key := o.Key() + if key > marker { + out <- o + } + if o.IsDir() && (key > marker || strings.HasPrefix(marker, key)) { + n.doFind(n.root+key, marker, out) + } + } +} + +func (n *nfsStore) find(path, marker string, out chan Object) { + if strings.HasSuffix(path, dirSuffix) { + fi, _, err := n.target.Lookup(path) + if err != nil { + logger.Errorf("Stat %s error: %q", path, err) + return + } + if marker == "" { + out <- n.fileInfo(path[len(n.root):], fi) + } + n.doFind(path, marker, out) + } else { + // As files or dirs in the same directory of file `path` resides + // may have prefix `path`, we should list the directory. + dir := filepath.Dir(path) + dirSuffix + infos, err := n.target.ReadDirPlus(dir) + if err != nil { + logger.Errorf("readdir %s: %s", dir, err) + return + } + + obs := n.sortByName(dir, infos) + for _, o := range obs { + key := o.Key() + p := n.root + o.Key() + if strings.HasPrefix(p, path) { + if key > marker || marker == "" { + out <- o + } + if o.IsDir() && (key > marker || strings.HasPrefix(marker, key)) { + n.doFind(p, marker, out) + } + } + } + } +} + +func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object, error) { + return nil, notSupported +} + +func (n *nfsStore) ListAll(prefix, marker string) (<-chan Object, error) { + return nil, notSupported +} + +func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { + if strings.Contains(addr, "@") { + ps := strings.Split(addr, "@") + username = ps[0] + addr = ps[1] + } else { + u, err := user.Current() + if err != nil { + return nil, fmt.Errorf("current user: %s", err) + } + username = u.Username + } + b := strings.Split(addr, ":") + host := b[0] + path := b[1] + mount, err := nfs.DialMount(host, time.Second*3) + if err != nil { + return nil, fmt.Errorf("unable to dial MOUNT service %s: %v", addr, err) + } + auth := rpc.NewAuthUnix(username, uint32(os.Getuid()), uint32(os.Getgid())) + target, err := mount.Mount(path, auth.Auth()) + if err != nil { + return nil, fmt.Errorf("unable to mount %s: %v", addr, err) + } + return &nfsStore{DefaultObjectStorage{}, username, host, path, target}, nil + +} + +func init() { + Register("nfs", newNFSStore) +} From 7050d56e5e53b129d530047fc041604e2c1958f8 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 9 Jun 2023 15:00:30 +0800 Subject: [PATCH 02/26] upgrade nfs client: file support ReadAt Signed-off-by: xixi --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 46b4a89dcb70..fd4a2feac9df 100644 --- a/go.mod +++ b/go.mod @@ -263,4 +263,4 @@ replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => g replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 -replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13 +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c diff --git a/go.sum b/go.sum index a9cf804bde51..42ffba405f46 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,8 @@ github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnT github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5u5iRL/e1yr/zXzplhdlaIGwKcOmGRSOfasQ= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= -github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13 h1:/mwt/cUF+mj3+G7qneLsqwAvtAPvKlLbnsPCDn2rW0k= -github.com/juicedata/go-nfs-client v0.0.0-20211031095452-1aeeb221bf13/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= +github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c h1:mHLyhF5JfwCZxfOz0y90xWBi5UvHwqOVKIbNWw7iAUo= +github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= From 53cd38b9a56bff17469c62fff245202737b0cebe Mon Sep 17 00:00:00 2001 From: xixi Date: Mon, 12 Jun 2023 15:43:19 +0800 Subject: [PATCH 03/26] implement List for nfsStore Signed-off-by: xixi --- pkg/object/nfs.go | 95 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index 9e84f1ec081f..a36631cce245 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -26,16 +26,20 @@ import ( "io/fs" "os" "os/user" + "path" "path/filepath" "sort" "strings" "syscall" "time" + "github.com/pkg/errors" "github.com/vmware/go-nfs-client/nfs" "github.com/vmware/go-nfs-client/nfs/rpc" ) +var _ ObjectStorage = &nfsStore{} + type nfsStore struct { DefaultObjectStorage username string @@ -45,6 +49,11 @@ type nfsStore struct { target *nfs.Target } +type nfsEntry struct { + *nfs.EntryPlus + name string +} + func (n *nfsStore) String() string { return fmt.Sprintf("nfs://%s@%s:%s", n.username, n.host, n.root) } @@ -235,8 +244,92 @@ func (n *nfsStore) find(path, marker string, out chan Object) { } } +func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { + entries, err := n.target.ReadDirPlus(dirname) + if err != nil { + return nil, errors.Wrapf(err, "readdir %s", dirname) + } + nfsEntries := make([]*nfsEntry, len(entries)) + + for i, e := range entries { + if e.IsDir() { + nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix} + } else { + nfsEntries[i] = &nfsEntry{e, e.Name()} + } + } + sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() }) + return nfsEntries, err +} + +func (d *nfsStore) toFile(key string, e *nfsEntry) *file { + size := e.Size() + if e.IsDir() { + size = 0 + } + owner, group := getOwnerGroup(e) + return &file{ + obj{ + key, + size, + e.ModTime(), + e.IsDir(), + "", + }, + owner, + group, + e.Mode(), + false, + } +} + func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object, error) { - return nil, notSupported + if delimiter != "/" { + return nil, notSupported + } + var dir string = n.root + prefix + var objs []Object + if !strings.HasSuffix(dir, dirSuffix) { + dir = path.Dir(dir) + if !strings.HasSuffix(dir, dirSuffix) { + dir += dirSuffix + } + } else if marker == "" { + obj, err := n.Head(prefix) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + objs = append(objs, obj) + } + entries, err := n.readDirSorted(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + for _, e := range entries { + p := filepath.Join(dir, e.Name()) + if e.IsDir() { + p = filepath.ToSlash(p + "/") + } + if !strings.HasPrefix(p, n.root) { + continue + } + key := p[len(n.root):] + if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) { + continue + } + f := n.toFile(key, e) + objs = append(objs, f) + if len(objs) == int(limit) { + break + } + } + return objs, nil } func (n *nfsStore) ListAll(prefix, marker string) (<-chan Object, error) { From da3220133b170e387d89bf86beaa894924de8ee5 Mon Sep 17 00:00:00 2001 From: xixi Date: Mon, 12 Jun 2023 15:44:45 +0800 Subject: [PATCH 04/26] remove unused methods Signed-off-by: xixi --- pkg/object/nfs.go | 69 ----------------------------------------------- 1 file changed, 69 deletions(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index a36631cce245..b29f82c873b9 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -175,75 +175,6 @@ func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object { return ff } -func (n *nfsStore) sortByName(path string, fis []*nfs.EntryPlus) []Object { - var obs = make([]Object, 0, len(fis)) - for _, fi := range fis { - p := path + fi.Name() - if strings.HasPrefix(p, n.root) { - key := p[len(n.root):] - obs = append(obs, n.fileInfo(key, fi)) - } - } - sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() }) - return obs -} - -func (n *nfsStore) doFind(path, marker string, out chan Object) { - infos, err := n.target.ReadDirPlus(path) - if err != nil { - logger.Errorf("readdir %s: %s", path, err) - return - } - - obs := n.sortByName(path, infos) - for _, o := range obs { - key := o.Key() - if key > marker { - out <- o - } - if o.IsDir() && (key > marker || strings.HasPrefix(marker, key)) { - n.doFind(n.root+key, marker, out) - } - } -} - -func (n *nfsStore) find(path, marker string, out chan Object) { - if strings.HasSuffix(path, dirSuffix) { - fi, _, err := n.target.Lookup(path) - if err != nil { - logger.Errorf("Stat %s error: %q", path, err) - return - } - if marker == "" { - out <- n.fileInfo(path[len(n.root):], fi) - } - n.doFind(path, marker, out) - } else { - // As files or dirs in the same directory of file `path` resides - // may have prefix `path`, we should list the directory. - dir := filepath.Dir(path) + dirSuffix - infos, err := n.target.ReadDirPlus(dir) - if err != nil { - logger.Errorf("readdir %s: %s", dir, err) - return - } - - obs := n.sortByName(dir, infos) - for _, o := range obs { - key := o.Key() - p := n.root + o.Key() - if strings.HasPrefix(p, path) { - if key > marker || marker == "" { - out <- o - } - if o.IsDir() && (key > marker || strings.HasPrefix(marker, key)) { - n.doFind(p, marker, out) - } - } - } - } -} - func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { entries, err := n.target.ReadDirPlus(dirname) if err != nil { From 332b0a8cf716e6e0b3aa1deff70f0061d01cfaf7 Mon Sep 17 00:00:00 2001 From: xixi Date: Tue, 13 Jun 2023 16:09:12 +0800 Subject: [PATCH 05/26] fix nfs store Signed-off-by: xixi --- cmd/sync.go | 3 ++- pkg/object/nfs.go | 37 +++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 977178a45699..d681cfe771b0 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -304,7 +304,6 @@ func createSyncStorage(uri string, conf *sync.Config) (object.ObjectStorage, err secretKey, _ = user.Password() } name := strings.ToLower(u.Scheme) - var endpoint string if name == "file" { endpoint = u.Path @@ -315,6 +314,8 @@ func createSyncStorage(uri string, conf *sync.Config) (object.ObjectStorage, err if err != nil { return nil, fmt.Errorf("unescape %s: %s", u.Host, err) } + } else if name == "nfs" { + endpoint = u.Host + u.Path } else if !conf.NoHTTPS && supportHTTPS(name, u.Host) { endpoint = "https://" + u.Host } else { diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index b29f82c873b9..978ec45af8dc 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -59,11 +59,16 @@ func (n *nfsStore) String() string { } func (n *nfsStore) path(key string) string { - return n.root + key + root := strings.Trim(n.root, "/") + if strings.HasPrefix(key, root) { + return key[len(root):] + } + return key } func (n *nfsStore) Head(key string) (Object, error) { - fi, _, err := n.target.Lookup(n.path(key)) + p := n.path(key) + fi, _, err := n.target.Lookup(p) if err != nil { return nil, err } @@ -72,15 +77,15 @@ func (n *nfsStore) Head(key string) (Object, error) { func (n *nfsStore) Get(key string, off, limit int64) (io.ReadCloser, error) { p := n.path(key) - ff, err := n.target.Open(p) - if err != nil { - return nil, err - } if strings.HasSuffix(p, "/") { - _ = ff.Close() return io.NopCloser(bytes.NewBuffer([]byte{})), nil } + ff, err := n.target.Open(p) + if err != nil { + return nil, errors.Wrapf(err, "open %s", p) + } + if limit > 0 { return &SectionReaderCloser{ SectionReader: io.NewSectionReader(ff, off, limit), @@ -126,7 +131,7 @@ func (n *nfsStore) Put(key string, in io.Reader) error { _, err = n.target.Create(tmp, 0777) } if err != nil { - return err + return errors.Wrapf(err, "create %s", tmp) } ff, err := n.target.Open(tmp) if err != nil { @@ -218,7 +223,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object if delimiter != "/" { return nil, notSupported } - var dir string = n.root + prefix + dir := n.path(prefix) var objs []Object if !strings.HasSuffix(dir, dirSuffix) { dir = path.Dir(dir) @@ -247,14 +252,10 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object if e.IsDir() { p = filepath.ToSlash(p + "/") } - if !strings.HasPrefix(p, n.root) { - continue - } - key := p[len(n.root):] - if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) { + if !strings.HasPrefix(p, prefix) || (marker != "" && p <= marker) { continue } - f := n.toFile(key, e) + f := n.toFile(p, e) objs = append(objs, f) if len(objs) == int(limit) { break @@ -268,11 +269,7 @@ func (n *nfsStore) ListAll(prefix, marker string) (<-chan Object, error) { } func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { - if strings.Contains(addr, "@") { - ps := strings.Split(addr, "@") - username = ps[0] - addr = ps[1] - } else { + if username == "" { u, err := user.Current() if err != nil { return nil, fmt.Errorf("current user: %s", err) From d3615c7db09b6a77631e9471f59c805bd15c2df5 Mon Sep 17 00:00:00 2001 From: xixi Date: Wed, 14 Jun 2023 17:41:31 +0800 Subject: [PATCH 06/26] implement chtimes, chown and chmod Signed-off-by: xixi --- pkg/object/nfs.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index 978ec45af8dc..dbd0b48f2120 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -33,6 +33,7 @@ import ( "syscall" "time" + "github.com/juicedata/juicefs/pkg/utils" "github.com/pkg/errors" "github.com/vmware/go-nfs-client/nfs" "github.com/vmware/go-nfs-client/nfs/rpc" @@ -264,6 +265,58 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object return objs, nil } +func (n *nfsStore) setAttr(path string, attrSet func(attr *nfs.Fattr) nfs.Sattr3) error { + p := n.path(path) + fi, fh, err := n.target.Lookup(p) + if err != nil { + return err + } + fattr := fi.(*nfs.Fattr) + _, err = n.target.SetAttr(fh, attrSet(fattr)) + return err +} + +func (n *nfsStore) Chtimes(path string, mtime time.Time) error { + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + Mtime: nfs.SetTime{ + Time: nfs.NFS3Time{ + Seconds: uint32(mtime.Second()), + Nseconds: uint32(mtime.Nanosecond()), + }, + }, + } + }) +} + +func (n *nfsStore) Chmod(path string, mode os.FileMode) error { + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + Mode: nfs.SetMode{ + SetIt: true, + Mode: uint32(mode), + }, + } + }) +} + +func (n *nfsStore) Chown(path string, owner, group string) error { + uid := utils.LookupUser(owner) + gid := utils.LookupGroup(group) + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + UID: nfs.SetUID{ + SetIt: true, + UID: uint32(uid), + }, + GID: nfs.SetUID{ + SetIt: true, + UID: uint32(gid), + }, + } + }) +} + func (n *nfsStore) ListAll(prefix, marker string) (<-chan Object, error) { return nil, notSupported } From 2165e5f6f6c162931edf4e163df8931e4db67071 Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 15 Jun 2023 17:40:07 +0800 Subject: [PATCH 07/26] fix list Signed-off-by: xixi --- pkg/object/nfs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index dbd0b48f2120..fd7c3967c3bd 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -60,7 +60,7 @@ func (n *nfsStore) String() string { } func (n *nfsStore) path(key string) string { - root := strings.Trim(n.root, "/") + root := strings.TrimLeft(n.root, "/") if strings.HasPrefix(key, root) { return key[len(root):] } @@ -232,7 +232,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object dir += dirSuffix } } else if marker == "" { - obj, err := n.Head(prefix) + obj, err := n.Head(dir) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -249,7 +249,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object return nil, err } for _, e := range entries { - p := filepath.Join(dir, e.Name()) + p := filepath.Join(prefix, e.Name()) if e.IsDir() { p = filepath.ToSlash(p + "/") } From 5ed16ece949eafaf1b6c97c9895c71a40ab0cb2f Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 15 Jun 2023 22:05:16 +0800 Subject: [PATCH 08/26] fix set mtime Signed-off-by: xixi --- pkg/object/nfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index fd7c3967c3bd..f5ea66ca52aa 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -280,8 +280,9 @@ func (n *nfsStore) Chtimes(path string, mtime time.Time) error { return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { return nfs.Sattr3{ Mtime: nfs.SetTime{ + SetIt: nfs.SetToClientTime, Time: nfs.NFS3Time{ - Seconds: uint32(mtime.Second()), + Seconds: uint32(mtime.Unix()), Nseconds: uint32(mtime.Nanosecond()), }, }, From 111f57d4bcee2e0ba17859eae99d6bbe42024631 Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 15 Jun 2023 22:18:39 +0800 Subject: [PATCH 09/26] fix nfs uid and gid Signed-off-by: xixi --- pkg/object/file_unix.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/object/file_unix.go b/pkg/object/file_unix.go index b01a27b4fc5c..4d4f069c1a8a 100644 --- a/pkg/object/file_unix.go +++ b/pkg/object/file_unix.go @@ -25,6 +25,7 @@ import ( "github.com/juicedata/juicefs/pkg/utils" "github.com/pkg/sftp" + "github.com/vmware/go-nfs-client/nfs" ) func getOwnerGroup(info os.FileInfo) (string, string) { @@ -36,6 +37,9 @@ func getOwnerGroup(info os.FileInfo) (string, string) { case *sftp.FileStat: owner = utils.UserName(int(st.UID)) group = utils.GroupName(int(st.GID)) + case *nfs.Fattr: + owner = utils.UserName(int(st.UID)) + group = utils.GroupName(int(st.GID)) } return owner, group } From 390e86f13a3286f7f02f0d1c9660a4ee0980055b Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 15 Jun 2023 22:36:08 +0800 Subject: [PATCH 10/26] fix needCopyPerms Signed-off-by: xixi --- pkg/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index dcb90b38618a..df48b35e2118 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -207,7 +207,7 @@ func deleteObj(storage object.ObjectStorage, key string, dry bool) { func needCopyPerms(o1, o2 object.Object) bool { f1 := o1.(object.File) f2 := o2.(object.File) - return f2.Mode() != f1.Mode() || f2.Owner() != f1.Owner() || f2.Group() != f1.Group() + return f2.Mode()&^os.ModeType != f1.Mode()&^os.ModeType || f2.Owner() != f1.Owner() || f2.Group() != f1.Group() } func copyPerms(dst object.ObjectStorage, obj object.Object) { From dae60ef163de896b78b6a3d42daae42ac5bc20ff Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 15 Jun 2023 23:37:03 +0800 Subject: [PATCH 11/26] support symlink Signed-off-by: xixi --- go.mod | 3 ++- go.sum | 2 -- pkg/object/file.go | 6 ++--- pkg/object/nfs.go | 55 +++++++++++++++++++++++++--------------------- test/ccc/aaa | 0 test/ccc/bbb | 1 + test/ddd | 1 + 7 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 test/ccc/aaa create mode 120000 test/ccc/bbb create mode 120000 test/ddd diff --git a/go.mod b/go.mod index fd4a2feac9df..2bf0749b2a3f 100644 --- a/go.mod +++ b/go.mod @@ -263,4 +263,5 @@ replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => g replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 -replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c +// replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => ../go-nfs-client diff --git a/go.sum b/go.sum index 42ffba405f46..d84600ef94bf 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,6 @@ github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnT github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5u5iRL/e1yr/zXzplhdlaIGwKcOmGRSOfasQ= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= -github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c h1:mHLyhF5JfwCZxfOz0y90xWBi5UvHwqOVKIbNWw7iAUo= -github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= diff --git a/pkg/object/file.go b/pkg/object/file.go index 8a605035f137..a1a143285fcb 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -81,10 +81,10 @@ func (d *filestore) Head(key string) (Object, error) { if err != nil { return nil, err } - return d.toFile(key, fi, false), nil + return toFile(key, fi, false), nil } -func (d *filestore) toFile(key string, fi fs.FileInfo, isSymlink bool) *file { +func toFile(key string, fi fs.FileInfo, isSymlink bool) *file { size := fi.Size() if fi.IsDir() { size = 0 @@ -301,7 +301,7 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64) ([]Objec logger.Warnf("stat %s: %s", p, err) continue } - f := d.toFile(key, info, e.isSymlink) + f := toFile(key, info, e.isSymlink) objs = append(objs, f) if len(objs) == int(limit) { break diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index f5ea66ca52aa..4b34f213f65b 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -52,7 +52,9 @@ type nfsStore struct { type nfsEntry struct { *nfs.EntryPlus - name string + name string + fi os.FileInfo + isSymlink bool } func (n *nfsStore) String() string { @@ -190,36 +192,27 @@ func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { for i, e := range entries { if e.IsDir() { - nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix} + nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix, nil, false} + } else if e.Attr.Attr.Type == nfs.NF3Lnk { + // follow symlink + fi, _, err := n.target.Lookup(filepath.Join(dirname, e.Name())) + if err != nil { + nfsEntries[i] = &nfsEntry{e, e.Name(), nil, true} + continue + } + name := e.Name() + if fi.IsDir() { + name = e.Name() + dirSuffix + } + nfsEntries[i] = &nfsEntry{e, name, fi, true} } else { - nfsEntries[i] = &nfsEntry{e, e.Name()} + nfsEntries[i] = &nfsEntry{e, e.Name(), nil, false} } } sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() }) return nfsEntries, err } -func (d *nfsStore) toFile(key string, e *nfsEntry) *file { - size := e.Size() - if e.IsDir() { - size = 0 - } - owner, group := getOwnerGroup(e) - return &file{ - obj{ - key, - size, - e.ModTime(), - e.IsDir(), - "", - }, - owner, - group, - e.Mode(), - false, - } -} - func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object, error) { if delimiter != "/" { return nil, notSupported @@ -256,7 +249,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object if !strings.HasPrefix(p, prefix) || (marker != "" && p <= marker) { continue } - f := n.toFile(p, e) + f := toFile(p, e, e.isSymlink) objs = append(objs, f) if len(objs) == int(limit) { break @@ -318,6 +311,18 @@ func (n *nfsStore) Chown(path string, owner, group string) error { }) } +func (n *nfsStore) Symlink(oldName, newName string) error { + return n.target.Symlink(n.path(oldName), n.path(newName)) +} + +func (n *nfsStore) Readlink(name string) (string, error) { + f, err := n.target.Open(n.path(name)) + if err != nil { + return "", errors.Wrapf(err, "open %s", name) + } + return f.Readlink() +} + func (n *nfsStore) ListAll(prefix, marker string) (<-chan Object, error) { return nil, notSupported } diff --git a/test/ccc/aaa b/test/ccc/aaa new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/ccc/bbb b/test/ccc/bbb new file mode 120000 index 000000000000..e20608276303 --- /dev/null +++ b/test/ccc/bbb @@ -0,0 +1 @@ +/home/hexi/src/juicefs/test/ccc/aaa \ No newline at end of file diff --git a/test/ddd b/test/ddd new file mode 120000 index 000000000000..1d8159eaf88a --- /dev/null +++ b/test/ddd @@ -0,0 +1 @@ +/home/hexi/src/juicefs/test/ccc \ No newline at end of file From 93cc037dd1074eb06a76e5458c698a3f66c8a056 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 11:19:42 +0800 Subject: [PATCH 12/26] fix sort Signed-off-by: xixi --- pkg/object/nfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index 4b34f213f65b..d2d3bc685e36 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -209,7 +209,7 @@ func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { nfsEntries[i] = &nfsEntry{e, e.Name(), nil, false} } } - sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() }) + sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].name < nfsEntries[j].name }) return nfsEntries, err } From 860e91bbdc30f7ea12074298a32e4869e4fa653f Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 11:20:57 +0800 Subject: [PATCH 13/26] remove test file Signed-off-by: xixi --- test/ccc/aaa | 0 test/ccc/bbb | 1 - test/ddd | 1 - 3 files changed, 2 deletions(-) delete mode 100644 test/ccc/aaa delete mode 120000 test/ccc/bbb delete mode 120000 test/ddd diff --git a/test/ccc/aaa b/test/ccc/aaa deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/ccc/bbb b/test/ccc/bbb deleted file mode 120000 index e20608276303..000000000000 --- a/test/ccc/bbb +++ /dev/null @@ -1 +0,0 @@ -/home/hexi/src/juicefs/test/ccc/aaa \ No newline at end of file diff --git a/test/ddd b/test/ddd deleted file mode 120000 index 1d8159eaf88a..000000000000 --- a/test/ddd +++ /dev/null @@ -1 +0,0 @@ -/home/hexi/src/juicefs/test/ccc \ No newline at end of file From 95ef6d4056a3066e50faa76e7caea4967969fe3a Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 11:55:29 +0800 Subject: [PATCH 14/26] mkdir when parent of symlink not exist Signed-off-by: xixi --- pkg/object/nfs.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index d2d3bc685e36..c9bd7eebe078 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -312,6 +312,14 @@ func (n *nfsStore) Chown(path string, owner, group string) error { } func (n *nfsStore) Symlink(oldName, newName string) error { + p := n.path(newName) + if _, _, err := n.target.Lookup(filepath.Dir(p)); err != nil && os.IsNotExist(err) { + if _, err := n.target.Mkdir(filepath.Dir(p), os.FileMode(0777)); err != nil { + return err + } + } else if err != nil && !os.IsNotExist(err) { + return err + } return n.target.Symlink(n.path(oldName), n.path(newName)) } From 60e8dd61344c39d8cdaf561d0bc12ddf283f9398 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 15:08:49 +0800 Subject: [PATCH 15/26] fix List symlink in symlink Signed-off-by: xixi --- pkg/object/nfs.go | 14 ++++++++++++++ pkg/object/object_storage.go | 5 ++--- pkg/object/sharding.go | 2 +- pkg/sync/sync.go | 8 ++++++-- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index c9bd7eebe078..74e89604d428 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -57,6 +57,20 @@ type nfsEntry struct { isSymlink bool } +func (e *nfsEntry) Info() (os.FileInfo, error) { + if e.fi != nil { + return e.fi, nil + } + return e.EntryPlus, nil +} + +func (e *nfsEntry) IsDir() bool { + if e.fi != nil { + return e.fi.IsDir() + } + return e.EntryPlus.IsDir() +} + func (n *nfsStore) String() string { return fmt.Sprintf("nfs://%s@%s:%s", n.username, n.host, n.root) } diff --git a/pkg/object/object_storage.go b/pkg/object/object_storage.go index df70ef2b503d..9e07dc67962a 100644 --- a/pkg/object/object_storage.go +++ b/pkg/object/object_storage.go @@ -189,7 +189,7 @@ type listThread struct { entries []Object } -func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string) (<-chan Object, error) { +func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, followLink bool) (<-chan Object, error) { entries, err := store.List(prefix, "", "/", 1e9) if err != nil { logger.Errorf("list %s: %s", prefix, err) @@ -214,10 +214,9 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string) (<-cha if key < start && !strings.HasPrefix(start, key) { continue } - if !entries[i].IsDir() || key == prefix { + if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { continue } - t.entries, t.err = store.List(key, "\x00", "/", 1e9) // exclude itself t.Lock() t.ready = true diff --git a/pkg/object/sharding.go b/pkg/object/sharding.go index 61cebac54512..ba8847fe7bbb 100644 --- a/pkg/object/sharding.go +++ b/pkg/object/sharding.go @@ -100,7 +100,7 @@ func ListAll(store ObjectStorage, prefix, marker string) (<-chan Object, error) logger.Debugf("Listing objects from %s marker %q", store, marker) objs, err := store.List(prefix, marker, "", maxResults) if err == notSupported { - return ListAllWithDelimiter(store, prefix, marker, "") + return ListAllWithDelimiter(store, prefix, marker, "", true) } if err != nil { logger.Errorf("Can't list %s: %s", store, err.Error()) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index df48b35e2118..ed45e05e5234 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -73,9 +73,13 @@ func formatSize(bytes int64) string { } // ListAll on all the keys that starts at marker from object storage. -func ListAll(store object.ObjectStorage, prefix, start, end string) (<-chan object.Object, error) { +func ListAll(store object.ObjectStorage, prefix, start, end string, noFollow ...bool) (<-chan object.Object, error) { startTime := time.Now() logger.Debugf("Iterating objects from %s with prefix %s start %q", store, prefix, start) + follow := true + if len(noFollow) > 0 && noFollow[0] { + follow = false + } out := make(chan object.Object, maxResults*10) @@ -107,7 +111,7 @@ func ListAll(store object.ObjectStorage, prefix, start, end string) (<-chan obje logger.Debugf("Listing objects from %s marker %q", store, marker) objs, err := store.List(prefix, marker, "", maxResults) if err == utils.ENOTSUP { - return object.ListAllWithDelimiter(store, prefix, start, end) + return object.ListAllWithDelimiter(store, prefix, start, end, follow) } if err != nil { logger.Errorf("Can't list %s: %s", store, err.Error()) From 309455b82209c3e5af684e653168008a30f5ca77 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 15:24:41 +0800 Subject: [PATCH 16/26] fix ListAllWithDelimiter Signed-off-by: xixi --- pkg/object/object_storage.go | 3 ++- pkg/sync/sync.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/object/object_storage.go b/pkg/object/object_storage.go index 9e07dc67962a..f375ab4833d1 100644 --- a/pkg/object/object_storage.go +++ b/pkg/object/object_storage.go @@ -217,6 +217,7 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, follow if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { continue } + fmt.Printf("list %s, is dir: %t, followLink: %t, is symlink: %t\n", key, entries[i].IsDir(), followLink, entries[i].IsSymlink()) t.entries, t.err = store.List(key, "\x00", "/", 1e9) // exclude itself t.Lock() t.ready = true @@ -243,7 +244,7 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, follow } else if !strings.HasPrefix(start, key) { continue } - if !e.IsDir() || key == prefix { + if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { continue } diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index ed45e05e5234..e821b747c6c1 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -645,12 +645,12 @@ func startSingleProducer(tasks chan<- object.Object, src, dst object.ObjectStora start, end := config.Start, config.End logger.Debugf("maxResults: %d, defaultPartSize: %d, maxBlock: %d", maxResults, defaultPartSize, maxBlock) - srckeys, err := ListAll(src, prefix, start, end) + srckeys, err := ListAll(src, prefix, start, end, config.Links) if err != nil { return fmt.Errorf("list %s: %s", src, err) } - dstkeys, err := ListAll(dst, prefix, start, end) + dstkeys, err := ListAll(dst, prefix, start, end, config.Links) if err != nil { return fmt.Errorf("list %s: %s", dst, err) } From e6b079c630f720d98669e3c381d5bf7d5c4b2592 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 17:09:55 +0800 Subject: [PATCH 17/26] fix symlink Signed-off-by: xixi --- pkg/object/nfs.go | 16 +++++++++++----- pkg/object/object_storage.go | 1 - test/ccc/aaa | 0 test/ccc/bbb | 1 + test/ddd | 1 + 5 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 test/ccc/aaa create mode 120000 test/ccc/bbb create mode 120000 test/ddd diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index 74e89604d428..b9282d523d77 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -57,6 +57,10 @@ type nfsEntry struct { isSymlink bool } +func (e *nfsEntry) Name() string { + return e.name +} + func (e *nfsEntry) Info() (os.FileInfo, error) { if e.fi != nil { return e.fi, nil @@ -223,7 +227,7 @@ func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { nfsEntries[i] = &nfsEntry{e, e.Name(), nil, false} } } - sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].name < nfsEntries[j].name }) + sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() }) return nfsEntries, err } @@ -257,7 +261,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64) ([]Object } for _, e := range entries { p := filepath.Join(prefix, e.Name()) - if e.IsDir() { + if e.IsDir() && !e.isSymlink { p = filepath.ToSlash(p + "/") } if !strings.HasPrefix(p, prefix) || (marker != "" && p <= marker) { @@ -326,10 +330,12 @@ func (n *nfsStore) Chown(path string, owner, group string) error { } func (n *nfsStore) Symlink(oldName, newName string) error { + newName = strings.TrimRight(newName, "/") p := n.path(newName) - if _, _, err := n.target.Lookup(filepath.Dir(p)); err != nil && os.IsNotExist(err) { - if _, err := n.target.Mkdir(filepath.Dir(p), os.FileMode(0777)); err != nil { - return err + dir := filepath.Dir(p) + if _, _, err := n.target.Lookup(dir); err != nil && os.IsNotExist(err) { + if _, err := n.target.Mkdir(dir, os.FileMode(0777)); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "mkdir %s", dir) } } else if err != nil && !os.IsNotExist(err) { return err diff --git a/pkg/object/object_storage.go b/pkg/object/object_storage.go index f375ab4833d1..a4dae48f5247 100644 --- a/pkg/object/object_storage.go +++ b/pkg/object/object_storage.go @@ -217,7 +217,6 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, follow if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { continue } - fmt.Printf("list %s, is dir: %t, followLink: %t, is symlink: %t\n", key, entries[i].IsDir(), followLink, entries[i].IsSymlink()) t.entries, t.err = store.List(key, "\x00", "/", 1e9) // exclude itself t.Lock() t.ready = true diff --git a/test/ccc/aaa b/test/ccc/aaa new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/ccc/bbb b/test/ccc/bbb new file mode 120000 index 000000000000..e20608276303 --- /dev/null +++ b/test/ccc/bbb @@ -0,0 +1 @@ +/home/hexi/src/juicefs/test/ccc/aaa \ No newline at end of file diff --git a/test/ddd b/test/ddd new file mode 120000 index 000000000000..1d8159eaf88a --- /dev/null +++ b/test/ddd @@ -0,0 +1 @@ +/home/hexi/src/juicefs/test/ccc \ No newline at end of file From 77afe7a7d4af1a2de016021133453ba9aad56ea2 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 17:11:59 +0800 Subject: [PATCH 18/26] remove test dir Signed-off-by: xixi --- test/ccc/aaa | 0 test/ccc/bbb | 1 - test/ddd | 1 - 3 files changed, 2 deletions(-) delete mode 100644 test/ccc/aaa delete mode 120000 test/ccc/bbb delete mode 120000 test/ddd diff --git a/test/ccc/aaa b/test/ccc/aaa deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/ccc/bbb b/test/ccc/bbb deleted file mode 120000 index e20608276303..000000000000 --- a/test/ccc/bbb +++ /dev/null @@ -1 +0,0 @@ -/home/hexi/src/juicefs/test/ccc/aaa \ No newline at end of file diff --git a/test/ddd b/test/ddd deleted file mode 120000 index 1d8159eaf88a..000000000000 --- a/test/ddd +++ /dev/null @@ -1 +0,0 @@ -/home/hexi/src/juicefs/test/ccc \ No newline at end of file From 23d3f790591f14e5310a5293e19cb3281b0120da Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 17:29:23 +0800 Subject: [PATCH 19/26] fix symlink chtimes Signed-off-by: xixi --- pkg/sync/sync.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index e821b747c6c1..91c6b071d031 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -560,11 +560,7 @@ func worker(tasks <-chan object.Object, src, dst object.ObjectStorage, config *C } var err error if config.Links && obj.IsSymlink() { - if err = copyLink(src, dst, key); err == nil { - copied.Increment() - break - } - logger.Errorf("copy link failed: %s", err) + err = copyLink(src, dst, key) } else { err = copyData(src, dst, key, obj.Size()) } From 2146e55af50661a27d4a543f9b950c10177bf365 Mon Sep 17 00:00:00 2001 From: xixi Date: Fri, 16 Jun 2023 17:36:45 +0800 Subject: [PATCH 20/26] check url Signed-off-by: xixi --- pkg/object/nfs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index b9282d523d77..50cc0e0ba7bb 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -364,6 +364,9 @@ func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { username = u.Username } b := strings.Split(addr, ":") + if len(b) != 2 { + return nil, fmt.Errorf("invalid NFS address %s", addr) + } host := b[0] path := b[1] mount, err := nfs.DialMount(host, time.Second*3) From 9afd90f3328f2934eceb8ff99d66f0f983b705fc Mon Sep 17 00:00:00 2001 From: xixi Date: Mon, 19 Jun 2023 10:27:12 +0800 Subject: [PATCH 21/26] fix building Signed-off-by: xixi --- go.mod | 3 +-- go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2bf0749b2a3f..d79cfdc6d896 100644 --- a/go.mod +++ b/go.mod @@ -263,5 +263,4 @@ replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => g replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 -// replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230609065724-d6c1e68f478c -replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => ../go-nfs-client +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d diff --git a/go.sum b/go.sum index d84600ef94bf..e92b6992961b 100644 --- a/go.sum +++ b/go.sum @@ -625,6 +625,8 @@ github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnT github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5u5iRL/e1yr/zXzplhdlaIGwKcOmGRSOfasQ= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= +github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d h1:YzaOMLugLWVNFk16CyhTfrRJytKFqMdvXTPlOwZpz5E= +github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= From b49a42e3c29980b02d907db80348e80cf6679aeb Mon Sep 17 00:00:00 2001 From: xixi Date: Mon, 19 Jun 2023 15:31:20 +0800 Subject: [PATCH 22/26] update nfs client Signed-off-by: xixi --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d79cfdc6d896..8933f6e40c68 100644 --- a/go.mod +++ b/go.mod @@ -263,4 +263,4 @@ replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => g replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 -replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b diff --git a/go.sum b/go.sum index e92b6992961b..b8ad9480c535 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,8 @@ github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnT github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5u5iRL/e1yr/zXzplhdlaIGwKcOmGRSOfasQ= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= -github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d h1:YzaOMLugLWVNFk16CyhTfrRJytKFqMdvXTPlOwZpz5E= -github.com/juicedata/go-nfs-client v0.0.0-20230619021701-63e139054b5d/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= +github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b h1:tZixumON7boPNeTPBkaD5ObOUJ8DaES4VL17XMrY0BU= +github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= From f42ba41adab2c9daa408fbcd7297edf7807462d6 Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 21 Sep 2023 11:50:13 +0800 Subject: [PATCH 23/26] nfs support followLink Signed-off-by: xixi --- pkg/object/nfs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index 4528d101d5b9..d26db85af8f0 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -201,7 +201,7 @@ func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object { return ff } -func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { +func (n *nfsStore) readDirSorted(dirname string, followLink bool) ([]*nfsEntry, error) { entries, err := n.target.ReadDirPlus(dirname) if err != nil { return nil, errors.Wrapf(err, "readdir %s", dirname) @@ -211,7 +211,7 @@ func (n *nfsStore) readDirSorted(dirname string) ([]*nfsEntry, error) { for i, e := range entries { if e.IsDir() { nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix, nil, false} - } else if e.Attr.Attr.Type == nfs.NF3Lnk { + } else if e.Attr.Attr.Type == nfs.NF3Lnk && followLink { // follow symlink fi, _, err := n.target.Lookup(filepath.Join(dirname, e.Name())) if err != nil { @@ -252,7 +252,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLin } objs = append(objs, obj) } - entries, err := n.readDirSorted(dir) + entries, err := n.readDirSorted(dir, followLink) if err != nil { if os.IsNotExist(err) { return nil, nil From 261aa5ac9d6ebc589d5b4bf8fcdd46672e7462d3 Mon Sep 17 00:00:00 2001 From: xixi Date: Thu, 21 Sep 2023 11:52:55 +0800 Subject: [PATCH 24/26] fix building with nonfs Signed-off-by: xixi --- pkg/object/file_unix.go | 4 ---- pkg/object/nfs.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/object/file_unix.go b/pkg/object/file_unix.go index 4d4f069c1a8a..b01a27b4fc5c 100644 --- a/pkg/object/file_unix.go +++ b/pkg/object/file_unix.go @@ -25,7 +25,6 @@ import ( "github.com/juicedata/juicefs/pkg/utils" "github.com/pkg/sftp" - "github.com/vmware/go-nfs-client/nfs" ) func getOwnerGroup(info os.FileInfo) (string, string) { @@ -37,9 +36,6 @@ func getOwnerGroup(info os.FileInfo) (string, string) { case *sftp.FileStat: owner = utils.UserName(int(st.UID)) group = utils.GroupName(int(st.GID)) - case *nfs.Fattr: - owner = utils.UserName(int(st.UID)) - group = utils.GroupName(int(st.GID)) } return owner, group } diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index d26db85af8f0..898068d32996 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -183,7 +183,7 @@ func (n *nfsStore) Delete(key string) error { } func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object { - owner, group := getOwnerGroup(fi) + owner, group := n.getOwnerGroup(fi) isSymlink := !fi.Mode().IsDir() && !fi.Mode().IsRegular() ff := &file{ obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""}, @@ -355,6 +355,16 @@ func (n *nfsStore) ListAll(prefix, marker string, followLink bool) (<-chan Objec return nil, notSupported } +func (n *nfsStore) getOwnerGroup(info os.FileInfo) (string, string) { + var owner, group string + switch st := info.Sys().(type) { + case *nfs.Fattr: + owner = utils.UserName(int(st.UID)) + group = utils.GroupName(int(st.GID)) + } + return owner, group +} + func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { if username == "" { u, err := user.Current() @@ -379,7 +389,6 @@ func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { return nil, fmt.Errorf("unable to mount %s: %v", addr, err) } return &nfsStore{DefaultObjectStorage{}, username, host, path, target}, nil - } func init() { From e6eda47e19ae75007ea6e0dc17d29b16e4a49491 Mon Sep 17 00:00:00 2001 From: xixi Date: Sat, 7 Oct 2023 12:16:03 +0800 Subject: [PATCH 25/26] skip check sum for symlink Signed-off-by: xixi --- pkg/sync/sync.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index 478faa684233..a8caaf9ffd15 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -554,13 +554,15 @@ func worker(tasks <-chan object.Object, src, dst object.ObjectStorage, config *C break } var err error + var dataCopied bool if config.Links && obj.IsSymlink() { err = copyLink(src, dst, key) } else { + dataCopied = true err = copyData(src, dst, key, obj.Size()) } - if err == nil && (config.CheckAll || config.CheckNew) { + if err == nil && (config.CheckAll || config.CheckNew) && dataCopied { var equal bool if equal, err = checkSum(src, dst, key, obj.Size()); err == nil && !equal { err = fmt.Errorf("checksums of copied object %s don't match", key) From 538505bfa5ed911575577a2e03a358a4a48d9ad4 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Sat, 7 Oct 2023 14:12:27 +0800 Subject: [PATCH 26/26] rollback some unreleted changes --- Makefile | 2 +- cmd/sync.go | 1 + pkg/object/object_storage.go | 4 ++-- pkg/sync/sync.go | 13 ++++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index d5f7ea2c3a97..ef0b693610ee 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ juicefs: Makefile cmd/*.go pkg/*/*.go go.* go build -ldflags="$(LDFLAGS)" -o juicefs . juicefs.lite: Makefile cmd/*.go pkg/*/*.go - go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,noscs,nosftp,noswift,noupyun,noazure,nogs,noufile,nob2,nosqlite,nomysql,nopg,notikv,nobadger,noetcd \ + go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,noscs,nosftp,noswift,noupyun,noazure,nogs,noufile,nob2,nonfs,nosqlite,nomysql,nopg,notikv,nobadger,noetcd \ -ldflags="$(LDFLAGS)" -o juicefs.lite . juicefs.ceph: Makefile cmd/*.go pkg/*/*.go diff --git a/cmd/sync.go b/cmd/sync.go index cc34a8972c2b..96ccfd880b7b 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -309,6 +309,7 @@ func createSyncStorage(uri string, conf *sync.Config) (object.ObjectStorage, err secretKey, _ = user.Password() } name := strings.ToLower(u.Scheme) + var endpoint string if name == "file" { endpoint = u.Path diff --git a/pkg/object/object_storage.go b/pkg/object/object_storage.go index 6d3824b445c9..a45f58693d84 100644 --- a/pkg/object/object_storage.go +++ b/pkg/object/object_storage.go @@ -214,7 +214,7 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, follow if key < start && !strings.HasPrefix(start, key) { continue } - if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { + if !entries[i].IsDir() || key == prefix { continue } @@ -244,7 +244,7 @@ func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, follow } else if !strings.HasPrefix(start, key) { continue } - if !entries[i].IsDir() || key == prefix || !followLink && entries[i].IsSymlink() { + if !e.IsDir() || key == prefix { continue } diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index a8caaf9ffd15..768f6b377e75 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -76,6 +76,7 @@ func formatSize(bytes int64) string { func ListAll(store object.ObjectStorage, prefix, start, end string, followLink bool) (<-chan object.Object, error) { startTime := time.Now() logger.Debugf("Iterating objects from %s with prefix %s start %q", store, prefix, start) + out := make(chan object.Object, maxResults*10) // As the result of object storage's List method doesn't include the marker key, @@ -206,7 +207,7 @@ func deleteObj(storage object.ObjectStorage, key string, dry bool) { func needCopyPerms(o1, o2 object.Object) bool { f1 := o1.(object.File) f2 := o2.(object.File) - return f2.Mode()&^os.ModeType != f1.Mode()&^os.ModeType || f2.Owner() != f1.Owner() || f2.Group() != f1.Group() + return f2.Mode() != f1.Mode() || f2.Owner() != f1.Owner() || f2.Group() != f1.Group() } func copyPerms(dst object.ObjectStorage, obj object.Object) { @@ -554,15 +555,17 @@ func worker(tasks <-chan object.Object, src, dst object.ObjectStorage, config *C break } var err error - var dataCopied bool if config.Links && obj.IsSymlink() { - err = copyLink(src, dst, key) + if err = copyLink(src, dst, key); err == nil { + copied.Increment() + break + } + logger.Errorf("copy link failed: %s", err) } else { - dataCopied = true err = copyData(src, dst, key, obj.Size()) } - if err == nil && (config.CheckAll || config.CheckNew) && dataCopied { + if err == nil && (config.CheckAll || config.CheckNew) { var equal bool if equal, err = checkSum(src, dst, key, obj.Size()); err == nil && !equal { err = fmt.Errorf("checksums of copied object %s don't match", key)