diff --git a/Makefile b/Makefile index d5f7ea2c3a97..ef0b693610ee 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ juicefs: Makefile cmd/*.go pkg/*/*.go go.* go build -ldflags="$(LDFLAGS)" -o juicefs . juicefs.lite: Makefile cmd/*.go pkg/*/*.go - go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,noscs,nosftp,noswift,noupyun,noazure,nogs,noufile,nob2,nosqlite,nomysql,nopg,notikv,nobadger,noetcd \ + go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,noscs,nosftp,noswift,noupyun,noazure,nogs,noufile,nob2,nonfs,nosqlite,nomysql,nopg,notikv,nobadger,noetcd \ -ldflags="$(LDFLAGS)" -o juicefs.lite . juicefs.ceph: Makefile cmd/*.go pkg/*/*.go diff --git a/cmd/sync.go b/cmd/sync.go index b7f60f3b2055..96ccfd880b7b 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -323,6 +323,8 @@ func createSyncStorage(uri string, conf *sync.Config) (object.ObjectStorage, err if os.Getenv(endpoint) != "" { conf.Env[endpoint] = os.Getenv(endpoint) } + } else if name == "nfs" { + endpoint = u.Host + u.Path } else if !conf.NoHTTPS && supportHTTPS(name, u.Host) { endpoint = "https://" + u.Host } else { diff --git a/go.mod b/go.mod index 83e3008f444d..370eff6ced57 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/urfave/cli/v2 v2.19.3 github.com/vbauerster/mpb/v7 v7.0.3 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 + github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.3 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.etcd.io/etcd v3.3.27+incompatible @@ -80,6 +81,11 @@ require ( xorm.io/xorm v1.0.7 ) +require ( + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect +) + require ( cloud.google.com/go v0.102.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect @@ -258,3 +264,5 @@ replace xorm.io/xorm v1.0.7 => gitea.com/davies/xorm v1.0.8-0.20220528043536-552 replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 + +replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b diff --git a/go.sum b/go.sum index 92c2b302100a..5ed9e0bb5c0b 100644 --- a/go.sum +++ b/go.sum @@ -622,6 +622,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnTaUqfRYjlwcQlr53aasmkvHEbYLXthqr8= github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b h1:tZixumON7boPNeTPBkaD5ObOUJ8DaES4VL17XMrY0BU= +github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7 h1:4evzoVz1/AZfk9tqxWdzVYTMl2dC7VjEJHfaSFDrKS8= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= @@ -898,6 +900,8 @@ github.com/qiniu/go-sdk/v7 v7.15.0 h1:vkxZZHM2Ed0qHeIx7NF3unXav+guaVIXlEsCCkpQAw github.com/qiniu/go-sdk/v7 v7.15.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= diff --git a/pkg/object/file.go b/pkg/object/file.go index 0fee777ecdc1..cf2e41bcd070 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -81,10 +81,10 @@ func (d *filestore) Head(key string) (Object, error) { if err != nil { return nil, err } - return d.toFile(key, fi, false), nil + return toFile(key, fi, false), nil } -func (d *filestore) toFile(key string, fi fs.FileInfo, isSymlink bool) *file { +func toFile(key string, fi fs.FileInfo, isSymlink bool) *file { size := fi.Size() if fi.IsDir() { size = 0 @@ -299,7 +299,7 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi continue } info := e.Info() - f := d.toFile(key, info, e.isSymlink) + f := toFile(key, info, e.isSymlink) objs = append(objs, f) if len(objs) == int(limit) { break diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go new file mode 100644 index 000000000000..898068d32996 --- /dev/null +++ b/pkg/object/nfs.go @@ -0,0 +1,396 @@ +//go:build !nonfs +// +build !nonfs + +/* + * JuiceFS, Copyright 2023 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "os/user" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/juicedata/juicefs/pkg/utils" + "github.com/pkg/errors" + "github.com/vmware/go-nfs-client/nfs" + "github.com/vmware/go-nfs-client/nfs/rpc" +) + +var _ ObjectStorage = &nfsStore{} + +type nfsStore struct { + DefaultObjectStorage + username string + host string + root string + + target *nfs.Target +} + +type nfsEntry struct { + *nfs.EntryPlus + name string + fi os.FileInfo + isSymlink bool +} + +func (e *nfsEntry) Name() string { + return e.name +} + +func (e *nfsEntry) Info() (os.FileInfo, error) { + if e.fi != nil { + return e.fi, nil + } + return e.EntryPlus, nil +} + +func (e *nfsEntry) IsDir() bool { + if e.fi != nil { + return e.fi.IsDir() + } + return e.EntryPlus.IsDir() +} + +func (n *nfsStore) String() string { + return fmt.Sprintf("nfs://%s@%s:%s", n.username, n.host, n.root) +} + +func (n *nfsStore) path(key string) string { + root := strings.TrimLeft(n.root, "/") + if strings.HasPrefix(key, root) { + return key[len(root):] + } + return key +} + +func (n *nfsStore) Head(key string) (Object, error) { + p := n.path(key) + fi, _, err := n.target.Lookup(p) + if err != nil { + return nil, err + } + return n.fileInfo(key, fi), nil +} + +func (n *nfsStore) Get(key string, off, limit int64) (io.ReadCloser, error) { + p := n.path(key) + if strings.HasSuffix(p, "/") { + return io.NopCloser(bytes.NewBuffer([]byte{})), nil + } + + ff, err := n.target.Open(p) + if err != nil { + return nil, errors.Wrapf(err, "open %s", p) + } + + if limit > 0 { + return &SectionReaderCloser{ + SectionReader: io.NewSectionReader(ff, off, limit), + Closer: ff, + }, nil + } + return ff, err +} + +func (n *nfsStore) mkdirAll(path string, perm fs.FileMode) error { + path = strings.TrimSuffix(path, "/") + fi, _, err := n.target.Lookup(path) + if err == nil { + if fi.IsDir() { + logger.Tracef("nfs mkdir: path %s already exists", path) + return nil + } else { + return syscall.ENOTDIR + } + } else if !os.IsNotExist(err) { + return err + } + + dir, _ := filepath.Split(path) + if dir != "." { + if err = n.mkdirAll(dir, perm); err != nil { + return err + } + } + _, err = n.target.Mkdir(path, perm) + return err +} + +func (n *nfsStore) Put(key string, in io.Reader) error { + p := n.path(key) + if strings.HasSuffix(p, dirSuffix) { + return n.mkdirAll(p, 0777) + } + tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp") + _, err := n.target.Create(tmp, 0777) + if os.IsNotExist(err) { + _ = n.mkdirAll(filepath.Dir(p), 0777) + _, err = n.target.Create(tmp, 0777) + } + if err != nil { + return errors.Wrapf(err, "create %s", tmp) + } + ff, err := n.target.Open(tmp) + if err != nil { + return err + } + defer func() { _ = n.target.Remove(tmp) }() + buf := bufPool.Get().(*[]byte) + defer bufPool.Put(buf) + _, err = io.CopyBuffer(ff, in, *buf) + if err != nil { + _ = ff.Close() + return err + } + err = ff.Close() + if err != nil { + return err + } + // _ = n.target.Remove(p) + return n.target.Rename(tmp, p) +} + +func (n *nfsStore) Delete(key string) error { + err := n.target.Remove(strings.TrimRight(n.path(key), dirSuffix)) + if err != nil && os.IsNotExist(err) { + err = nil + } + return err +} + +func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object { + owner, group := n.getOwnerGroup(fi) + isSymlink := !fi.Mode().IsDir() && !fi.Mode().IsRegular() + ff := &file{ + obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""}, + owner, + group, + fi.Mode(), + isSymlink, + } + if fi.IsDir() { + if key != "" && !strings.HasSuffix(key, "/") { + ff.key += "/" + } + ff.size = 0 + } + return ff +} + +func (n *nfsStore) readDirSorted(dirname string, followLink bool) ([]*nfsEntry, error) { + entries, err := n.target.ReadDirPlus(dirname) + if err != nil { + return nil, errors.Wrapf(err, "readdir %s", dirname) + } + nfsEntries := make([]*nfsEntry, len(entries)) + + for i, e := range entries { + if e.IsDir() { + nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix, nil, false} + } else if e.Attr.Attr.Type == nfs.NF3Lnk && followLink { + // follow symlink + fi, _, err := n.target.Lookup(filepath.Join(dirname, e.Name())) + if err != nil { + nfsEntries[i] = &nfsEntry{e, e.Name(), nil, true} + continue + } + name := e.Name() + if fi.IsDir() { + name = e.Name() + dirSuffix + } + nfsEntries[i] = &nfsEntry{e, name, fi, true} + } else { + nfsEntries[i] = &nfsEntry{e, e.Name(), nil, false} + } + } + sort.Slice(nfsEntries, func(i, j int) bool { return nfsEntries[i].Name() < nfsEntries[j].Name() }) + return nfsEntries, err +} + +func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) { + if delimiter != "/" { + return nil, notSupported + } + dir := n.path(prefix) + var objs []Object + if !strings.HasSuffix(dir, dirSuffix) { + dir = path.Dir(dir) + if !strings.HasSuffix(dir, dirSuffix) { + dir += dirSuffix + } + } else if marker == "" { + obj, err := n.Head(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + objs = append(objs, obj) + } + entries, err := n.readDirSorted(dir, followLink) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + for _, e := range entries { + p := filepath.Join(prefix, e.Name()) + if e.IsDir() && !e.isSymlink { + p = filepath.ToSlash(p + "/") + } + if !strings.HasPrefix(p, prefix) || (marker != "" && p <= marker) { + continue + } + f := toFile(p, e, e.isSymlink) + objs = append(objs, f) + if len(objs) == int(limit) { + break + } + } + return objs, nil +} + +func (n *nfsStore) setAttr(path string, attrSet func(attr *nfs.Fattr) nfs.Sattr3) error { + p := n.path(path) + fi, fh, err := n.target.Lookup(p) + if err != nil { + return err + } + fattr := fi.(*nfs.Fattr) + _, err = n.target.SetAttr(fh, attrSet(fattr)) + return err +} + +func (n *nfsStore) Chtimes(path string, mtime time.Time) error { + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + Mtime: nfs.SetTime{ + SetIt: nfs.SetToClientTime, + Time: nfs.NFS3Time{ + Seconds: uint32(mtime.Unix()), + Nseconds: uint32(mtime.Nanosecond()), + }, + }, + } + }) +} + +func (n *nfsStore) Chmod(path string, mode os.FileMode) error { + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + Mode: nfs.SetMode{ + SetIt: true, + Mode: uint32(mode), + }, + } + }) +} + +func (n *nfsStore) Chown(path string, owner, group string) error { + uid := utils.LookupUser(owner) + gid := utils.LookupGroup(group) + return n.setAttr(path, func(attr *nfs.Fattr) nfs.Sattr3 { + return nfs.Sattr3{ + UID: nfs.SetUID{ + SetIt: true, + UID: uint32(uid), + }, + GID: nfs.SetUID{ + SetIt: true, + UID: uint32(gid), + }, + } + }) +} + +func (n *nfsStore) Symlink(oldName, newName string) error { + newName = strings.TrimRight(newName, "/") + p := n.path(newName) + dir := filepath.Dir(p) + if _, _, err := n.target.Lookup(dir); err != nil && os.IsNotExist(err) { + if _, err := n.target.Mkdir(dir, os.FileMode(0777)); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "mkdir %s", dir) + } + } else if err != nil && !os.IsNotExist(err) { + return err + } + return n.target.Symlink(n.path(oldName), n.path(newName)) +} + +func (n *nfsStore) Readlink(name string) (string, error) { + f, err := n.target.Open(n.path(name)) + if err != nil { + return "", errors.Wrapf(err, "open %s", name) + } + return f.Readlink() +} + +func (n *nfsStore) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) { + return nil, notSupported +} + +func (n *nfsStore) getOwnerGroup(info os.FileInfo) (string, string) { + var owner, group string + switch st := info.Sys().(type) { + case *nfs.Fattr: + owner = utils.UserName(int(st.UID)) + group = utils.GroupName(int(st.GID)) + } + return owner, group +} + +func newNFSStore(addr, username, pass, token string) (ObjectStorage, error) { + if username == "" { + u, err := user.Current() + if err != nil { + return nil, fmt.Errorf("current user: %s", err) + } + username = u.Username + } + b := strings.Split(addr, ":") + if len(b) != 2 { + return nil, fmt.Errorf("invalid NFS address %s", addr) + } + host := b[0] + path := b[1] + mount, err := nfs.DialMount(host, time.Second*3) + if err != nil { + return nil, fmt.Errorf("unable to dial MOUNT service %s: %v", addr, err) + } + auth := rpc.NewAuthUnix(username, uint32(os.Getuid()), uint32(os.Getgid())) + target, err := mount.Mount(path, auth.Auth()) + if err != nil { + return nil, fmt.Errorf("unable to mount %s: %v", addr, err) + } + return &nfsStore{DefaultObjectStorage{}, username, host, path, target}, nil +} + +func init() { + Register("nfs", newNFSStore) +}