From 9d262da18e9c0db301440ba152b4d8d66bcb1a40 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 9 Jan 2024 19:47:37 +0100 Subject: [PATCH 01/28] Add Bunny.net object storage implementation --- go.mod | 10 +++- go.sum | 11 ++++ pkg/object/bunny.go | 137 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 pkg/object/bunny.go diff --git a/go.mod b/go.mod index 2f7457c909a5..d98ecd9ee53b 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/tencentyun/cos-go-sdk-v5 v0.7.45 github.com/tikv/client-go/v2 v2.0.4 github.com/upyun/go-sdk/v3 v3.0.4 - github.com/urfave/cli/v2 v2.19.3 + github.com/urfave/cli/v2 v2.25.7 github.com/vbauerster/mpb/v7 v7.0.3 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b @@ -84,10 +84,18 @@ require ( xorm.io/xorm v1.0.7 ) +require ( + git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7 // indirect + git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088 // indirect + git.sr.ht/~jamesponddotco/recache-go v1.0.1 // indirect + git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382 // indirect +) + require ( cloud.google.com/go v0.102.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect git.apache.org/thrift.git v0.13.0 // indirect + git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect diff --git a/go.sum b/go.sum index efe3d05b3102..4c1d8bf8b5be 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,16 @@ contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRq dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.13.0 h1:/3bz5WZ+sqYArk7MBBBbDufMxKKOA56/6JO6psDpUDY= git.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0 h1:AwBxhQITDhJaJnEzbvgqwyMiscG8fo4PRFJvzz/ZnQg= +git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0/go.mod h1:x5kqzC+x93KaVYwBThSuNVFdHmErySaeUd2AwYNMo/0= +git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7 h1:uY+pFjSyrV1q54kai43xcu+HnHGw3AfG14TpYEIuURY= +git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7/go.mod h1:5b9IHkokuXKVcAG5bmf0ej26NVMpV1usjIwrwOyR2SE= +git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088 h1:LKK7NuFKNBduMQkhg/Q4vSOtDGc8BIKh8gwmsStrIME= +git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088/go.mod h1:/EPCk+d5n5/uv92yVu9qJ8HhhNMqibbgGeFIUzP3k14= +git.sr.ht/~jamesponddotco/recache-go v1.0.1 h1:O9S7SdGyMh4mD+Vom0WOkY45EhJJHDRBlvVpzcL16sM= +git.sr.ht/~jamesponddotco/recache-go v1.0.1/go.mod h1:oF6LkAuwZYQqHe8+G/4hP9ZSNyDjAk6J8qhuy44wXw0= +git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382 h1:3LjkxT6zVDvIlaOc5riiRAnAKVgpTWHxWgmzpzOuR8A= +git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382/go.mod h1:0tqdK5/MZYSPxAiwtG4LlVfdQ+iaFoksU/FTIGQ/v/Y= gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a h1:awR9qREIs6qSnKr/cmSewVwDo74/kQ32x0CDEXUtiB8= gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a/go.mod h1:uF9EtbhODq5kNWxMbnBEj8hRRZnlcNSz2t2N7HW/+A4= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= @@ -1023,6 +1033,7 @@ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go new file mode 100644 index 000000000000..6bc46983d67f --- /dev/null +++ b/pkg/object/bunny.go @@ -0,0 +1,137 @@ +/* + * JuiceFS, Copyright 2024 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "bytes" + "context" + "fmt" + "io" + "path/filepath" + "strconv" + "strings" + "time" + + "git.sr.ht/~jamesponddotco/bunnystorage-go" + "github.com/juicedata/juicefs/pkg/version" +) + + +type bunnyClient struct { + DefaultObjectStorage + client *bunnystorage.Client + endpoint string +} + +// Description of the object storage. +func (b bunnyClient) String() string { + return fmt.Sprintf("bunny://%v", b.endpoint) +} + +// Limits of the object storage. +func (b bunnyClient) Limits() Limits { + return Limits{ + IsSupportMultipartUpload: false, + IsSupportUploadPartCopy: false, + } +} + +// Get the data for the given object specified by key. +func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { + dir, file := filepath.Split(key) + r, _, err := b.client.Download(context.Background(), dir, file) + if limit != -1 { + return io.NopCloser(bytes.NewReader(r[off:limit-1])), err + } else { + return io.NopCloser(bytes.NewReader(r[off:])), err + } +} + +// Put data read from a reader to an object specified by key. +func (b bunnyClient) Put(key string, in io.Reader) error { + dir, file := filepath.Split(key) + _, err := b.client.Upload(context.Background(), dir, file, "", in ) + return err +} + +// Delete a object. +func (b bunnyClient) Delete(key string) error { + dir, file := filepath.Split(key) + _, err := b.client.Delete(context.Background(), dir, file) + return err +} + +// ListAll returns all the objects as an channel. +func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { + objects, _, err := b.client.List(context.Background(), prefix) + if err != nil { + return nil , err + } + + c := make(chan Object) + + go bunnyObjectsToJuiceObjects(objects, c) + + return c, nil +} + +func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { + for o := range objects { + f := objects[o] + lastChanged, _ := strconv.Atoi(f.LastChanged) + out <- &obj{ + f.ObjectName, + int64(f.Length), + time.Unix(int64(lastChanged), 0), + f.IsDirectory, + "", + } + } + close(out) +} + +func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { + + split_endpoint := strings.SplitN(endpoint, ".", 2) + + zone_name := split_endpoint[0] + bunny_endpoint := bunnystorage.Parse(split_endpoint[1]) + + cfg := &bunnystorage.Config{ + Application: &bunnystorage.Application{ + Name: "JuiceFS", + Version: version.Version(), + Contact: "team@juicedata.io", + }, + StorageZone: zone_name, + Debug: false, + Key: password, + Endpoint: bunny_endpoint, + Logger: &logger.Logger, + } + + client, err := bunnystorage.NewClient(cfg) + + if err != nil { + return nil, fmt.Errorf("Unable to create Bunny client: %v", err) + } + return bunnyClient{client: client, endpoint: endpoint}, nil +} + +func init() { + Register("bunny", newBunny) +} \ No newline at end of file From e0f4cc0b745054cbdfb06893a2b25a229cabd014 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 9 Jan 2024 20:17:58 +0100 Subject: [PATCH 02/28] Add Documentation --- .../reference/how_to_set_up_object_storage.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/en/reference/how_to_set_up_object_storage.md b/docs/en/reference/how_to_set_up_object_storage.md index e5639d7d19ed..5b0d932cb544 100644 --- a/docs/en/reference/how_to_set_up_object_storage.md +++ b/docs/en/reference/how_to_set_up_object_storage.md @@ -159,6 +159,7 @@ If you wish to use a storage system that is not listed, feel free to submit a re | [Storj DCS](#storj-dcs) | `s3` | | [Vultr Object Storage](#vultr-object-storage) | `s3` | | [Cloudflare R2](#r2) | `s3` | +| [Bunny Storage](#bunny) | `bunny` | | [Alibaba Cloud OSS](#alibaba-cloud-oss) | `oss` | | [Tencent Cloud COS](#tencent-cloud-cos) | `cos` | | [Huawei Cloud OBS](#huawei-cloud-obs) | `obs` | @@ -502,6 +503,22 @@ juicefs format \ Cloudflare R2 `ListObjects` API is not fully S3 compatible (result list is not sorted), so some features of JuiceFS do not work. For example, `juicefs gc`, `juicefs fsck`, `juicefs sync`, `juicefs destroy`. And when using `juicefs mount`, you need to disable [automatic-backup](../administration/metadata_dump_load.md#backup-automatically) function by adding `--backup-meta 0`. ::: +### Bunny Storage {#bunny} + +Bunny Storage offers a non-S3 compatible object storage with multiple performance tiers and many storage regions. It uses [it uses a custom API](https://docs.bunny.net/reference/storage-api). + +#### Usage: + +Create a Storage Zone and use the Zone Name with the Hostname of the Location seperated by a dot as Bucket name and the `Write Password` as Secret Key. + +```shell +juicefs format \ + --storage bunny \ + --secret-key "write-password" \ + --bucket "juicefs.uk.storage.bunnycdn.com" \ # . + myjfs +``` + ### Alibaba Cloud OSS Please follow [this document](https://www.alibabacloud.com/help/doc-detail/125558.htm) to learn how to get access key and secret key. If you have already created [RAM role](https://www.alibabacloud.com/help/doc-detail/110376.htm) and assigned it to a VM instance, you could omit the options `--access-key` and `--secret-key`. From 9f265fa1d6f63b8e102d9612e5e47c46d83b0818 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 9 Jan 2024 23:13:24 +0100 Subject: [PATCH 03/28] Fix incorrect mtime parsing --- pkg/object/bunny.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 6bc46983d67f..443bf23e2b5f 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -92,11 +92,11 @@ func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-c func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { for o := range objects { f := objects[o] - lastChanged, _ := strconv.Atoi(f.LastChanged) + lastChanged, _ := time.Parse("2006-01-02T15:04:05", f.LastChanged) out <- &obj{ f.ObjectName, int64(f.Length), - time.Unix(int64(lastChanged), 0), + lastChanged, f.IsDirectory, "", } @@ -134,4 +134,4 @@ func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error func init() { Register("bunny", newBunny) -} \ No newline at end of file +} From 447495fe82556bef5971a9c6685540623cbbba37 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 10 Jan 2024 12:25:21 +0100 Subject: [PATCH 04/28] fix: Emulate bunny.net ranged reads properly --- pkg/object/bunny.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 443bf23e2b5f..4f32d259ed5d 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -21,8 +21,9 @@ import ( "context" "fmt" "io" + "net/http" + "os" "path/filepath" - "strconv" "strings" "time" @@ -53,12 +54,11 @@ func (b bunnyClient) Limits() Limits { // Get the data for the given object specified by key. func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { dir, file := filepath.Split(key) - r, _, err := b.client.Download(context.Background(), dir, file) - if limit != -1 { - return io.NopCloser(bytes.NewReader(r[off:limit-1])), err - } else { - return io.NopCloser(bytes.NewReader(r[off:])), err + body, response, err := b.client.Download(context.Background(), dir, file) + if response.Status == http.StatusNotFound { + return nil, os.ErrNotExist } + return io.NopCloser(io.NewSectionReader(bytes.NewReader(body), off, limit)), err } // Put data read from a reader to an object specified by key. @@ -106,7 +106,7 @@ func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Objec func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { - split_endpoint := strings.SplitN(endpoint, ".", 2) + split_endpoint := strings.SplitN(endpoint, ".", 2) // TODO: Validate Endpoint and Logging zone_name := split_endpoint[0] bunny_endpoint := bunnystorage.Parse(split_endpoint[1]) @@ -118,7 +118,7 @@ func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error Contact: "team@juicedata.io", }, StorageZone: zone_name, - Debug: false, + Debug: false, // TODO: Expose Debug Flag Key: password, Endpoint: bunny_endpoint, Logger: &logger.Logger, From cbac60aba92691c297ef2ca73d1caee80a5a09ee Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 10 Jan 2024 14:00:03 +0100 Subject: [PATCH 05/28] fix: split keys into file and dir correctly --- pkg/object/bunny.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 4f32d259ed5d..57a9d3d15f3d 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -53,7 +53,7 @@ func (b bunnyClient) Limits() Limits { // Get the data for the given object specified by key. func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { - dir, file := filepath.Split(key) + dir, file := splitKey(key) body, response, err := b.client.Download(context.Background(), dir, file) if response.Status == http.StatusNotFound { return nil, os.ErrNotExist @@ -63,14 +63,14 @@ func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, err // Put data read from a reader to an object specified by key. func (b bunnyClient) Put(key string, in io.Reader) error { - dir, file := filepath.Split(key) + dir, file := splitKey(key) _, err := b.client.Upload(context.Background(), dir, file, "", in ) return err } // Delete a object. func (b bunnyClient) Delete(key string) error { - dir, file := filepath.Split(key) + dir, file := splitKey(key) _, err := b.client.Delete(context.Background(), dir, file) return err } @@ -89,6 +89,15 @@ func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-c return c, nil } +// Helper function that is needed because Bunnystorage API Client +// requires seperate directory and file inputs but does not +// sanitize it's input properly before assembling the request URL +func splitKey(key string) (dir string, file string) { + dir, file = filepath.Split(key) + dir, _ = strings.CutSuffix(dir, "/") + return dir, file +} + func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { for o := range objects { f := objects[o] From 2350270c275e4071da91cfc8ec1ca6bbeb929fd5 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 10 Jan 2024 14:10:52 +0100 Subject: [PATCH 06/28] fix: Improve error handling --- pkg/object/bunny.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 57a9d3d15f3d..8de3dcc75d68 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -29,6 +29,7 @@ import ( "git.sr.ht/~jamesponddotco/bunnystorage-go" "github.com/juicedata/juicefs/pkg/version" + "github.com/pkg/errors" ) @@ -64,14 +65,25 @@ func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, err // Put data read from a reader to an object specified by key. func (b bunnyClient) Put(key string, in io.Reader) error { dir, file := splitKey(key) - _, err := b.client.Upload(context.Background(), dir, file, "", in ) + + resp, err := b.client.Upload(context.Background(), dir, file, "", in ) + if err != nil { + return err + } + if resp.Status != http.StatusCreated { + return os.ErrInvalid + } return err } // Delete a object. func (b bunnyClient) Delete(key string) error { dir, file := splitKey(key) - _, err := b.client.Delete(context.Background(), dir, file) + resp, err := b.client.Delete(context.Background(), dir, file) + + if resp.Status == http.StatusBadRequest { + return errors.Errorf("Unable to delete key %v", resp.Header) + } return err } From 35e9b134fac41b20991c3a2947a85354b0803ded Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 10 Jan 2024 16:13:43 +0100 Subject: [PATCH 07/28] fix: handle errors on delete correctly --- pkg/object/bunny.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 8de3dcc75d68..2f3c9068bde3 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -80,11 +80,13 @@ func (b bunnyClient) Put(key string, in io.Reader) error { func (b bunnyClient) Delete(key string) error { dir, file := splitKey(key) resp, err := b.client.Delete(context.Background(), dir, file) - + if err != nil { + return err + } if resp.Status == http.StatusBadRequest { return errors.Errorf("Unable to delete key %v", resp.Header) } - return err + return nil } // ListAll returns all the objects as an channel. From a8070ad0d482f44e6f44ce1ddbd820f63cfe338b Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 10 Jan 2024 22:10:50 +0100 Subject: [PATCH 08/28] fix: go fmt --- pkg/object/bunny.go | 53 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 2f3c9068bde3..93bc7a700a48 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -32,10 +32,9 @@ import ( "github.com/pkg/errors" ) - -type bunnyClient struct { +type bunnyClient struct { DefaultObjectStorage - client *bunnystorage.Client + client *bunnystorage.Client endpoint string } @@ -48,15 +47,15 @@ func (b bunnyClient) String() string { func (b bunnyClient) Limits() Limits { return Limits{ IsSupportMultipartUpload: false, - IsSupportUploadPartCopy: false, + IsSupportUploadPartCopy: false, } } // Get the data for the given object specified by key. func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { - dir, file := splitKey(key) + dir, file := splitKey(key) body, response, err := b.client.Download(context.Background(), dir, file) - if response.Status == http.StatusNotFound { + if response.Status == http.StatusNotFound { return nil, os.ErrNotExist } return io.NopCloser(io.NewSectionReader(bytes.NewReader(body), off, limit)), err @@ -64,13 +63,13 @@ func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, err // Put data read from a reader to an object specified by key. func (b bunnyClient) Put(key string, in io.Reader) error { - dir, file := splitKey(key) + dir, file := splitKey(key) - resp, err := b.client.Upload(context.Background(), dir, file, "", in ) - if err != nil { + resp, err := b.client.Upload(context.Background(), dir, file, "", in) + if err != nil { return err } - if resp.Status != http.StatusCreated { + if resp.Status != http.StatusCreated { return os.ErrInvalid } return err @@ -78,12 +77,12 @@ func (b bunnyClient) Put(key string, in io.Reader) error { // Delete a object. func (b bunnyClient) Delete(key string) error { - dir, file := splitKey(key) + dir, file := splitKey(key) resp, err := b.client.Delete(context.Background(), dir, file) - if err != nil { + if err != nil { return err } - if resp.Status == http.StatusBadRequest { + if resp.Status == http.StatusBadRequest { return errors.Errorf("Unable to delete key %v", resp.Header) } return nil @@ -92,8 +91,8 @@ func (b bunnyClient) Delete(key string) error { // ListAll returns all the objects as an channel. func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { objects, _, err := b.client.List(context.Background(), prefix) - if err != nil { - return nil , err + if err != nil { + return nil, err } c := make(chan Object) @@ -106,14 +105,14 @@ func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-c // Helper function that is needed because Bunnystorage API Client // requires seperate directory and file inputs but does not // sanitize it's input properly before assembling the request URL -func splitKey(key string) (dir string, file string) { - dir, file = filepath.Split(key) +func splitKey(key string) (dir string, file string) { + dir, file = filepath.Split(key) dir, _ = strings.CutSuffix(dir, "/") return dir, file } -func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { - for o := range objects { +func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { + for o := range objects { f := objects[o] lastChanged, _ := time.Parse("2006-01-02T15:04:05", f.LastChanged) out <- &obj{ @@ -127,7 +126,7 @@ func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Objec close(out) } -func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { +func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { split_endpoint := strings.SplitN(endpoint, ".", 2) // TODO: Validate Endpoint and Logging @@ -136,25 +135,25 @@ func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error cfg := &bunnystorage.Config{ Application: &bunnystorage.Application{ - Name: "JuiceFS", + Name: "JuiceFS", Version: version.Version(), Contact: "team@juicedata.io", }, StorageZone: zone_name, - Debug: false, // TODO: Expose Debug Flag - Key: password, - Endpoint: bunny_endpoint, - Logger: &logger.Logger, + Debug: false, // TODO: Expose Debug Flag + Key: password, + Endpoint: bunny_endpoint, + Logger: &logger.Logger, } client, err := bunnystorage.NewClient(cfg) - if err != nil { + if err != nil { return nil, fmt.Errorf("Unable to create Bunny client: %v", err) } return bunnyClient{client: client, endpoint: endpoint}, nil } -func init() { +func init() { Register("bunny", newBunny) } From a13c27542a0b1bbf29139a15c780cda9d4fd96b1 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 19 Jan 2024 23:51:02 +0100 Subject: [PATCH 09/28] Replace https://sr.ht/~jamesponddotco/bunnystorage-go with github.com/l0wl3vel/bunny-storage-go-sdk - Update modules - Update documentation --- .../reference/how_to_set_up_object_storage.md | 2 +- go.mod | 21 ++--- go.sum | 46 ++++++----- pkg/object/bunny.go | 82 +++++-------------- 4 files changed, 53 insertions(+), 98 deletions(-) diff --git a/docs/en/reference/how_to_set_up_object_storage.md b/docs/en/reference/how_to_set_up_object_storage.md index 1fb0e9b06fc3..4fd3a5ef50c6 100644 --- a/docs/en/reference/how_to_set_up_object_storage.md +++ b/docs/en/reference/how_to_set_up_object_storage.md @@ -515,7 +515,7 @@ Create a Storage Zone and use the Zone Name with the Hostname of the Location se juicefs format \ --storage bunny \ --secret-key "write-password" \ - --bucket "juicefs.uk.storage.bunnycdn.com" \ # . + --bucket "https://uk.storage.bunnycdn.com/myzone" \ # https:/// myjfs ``` diff --git a/go.mod b/go.mod index ff5a8f420d77..41eaf5e91cc8 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/gofrs/flock v0.8.1 github.com/google/btree v1.1.2 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.5.0 github.com/grafana/pyroscope-go v1.0.4 github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 github.com/hashicorp/consul/api v1.15.2 @@ -38,6 +38,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -54,7 +55,7 @@ require ( github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 github.com/qiniu/go-sdk/v7 v7.15.0 github.com/redis/go-redis/v9 v9.0.2 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/smartystreets/goconvey v1.7.2 github.com/stretchr/testify v1.8.4 github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 @@ -71,12 +72,12 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.uber.org/automaxprocs v1.5.2 go.uber.org/zap v1.20.0 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.17.0 + golang.org/x/crypto v0.18.0 + golang.org/x/net v0.20.0 golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.2.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/sys v0.16.0 + golang.org/x/term v0.16.0 golang.org/x/text v0.14.0 google.golang.org/api v0.94.0 google.golang.org/protobuf v1.30.0 @@ -85,18 +86,12 @@ require ( xorm.io/xorm v1.0.7 ) -require ( - git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7 // indirect - git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088 // indirect - git.sr.ht/~jamesponddotco/recache-go v1.0.1 // indirect - git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382 // indirect -) +require github.com/go-resty/resty/v2 v2.11.0 // indirect require ( cloud.google.com/go v0.102.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect git.apache.org/thrift.git v0.13.0 // indirect - git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect diff --git a/go.sum b/go.sum index 8881fa7ef7dc..a332b24323cf 100644 --- a/go.sum +++ b/go.sum @@ -64,16 +64,6 @@ contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRq dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.13.0 h1:/3bz5WZ+sqYArk7MBBBbDufMxKKOA56/6JO6psDpUDY= git.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0 h1:AwBxhQITDhJaJnEzbvgqwyMiscG8fo4PRFJvzz/ZnQg= -git.sr.ht/~jamesponddotco/bunnystorage-go v0.3.0/go.mod h1:x5kqzC+x93KaVYwBThSuNVFdHmErySaeUd2AwYNMo/0= -git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7 h1:uY+pFjSyrV1q54kai43xcu+HnHGw3AfG14TpYEIuURY= -git.sr.ht/~jamesponddotco/httpx-go v0.0.0-20230427215504-7c26a7f028e7/go.mod h1:5b9IHkokuXKVcAG5bmf0ej26NVMpV1usjIwrwOyR2SE= -git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088 h1:LKK7NuFKNBduMQkhg/Q4vSOtDGc8BIKh8gwmsStrIME= -git.sr.ht/~jamesponddotco/pagecache-go v0.0.0-20230411150210-54b704d32088/go.mod h1:/EPCk+d5n5/uv92yVu9qJ8HhhNMqibbgGeFIUzP3k14= -git.sr.ht/~jamesponddotco/recache-go v1.0.1 h1:O9S7SdGyMh4mD+Vom0WOkY45EhJJHDRBlvVpzcL16sM= -git.sr.ht/~jamesponddotco/recache-go v1.0.1/go.mod h1:oF6LkAuwZYQqHe8+G/4hP9ZSNyDjAk6J8qhuy44wXw0= -git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382 h1:3LjkxT6zVDvIlaOc5riiRAnAKVgpTWHxWgmzpzOuR8A= -git.sr.ht/~jamesponddotco/xstd-go v0.0.0-20230709232003-22489c0e7382/go.mod h1:0tqdK5/MZYSPxAiwtG4LlVfdQ+iaFoksU/FTIGQ/v/Y= gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a h1:awR9qREIs6qSnKr/cmSewVwDo74/kQ32x0CDEXUtiB8= gitea.com/davies/xorm v1.0.8-0.20220528043536-552d84d1b34a/go.mod h1:uF9EtbhODq5kNWxMbnBEj8hRRZnlcNSz2t2N7HW/+A4= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= @@ -118,7 +108,7 @@ github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzU github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -351,6 +341,8 @@ github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -471,8 +463,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= @@ -630,8 +623,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnTaUqfRYjlwcQlr53aasmkvHEbYLXthqr8= -github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7 h1:4evzoVz1/AZfk9tqxWdzVYTMl2dC7VjEJHfaSFDrKS8= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/juicedata/go-nfs-client v0.0.0-20231018052507-dbca444fa7e8 h1:mVVipCbohnzKZPiHGzFncLKEJZpypqjpGr4End2PP48= @@ -690,6 +681,8 @@ github.com/ks3sdklib/aws-sdk-go v1.2.2 h1:kykh38h9jQ+NGRsn0lm8NAXn2RKB+DotZIyvJ5 github.com/ks3sdklib/aws-sdk-go v1.2.2/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3 h1:d5PTG/G09/lCyQ9czwgKdyAcYv6aGTxBerGjWlDtIcI= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3/go.mod h1:QbNrUr8iUm41SgXl8Za/WmssfEYv8KdAtQPCCmlvnSk= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -950,8 +943,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -1035,6 +1029,7 @@ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= @@ -1134,8 +1129,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1236,8 +1232,10 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1382,15 +1380,19 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1404,6 +1406,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 93bc7a700a48..fdb62e6b517f 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -18,18 +18,11 @@ package object import ( "bytes" - "context" - "fmt" "io" - "net/http" - "os" - "path/filepath" - "strings" + "net/url" "time" - "git.sr.ht/~jamesponddotco/bunnystorage-go" - "github.com/juicedata/juicefs/pkg/version" - "github.com/pkg/errors" + bunnystorage "github.com/l0wl3vel/bunny-storage-go-sdk" ) type bunnyClient struct { @@ -40,7 +33,7 @@ type bunnyClient struct { // Description of the object storage. func (b bunnyClient) String() string { - return fmt.Sprintf("bunny://%v", b.endpoint) + return b.endpoint } // Limits of the object storage. @@ -53,44 +46,31 @@ func (b bunnyClient) Limits() Limits { // Get the data for the given object specified by key. func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { - dir, file := splitKey(key) - body, response, err := b.client.Download(context.Background(), dir, file) - if response.Status == http.StatusNotFound { - return nil, os.ErrNotExist + body, err := b.client.Download(key) + if err != nil { + return nil, err } return io.NopCloser(io.NewSectionReader(bytes.NewReader(body), off, limit)), err } // Put data read from a reader to an object specified by key. func (b bunnyClient) Put(key string, in io.Reader) error { - dir, file := splitKey(key) - - resp, err := b.client.Upload(context.Background(), dir, file, "", in) - if err != nil { - return err - } - if resp.Status != http.StatusCreated { - return os.ErrInvalid + content, readErr := io.ReadAll(in) + if readErr != nil { + return readErr } + err := b.client.Upload(key, content, true) return err } // Delete a object. func (b bunnyClient) Delete(key string) error { - dir, file := splitKey(key) - resp, err := b.client.Delete(context.Background(), dir, file) - if err != nil { - return err - } - if resp.Status == http.StatusBadRequest { - return errors.Errorf("Unable to delete key %v", resp.Header) - } - return nil + return b.client.Delete(key, false) } // ListAll returns all the objects as an channel. func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { - objects, _, err := b.client.List(context.Background(), prefix) + objects, err := b.client.List(prefix) if err != nil { return nil, err } @@ -102,16 +82,7 @@ func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-c return c, nil } -// Helper function that is needed because Bunnystorage API Client -// requires seperate directory and file inputs but does not -// sanitize it's input properly before assembling the request URL -func splitKey(key string) (dir string, file string) { - dir, file = filepath.Split(key) - dir, _ = strings.CutSuffix(dir, "/") - return dir, file -} - -func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Object) { +func bunnyObjectsToJuiceObjects(objects []bunnystorage.Object, out chan<- Object) { for o := range objects { f := objects[o] lastChanged, _ := time.Parse("2006-01-02T15:04:05", f.LastChanged) @@ -128,30 +99,15 @@ func bunnyObjectsToJuiceObjects(objects []*bunnystorage.Object, out chan<- Objec func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { - split_endpoint := strings.SplitN(endpoint, ".", 2) // TODO: Validate Endpoint and Logging - - zone_name := split_endpoint[0] - bunny_endpoint := bunnystorage.Parse(split_endpoint[1]) - - cfg := &bunnystorage.Config{ - Application: &bunnystorage.Application{ - Name: "JuiceFS", - Version: version.Version(), - Contact: "team@juicedata.io", - }, - StorageZone: zone_name, - Debug: false, // TODO: Expose Debug Flag - Key: password, - Endpoint: bunny_endpoint, - Logger: &logger.Logger, - } - - client, err := bunnystorage.NewClient(cfg) + endpoint_url, err := url.Parse(endpoint) if err != nil { - return nil, fmt.Errorf("Unable to create Bunny client: %v", err) + return nil, err } - return bunnyClient{client: client, endpoint: endpoint}, nil + + client := bunnystorage.NewClient(*endpoint_url, password) + + return bunnyClient{client: &client, endpoint: endpoint}, nil } func init() { From f34b2a76b77197efecac7764faf40cce84f74ff9 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 21 Jan 2024 13:31:16 +0100 Subject: [PATCH 10/28] Update github.com/l0wl3vel/bunny-storage-go-sdk to v0.0.4 --- go.mod | 6 +++--- go.sum | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 41eaf5e91cc8..216a1871d975 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 - github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -172,7 +172,7 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.17.3 // indirect github.com/klauspost/cpuid v1.3.1 // indirect - github.com/klauspost/cpuid/v2 v2.0.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/readahead v1.3.1 // indirect github.com/klauspost/reedsolomon v1.9.11 // indirect @@ -185,7 +185,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/md5-simd v1.1.1 // indirect github.com/minio/selfupdate v0.3.1 // indirect - github.com/minio/sha256-simd v0.1.1 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/minio/simdjson-go v0.2.1 // indirect github.com/minio/sio v0.2.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index a332b24323cf..e777d4a05147 100644 --- a/go.sum +++ b/go.sum @@ -653,8 +653,9 @@ github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.3 h1:DNljyrHyxlkk8139OXIAAauCwV8eQGDD6Z8YqnDXdZw= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0= @@ -681,8 +682,8 @@ github.com/ks3sdklib/aws-sdk-go v1.2.2 h1:kykh38h9jQ+NGRsn0lm8NAXn2RKB+DotZIyvJ5 github.com/ks3sdklib/aws-sdk-go v1.2.2/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3 h1:d5PTG/G09/lCyQ9czwgKdyAcYv6aGTxBerGjWlDtIcI= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.3/go.mod h1:QbNrUr8iUm41SgXl8Za/WmssfEYv8KdAtQPCCmlvnSk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4 h1:uyDaYY6yQ7OWV/A439r7taIg4OlZW4Qq0kfYN/urMUs= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -741,8 +742,9 @@ github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauX github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/simdjson-go v0.2.1 h1:nxYlp4Qd0w2pwLlif00l5vTFL6PcNAKpyHq27/pageg= github.com/minio/simdjson-go v0.2.1/go.mod h1:JPUSkRykfSPS+AhO0YPA1h0l5vY7NqrF4zel2b12wxc= github.com/minio/sio v0.2.1 h1:NjzKiIMSMcHediVQR0AFVx2tp7Wxh9tKPfDI3kH7aHQ= @@ -1371,6 +1373,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 1b845357d17eb922d7c6f7a24966dad4bd65405a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 23 Jan 2024 17:39:09 +0100 Subject: [PATCH 11/28] Update bunny-storage-go-sdk to v0.0.6 --- go.mod | 4 ++-- go.sum | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 216a1871d975..6cfa6953d3a4 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 - github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -172,7 +172,7 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.17.3 // indirect github.com/klauspost/cpuid v1.3.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/readahead v1.3.1 // indirect github.com/klauspost/reedsolomon v1.9.11 // indirect diff --git a/go.sum b/go.sum index e777d4a05147..31d4757d420b 100644 --- a/go.sum +++ b/go.sum @@ -656,6 +656,8 @@ github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0= @@ -684,6 +686,10 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LE github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4 h1:uyDaYY6yQ7OWV/A439r7taIg4OlZW4Qq0kfYN/urMUs= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5 h1:JT1x1n3/eUPQvwas1G9dFb16eEYKERKPsRVkJKg9bH4= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6 h1:LUy7DxG556qFWGM2yL6DZvfQr/BNP9NIA6walkt0dkg= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= From a88e4f3f63378178a37d6de96a88be47ae227692 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 23 Jan 2024 17:41:21 +0100 Subject: [PATCH 12/28] Change all bunnyClient methods to pointer receivers --- pkg/object/bunny.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index fdb62e6b517f..d8d5d725fbb8 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -32,12 +32,12 @@ type bunnyClient struct { } // Description of the object storage. -func (b bunnyClient) String() string { +func (b *bunnyClient) String() string { return b.endpoint } // Limits of the object storage. -func (b bunnyClient) Limits() Limits { +func (b *bunnyClient) Limits() Limits { return Limits{ IsSupportMultipartUpload: false, IsSupportUploadPartCopy: false, @@ -45,7 +45,7 @@ func (b bunnyClient) Limits() Limits { } // Get the data for the given object specified by key. -func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { +func (b *bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { body, err := b.client.Download(key) if err != nil { return nil, err @@ -54,7 +54,7 @@ func (b bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, err } // Put data read from a reader to an object specified by key. -func (b bunnyClient) Put(key string, in io.Reader) error { +func (b *bunnyClient) Put(key string, in io.Reader) error { content, readErr := io.ReadAll(in) if readErr != nil { return readErr @@ -64,12 +64,12 @@ func (b bunnyClient) Put(key string, in io.Reader) error { } // Delete a object. -func (b bunnyClient) Delete(key string) error { +func (b *bunnyClient) Delete(key string) error { return b.client.Delete(key, false) } // ListAll returns all the objects as an channel. -func (b bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { +func (b *bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { objects, err := b.client.List(prefix) if err != nil { return nil, err @@ -107,7 +107,7 @@ func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error client := bunnystorage.NewClient(*endpoint_url, password) - return bunnyClient{client: &client, endpoint: endpoint}, nil + return &bunnyClient{client: &client, endpoint: endpoint}, nil } func init() { From d6f7599bb8779252e821da3166a1a0ca8b15d3f0 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 23 Jan 2024 17:45:07 +0100 Subject: [PATCH 13/28] Factor out parseObjectMetadata to its own method --- pkg/object/bunny.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index d8d5d725fbb8..ab81ff3790d4 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -85,18 +85,26 @@ func (b *bunnyClient) ListAll(prefix string, marker string, followLink bool) (<- func bunnyObjectsToJuiceObjects(objects []bunnystorage.Object, out chan<- Object) { for o := range objects { f := objects[o] - lastChanged, _ := time.Parse("2006-01-02T15:04:05", f.LastChanged) - out <- &obj{ - f.ObjectName, - int64(f.Length), - lastChanged, - f.IsDirectory, - "", - } + out <- parseObjectMetadata(f) } close(out) } +// Parse Bunnystorage API Object to JuiceFS Object +func parseObjectMetadata(object bunnystorage.Object) Object { + lastChanged, _ := time.Parse("2006-01-02T15:04:05", object.LastChanged) +/* out, _ := json.Marshal(object) + logger.Printf("Parsed Object: %v", string(out)) */ + return &obj{ + object.ObjectName, + int64(object.Length), + lastChanged, + object.IsDirectory, + "", + } +} + + func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { endpoint_url, err := url.Parse(endpoint) From 12d07125072ca657dd1af39a9501bdab02429482 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 23 Jan 2024 17:45:53 +0100 Subject: [PATCH 14/28] Add Bunny Head function --- pkg/object/bunny.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index ab81ff3790d4..52f78dc7a1e5 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -104,6 +104,15 @@ func parseObjectMetadata(object bunnystorage.Object) Object { } } +func (b *bunnyClient) Head(key string) (Object, error) { + logger.Debug(key) + object, err := b.client.Describe(key) + logger.Debug(object) + if err != nil { + return nil, err + } + return parseObjectMetadata(object), nil +} func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { From dd3011838fdcdd99556173916be52659eba0830d Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 25 Jan 2024 10:21:36 +0100 Subject: [PATCH 15/28] fix: Update Bunny.net Documentation --- docs/en/reference/how_to_set_up_object_storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/reference/how_to_set_up_object_storage.md b/docs/en/reference/how_to_set_up_object_storage.md index 4fd3a5ef50c6..2e76567300e6 100644 --- a/docs/en/reference/how_to_set_up_object_storage.md +++ b/docs/en/reference/how_to_set_up_object_storage.md @@ -507,7 +507,7 @@ Cloudflare R2 `ListObjects` API is not fully S3 compatible (result list is not s Bunny Storage offers a non-S3 compatible object storage with multiple performance tiers and many storage regions. It uses [it uses a custom API](https://docs.bunny.net/reference/storage-api). -#### Usage: +#### Usage Create a Storage Zone and use the Zone Name with the Hostname of the Location seperated by a dot as Bucket name and the `Write Password` as Secret Key. From cb3471d1c7ccf0dccef21798eee7c55e94b41068 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 26 Jan 2024 18:49:21 +0100 Subject: [PATCH 16/28] feat: Implement real ranged downloads for bunny.net --- pkg/object/bunny.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 52f78dc7a1e5..c349237a8317 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -19,6 +19,7 @@ package object import ( "bytes" "io" + "math" "net/url" "time" @@ -46,11 +47,14 @@ func (b *bunnyClient) Limits() Limits { // Get the data for the given object specified by key. func (b *bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { - body, err := b.client.Download(key) + if limit == -1 { + limit = math.MaxInt64 + } + body, err := b.client.DownloadPartial(key, off, limit+off-1) if err != nil { return nil, err } - return io.NopCloser(io.NewSectionReader(bytes.NewReader(body), off, limit)), err + return io.NopCloser(bytes.NewReader(body)), nil } // Put data read from a reader to an object specified by key. From 2b5f2c6166ca9c9f782cc5f04045193054623d54 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sat, 27 Jan 2024 15:39:15 +0100 Subject: [PATCH 17/28] feat: Use ReaderCloser semantics for Bunny Object Store Get Function --- go.mod | 2 +- go.sum | 2 ++ pkg/object/bunny.go | 17 ++++++----------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 6cfa6953d3a4..ccb67ca0e8b3 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 - github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 diff --git a/go.sum b/go.sum index 31d4757d420b..696c084ef154 100644 --- a/go.sum +++ b/go.sum @@ -690,6 +690,8 @@ github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5 h1:JT1x1n3/eUPQvwas1G9dFb16eEYKE github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6 h1:LUy7DxG556qFWGM2yL6DZvfQr/BNP9NIA6walkt0dkg= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 h1:pDH6sQ+KJO19M0g2hP5942FKb4fZ+zgs+3f1rUH+ULg= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index c349237a8317..691d61ffd01f 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -17,7 +17,6 @@ package object import ( - "bytes" "io" "math" "net/url" @@ -50,11 +49,11 @@ func (b *bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, er if limit == -1 { limit = math.MaxInt64 } - body, err := b.client.DownloadPartial(key, off, limit+off-1) + body, err := b.client.DownloadPartialWithReaderCloser(key, off, limit+off-1) if err != nil { return nil, err } - return io.NopCloser(bytes.NewReader(body)), nil + return body, nil } // Put data read from a reader to an object specified by key. @@ -63,8 +62,7 @@ func (b *bunnyClient) Put(key string, in io.Reader) error { if readErr != nil { return readErr } - err := b.client.Upload(key, content, true) - return err + return b.client.Upload(key, content, true) } // Delete a object. @@ -95,10 +93,8 @@ func bunnyObjectsToJuiceObjects(objects []bunnystorage.Object, out chan<- Object } // Parse Bunnystorage API Object to JuiceFS Object -func parseObjectMetadata(object bunnystorage.Object) Object { +func parseObjectMetadata(object bunnystorage.Object) Object { lastChanged, _ := time.Parse("2006-01-02T15:04:05", object.LastChanged) -/* out, _ := json.Marshal(object) - logger.Printf("Parsed Object: %v", string(out)) */ return &obj{ object.ObjectName, int64(object.Length), @@ -108,18 +104,17 @@ func parseObjectMetadata(object bunnystorage.Object) Object { } } -func (b *bunnyClient) Head(key string) (Object, error) { +func (b *bunnyClient) Head(key string) (Object, error) { logger.Debug(key) object, err := b.client.Describe(key) logger.Debug(object) - if err != nil { + if err != nil { return nil, err } return parseObjectMetadata(object), nil } func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { - endpoint_url, err := url.Parse(endpoint) if err != nil { From 91f0ba09b0535f5fe960de6ccf6f1fae0d13426a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sat, 27 Jan 2024 15:46:58 +0100 Subject: [PATCH 18/28] fix: Add l0wl3vel/bunny-storage-go-sdk dependencies --- go.mod | 10 ++++++---- go.sum | 27 +++++++++++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index ccb67ca0e8b3..d045fd057038 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,6 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 - github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -62,7 +61,7 @@ require ( github.com/tencentyun/cos-go-sdk-v5 v0.7.45 github.com/tikv/client-go/v2 v2.0.4 github.com/upyun/go-sdk/v3 v3.0.4 - github.com/urfave/cli/v2 v2.25.7 + github.com/urfave/cli/v2 v2.19.3 github.com/vbauerster/mpb/v7 v7.0.3 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b @@ -86,7 +85,10 @@ require ( xorm.io/xorm v1.0.7 ) -require github.com/go-resty/resty/v2 v2.11.0 // indirect +require ( + github.com/go-resty/resty/v2 v2.11.0 // indirect + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 // indirect +) require ( cloud.google.com/go v0.102.1 // indirect @@ -172,7 +174,7 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.17.3 // indirect github.com/klauspost/cpuid v1.3.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/readahead v1.3.1 // indirect github.com/klauspost/reedsolomon v1.9.11 // indirect diff --git a/go.sum b/go.sum index 696c084ef154..6c58d4f18515 100644 --- a/go.sum +++ b/go.sum @@ -108,7 +108,7 @@ github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzU github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -463,8 +463,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= @@ -623,6 +623,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnTaUqfRYjlwcQlr53aasmkvHEbYLXthqr8= +github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7 h1:4evzoVz1/AZfk9tqxWdzVYTMl2dC7VjEJHfaSFDrKS8= github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/juicedata/go-nfs-client v0.0.0-20231018052507-dbca444fa7e8 h1:mVVipCbohnzKZPiHGzFncLKEJZpypqjpGr4End2PP48= @@ -653,11 +655,10 @@ github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.3 h1:DNljyrHyxlkk8139OXIAAauCwV8eQGDD6Z8YqnDXdZw= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0= @@ -684,12 +685,6 @@ github.com/ks3sdklib/aws-sdk-go v1.2.2 h1:kykh38h9jQ+NGRsn0lm8NAXn2RKB+DotZIyvJ5 github.com/ks3sdklib/aws-sdk-go v1.2.2/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4 h1:uyDaYY6yQ7OWV/A439r7taIg4OlZW4Qq0kfYN/urMUs= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.4/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5 h1:JT1x1n3/eUPQvwas1G9dFb16eEYKERKPsRVkJKg9bH4= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.5/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6 h1:LUy7DxG556qFWGM2yL6DZvfQr/BNP9NIA6walkt0dkg= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.6/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 h1:pDH6sQ+KJO19M0g2hP5942FKb4fZ+zgs+3f1rUH+ULg= github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= @@ -750,6 +745,7 @@ github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauX github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= @@ -953,6 +949,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -1039,8 +1036,6 @@ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= @@ -1140,7 +1135,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1243,6 +1239,7 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= @@ -1392,6 +1389,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1402,7 +1400,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 2243ae1f0d1bf2add7cf834ef39501317928d95d Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 29 Jan 2024 12:36:26 +0100 Subject: [PATCH 19/28] fix: Add Bunny object storage test --- pkg/object/object_storage_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/object/object_storage_test.go b/pkg/object/object_storage_test.go index f26b8c86d4b9..cfab5b97f298 100644 --- a/pkg/object/object_storage_test.go +++ b/pkg/object/object_storage_test.go @@ -1027,6 +1027,17 @@ func TestDragonfly(t *testing.T) { //skip mutate testStorage(t, dragonfly) } +func TestBunny(t *testing.T) { //skip mutate + if os.Getenv("BUNNY_ENDPOINT") == "" { + t.SkipNow() + } + bunny, err := newBunny(os.Getenv("BUNNY_ENDPOINT"), "", os.Getenv("BUNNY_SECRET_KEY"), "") + if err != nil { + t.Fatalf("create: %s", err) + } + testStorage(t, bunny) +} + func TestMain(m *testing.M) { if envFile := os.Getenv("JUICEFS_ENV_FILE_FOR_TEST"); envFile != "" { // schema: S3 AWS_ENDPOINT=xxxxx From f5657a6da5f8b8f869cab0315e5c9174608f3594 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 7 Feb 2024 22:39:30 +0100 Subject: [PATCH 20/28] fix: Rework ListAll functionality to pass JuiceFS testsuite --- pkg/object/bunny.go | 62 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 691d61ffd01f..94c2ed041656 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -20,6 +20,9 @@ import ( "io" "math" "net/url" + "os" + "path" + "strings" "time" bunnystorage "github.com/l0wl3vel/bunny-storage-go-sdk" @@ -72,24 +75,60 @@ func (b *bunnyClient) Delete(key string) error { // ListAll returns all the objects as an channel. func (b *bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { + + out := make(chan Object) + go func() { + defer close(out) + b.walkObjects(prefix, marker, out) + }() + + return out, nil +} + +func (b *bunnyClient) walkObjects(prefix string, marker string, out chan<- Object) { objects, err := b.client.List(prefix) if err != nil { - return nil, err + return } - c := make(chan Object) + objectsToProcess := objects - go bunnyObjectsToJuiceObjects(objects, c) + // If no objects are returned list the parent directory and continue + if len(objectsToProcess) == 0 { + objectsToProcess, err = b.client.List(path.Dir(prefix)) + if err != nil { + return + } + } - return c, nil + markerEncountered := marker == "" + + for i := 0; i < len(objectsToProcess); i++ { + o := objectsToProcess[i] + normalizedPath := normalizedObjectNameWithinZone(o) + if strings.HasPrefix(normalizedPath, path.Join(prefix)) { + if normalizedPath == marker { + markerEncountered = true + } + if markerEncountered { + if o.IsDirectory { + objects, err := b.client.List(normalizedPath) + if err != nil { + return + } + objectsToProcess = append(objectsToProcess, objects...) + } else { + out <- parseObjectMetadata(o) + } + } + } + } } -func bunnyObjectsToJuiceObjects(objects []bunnystorage.Object, out chan<- Object) { - for o := range objects { - f := objects[o] - out <- parseObjectMetadata(f) - } - close(out) +// The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes +func normalizedObjectNameWithinZone(o bunnystorage.Object) string { + normalizedPath := path.Join(o.Path, o.ObjectName) + return strings.TrimPrefix(normalizedPath, "/"+o.StorageZoneName+"/") } // Parse Bunnystorage API Object to JuiceFS Object @@ -109,6 +148,9 @@ func (b *bunnyClient) Head(key string) (Object, error) { object, err := b.client.Describe(key) logger.Debug(object) if err != nil { + if err.Error() == "404 Not Found" { + return nil, os.ErrNotExist + } return nil, err } return parseObjectMetadata(object), nil From 680ce8c79866628e729e87e35bdcf9a4adc65d06 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 23 Feb 2024 18:08:29 +0100 Subject: [PATCH 21/28] fix: Use correct path normalization within Storage Zone previously the path parsed out of list responses did not contain the whole path. This commit allows juicefs delete, sync etc. to work properly --- pkg/object/bunny.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 94c2ed041656..f56e52b8e3c5 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -117,9 +117,8 @@ func (b *bunnyClient) walkObjects(prefix string, marker string, out chan<- Objec return } objectsToProcess = append(objectsToProcess, objects...) - } else { - out <- parseObjectMetadata(o) } + out <- parseObjectMetadata(o) } } } @@ -128,6 +127,9 @@ func (b *bunnyClient) walkObjects(prefix string, marker string, out chan<- Objec // The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes func normalizedObjectNameWithinZone(o bunnystorage.Object) string { normalizedPath := path.Join(o.Path, o.ObjectName) + if o.IsDirectory { + normalizedPath = normalizedPath+"/" // Append a trailing slash to allow deletion of directories + } return strings.TrimPrefix(normalizedPath, "/"+o.StorageZoneName+"/") } @@ -135,7 +137,7 @@ func normalizedObjectNameWithinZone(o bunnystorage.Object) string { func parseObjectMetadata(object bunnystorage.Object) Object { lastChanged, _ := time.Parse("2006-01-02T15:04:05", object.LastChanged) return &obj{ - object.ObjectName, + normalizedObjectNameWithinZone(object), int64(object.Length), lastChanged, object.IsDirectory, From f8d30f3c2a6fb8773c443e9f4cf1af24be915b90 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 23 Feb 2024 10:51:43 +0100 Subject: [PATCH 22/28] Update bunny-storage-go-sdk to 0.0.10 Required some fixes around error masking due to changed errors --- go.mod | 19 ++++++++++------- go.sum | 51 ++++++++++++++++++++++++--------------------- pkg/object/bunny.go | 30 ++++++++++++++++++++------ 3 files changed, 62 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index d045fd057038..b21077dbc6d4 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -71,12 +72,12 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.uber.org/automaxprocs v1.5.2 go.uber.org/zap v1.20.0 - golang.org/x/crypto v0.18.0 - golang.org/x/net v0.20.0 + golang.org/x/crypto v0.19.0 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.2.0 - golang.org/x/sys v0.16.0 - golang.org/x/term v0.16.0 + golang.org/x/sys v0.17.0 + golang.org/x/term v0.17.0 golang.org/x/text v0.14.0 google.golang.org/api v0.94.0 google.golang.org/protobuf v1.30.0 @@ -86,8 +87,10 @@ require ( ) require ( + github.com/andybalholm/brotli v1.1.0 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect - github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.52.0 // indirect ) require ( @@ -172,9 +175,9 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid v1.3.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/readahead v1.3.1 // indirect github.com/klauspost/reedsolomon v1.9.11 // indirect @@ -235,7 +238,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twmb/murmur3 v1.1.3 // indirect - github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a // indirect + github.com/valyala/tcplisten v1.0.0 // indirect github.com/willf/bitset v1.1.11 // indirect github.com/willf/bloom v2.0.3+incompatible // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect diff --git a/go.sum b/go.sum index 6c58d4f18515..f2a6d91d700a 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee/go.mod github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -463,8 +465,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= @@ -648,17 +650,17 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.3 h1:DNljyrHyxlkk8139OXIAAauCwV8eQGDD6Z8YqnDXdZw= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0= @@ -685,8 +687,12 @@ github.com/ks3sdklib/aws-sdk-go v1.2.2 h1:kykh38h9jQ+NGRsn0lm8NAXn2RKB+DotZIyvJ5 github.com/ks3sdklib/aws-sdk-go v1.2.2/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9 h1:pDH6sQ+KJO19M0g2hP5942FKb4fZ+zgs+3f1rUH+ULg= -github.com/l0wl3vel/bunny-storage-go-sdk v0.0.9/go.mod h1:gecEfS4DsHJX7Wr+X6dj5QuJh8vhyj2zyvhMgEdbfAk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240222214432-60c84bce3bf9 h1:D0B0I47sgXkXsYMVorBT04CgUVLXYS1xm9rEGL9nnTk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240222214432-60c84bce3bf9/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240223162144-5045a4b99139 h1:02FHEfQzys+SrCsx/rtOcvo5c7hLhd7aCeB/OAcBrDg= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240223162144-5045a4b99139/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10 h1:Vy8I4nGazW1QvwdIR3b/viHmBVFBf2i4RgR0dV0wJ/c= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -745,7 +751,6 @@ github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauX github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= @@ -949,7 +954,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -1036,8 +1040,12 @@ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0= +github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.3 h1:sc7EfqfTjMJtPtx8vYUDIL9WmmJtmamMFYxWF467IGw= @@ -1135,9 +1143,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1239,10 +1246,9 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1378,7 +1384,6 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1389,10 +1394,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1400,9 +1404,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index f56e52b8e3c5..b6d43708a5eb 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -17,6 +17,7 @@ package object import ( + "bytes" "io" "math" "net/url" @@ -52,11 +53,11 @@ func (b *bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, er if limit == -1 { limit = math.MaxInt64 } - body, err := b.client.DownloadPartialWithReaderCloser(key, off, limit+off-1) + body, err := b.client.DownloadPartial(key, off, limit+off-1) if err != nil { return nil, err } - return body, nil + return io.NopCloser(bytes.NewReader(body)), nil } // Put data read from a reader to an object specified by key. @@ -69,8 +70,25 @@ func (b *bunnyClient) Put(key string, in io.Reader) error { } // Delete a object. +// Requires a conditional retry, since deleting a directory or file called foo/bar requires two different calls to the Bunny API, which JuiceFS does not do +// Deleting a directory requires a trailing slash in the key to delete, which JuiceFS does not add to the path, leading to test case failures. +// We implement a conditional retry here to try deleting the directory if the delete for a key of the passed name fails. +// Deleting keys that do not exist are expected to not throw an error func (b *bunnyClient) Delete(key string) error { - return b.client.Delete(key, false) + if err := b.client.Delete(key, false); err != nil { + if err.Error() == "Not Found" { + // Retry delete the directory with the same name + if errDirectoryDelete := b.client.Delete(key, true); err != nil { + if err.Error() == "Not Found" { + return nil + } else { + return errDirectoryDelete + } + } + } + return err + } + return nil } // ListAll returns all the objects as an channel. @@ -127,8 +145,8 @@ func (b *bunnyClient) walkObjects(prefix string, marker string, out chan<- Objec // The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes func normalizedObjectNameWithinZone(o bunnystorage.Object) string { normalizedPath := path.Join(o.Path, o.ObjectName) - if o.IsDirectory { - normalizedPath = normalizedPath+"/" // Append a trailing slash to allow deletion of directories + if o.IsDirectory { + normalizedPath = normalizedPath + "/" // Append a trailing slash to allow deletion of directories } return strings.TrimPrefix(normalizedPath, "/"+o.StorageZoneName+"/") } @@ -150,7 +168,7 @@ func (b *bunnyClient) Head(key string) (Object, error) { object, err := b.client.Describe(key) logger.Debug(object) if err != nil { - if err.Error() == "404 Not Found" { + if err.Error() == "Not Found" { return nil, os.ErrNotExist } return nil, err From 434fcd53aa3291ee69fc88600d910a14b65ed3ff Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 28 Feb 2024 23:17:30 +0100 Subject: [PATCH 23/28] fix: return correct endpoint --- pkg/object/bunny.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index b6d43708a5eb..93c2bd5770f8 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -18,6 +18,7 @@ package object import ( "bytes" + "fmt" "io" "math" "net/url" @@ -37,7 +38,7 @@ type bunnyClient struct { // Description of the object storage. func (b *bunnyClient) String() string { - return b.endpoint + return fmt.Sprintf("bunny://%v", b.endpoint) } // Limits of the object storage. From a8ca86e64b523ed97e097c4a2a11a63aeb0de4dd Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 28 Feb 2024 23:37:58 +0100 Subject: [PATCH 24/28] fix: Remove verbose logging in the Head function --- pkg/object/bunny.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index 93c2bd5770f8..f6e23c14ae90 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -165,9 +165,7 @@ func parseObjectMetadata(object bunnystorage.Object) Object { } func (b *bunnyClient) Head(key string) (Object, error) { - logger.Debug(key) object, err := b.client.Describe(key) - logger.Debug(object) if err != nil { if err.Error() == "Not Found" { return nil, os.ErrNotExist From b3058c00e8a66c0ef6ed9bb777b73abd72cabb36 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 29 Feb 2024 14:56:26 +0100 Subject: [PATCH 25/28] feat: Implement List method instead of ListAll --- pkg/object/bunny.go | 51 +++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 34 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index f6e23c14ae90..fea1399e034c 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -92,55 +92,38 @@ func (b *bunnyClient) Delete(key string) error { return nil } -// ListAll returns all the objects as an channel. -func (b *bunnyClient) ListAll(prefix string, marker string, followLink bool) (<-chan Object, error) { - - out := make(chan Object) - go func() { - defer close(out) - b.walkObjects(prefix, marker, out) - }() - - return out, nil -} - -func (b *bunnyClient) walkObjects(prefix string, marker string, out chan<- Object) { - objects, err := b.client.List(prefix) - if err != nil { - return +func (b *bunnyClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) { + if delimiter != "/" { + return nil, notSupported } - objectsToProcess := objects + //listInParentDirectory := !strings.HasSuffix(prefix, "/") - // If no objects are returned list the parent directory and continue - if len(objectsToProcess) == 0 { - objectsToProcess, err = b.client.List(path.Dir(prefix)) - if err != nil { - return - } + listedObjects, err := b.client.List(path.Dir(prefix)) + if err != nil { + logger.Errorf("Unable to list objects in path %v", prefix) + return nil, err } markerEncountered := marker == "" + output := make([]Object, 0) - for i := 0; i < len(objectsToProcess); i++ { - o := objectsToProcess[i] + logger.Warnf("List: %v %v", prefix, marker) + for i := 0; i < len(listedObjects); i++ { + o := listedObjects[i] normalizedPath := normalizedObjectNameWithinZone(o) - if strings.HasPrefix(normalizedPath, path.Join(prefix)) { + if strings.HasPrefix(normalizedPath, prefix) { if normalizedPath == marker { markerEncountered = true } if markerEncountered { - if o.IsDirectory { - objects, err := b.client.List(normalizedPath) - if err != nil { - return - } - objectsToProcess = append(objectsToProcess, objects...) - } - out <- parseObjectMetadata(o) + logger.Warn(normalizedPath) + output = append(output, parseObjectMetadata(o)) } } } + + return output, nil } // The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes From c4bb0300a0af998b29780bf3503492aa603eff0d Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 29 Mar 2024 21:31:36 +0100 Subject: [PATCH 26/28] fix: rework list function --- pkg/object/bunny.go | 47 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index fea1399e034c..d208a6667b10 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -96,30 +96,41 @@ func (b *bunnyClient) List(prefix, marker, delimiter string, limit int64, follow if delimiter != "/" { return nil, notSupported } + var output []Object + var dir = prefix + if !strings.HasSuffix(dir, dirSuffix) { // If no Directory list in parent directory + dir = path.Dir(dir) + if !strings.HasSuffix(dir, dirSuffix) { + dir += dirSuffix + } + } else if marker == "" { // If Directory && no marker: Return prefix directory as well + parentPath := path.Dir(path.Dir(prefix)) + objects, err := b.client.List(parentPath+dirSuffix) + if err == nil { + for _, o := range objects { + logger.Warnf("%v == %v", normalizedObjectNameWithinZone(o), path.Dir(prefix)+dirSuffix) + if normalizedObjectNameWithinZone(o) == path.Dir(prefix) { + output = append(output, parseObjectMetadata(o)) + } + } + } + } - //listInParentDirectory := !strings.HasSuffix(prefix, "/") - - listedObjects, err := b.client.List(path.Dir(prefix)) + listedObjects, err := b.client.List(dir) if err != nil { - logger.Errorf("Unable to list objects in path %v", prefix) + logger.Errorf("Unable to list objects in path %v with prefix %v", dir, prefix) return nil, err } - markerEncountered := marker == "" - output := make([]Object, 0) - - logger.Warnf("List: %v %v", prefix, marker) - for i := 0; i < len(listedObjects); i++ { - o := listedObjects[i] + logger.Debugf("List: %v %v", prefix, marker) + for _, o := range listedObjects { normalizedPath := normalizedObjectNameWithinZone(o) - if strings.HasPrefix(normalizedPath, prefix) { - if normalizedPath == marker { - markerEncountered = true - } - if markerEncountered { - logger.Warn(normalizedPath) - output = append(output, parseObjectMetadata(o)) - } + if !strings.HasPrefix(normalizedPath, prefix) || (marker != "" && normalizedPath <= marker) { + continue + } + output = append(output, parseObjectMetadata(o)) + if len(output) == int(limit) { + break } } From 445e2ef530f728b5e2e16191f1c0bbbf8d3fe46f Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 29 Mar 2024 21:38:45 +0100 Subject: [PATCH 27/28] fix: use correct path normalization for parsing metadata --- pkg/object/bunny.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index d208a6667b10..efb3e0706228 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -149,8 +149,13 @@ func normalizedObjectNameWithinZone(o bunnystorage.Object) string { // Parse Bunnystorage API Object to JuiceFS Object func parseObjectMetadata(object bunnystorage.Object) Object { lastChanged, _ := time.Parse("2006-01-02T15:04:05", object.LastChanged) + + key := normalizedObjectNameWithinZone(object) + if object.IsDirectory && !strings.HasSuffix(key, "/") { + key = key+"/" + } return &obj{ - normalizedObjectNameWithinZone(object), + key, int64(object.Length), lastChanged, object.IsDirectory, From dc24eddce5156dbf81733fc8bcb3355ba4fab4e8 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 29 Mar 2024 21:40:19 +0100 Subject: [PATCH 28/28] fix: remove incorrect deletion workaround --- pkg/object/bunny.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go index efb3e0706228..8abf61124ca0 100644 --- a/pkg/object/bunny.go +++ b/pkg/object/bunny.go @@ -76,16 +76,11 @@ func (b *bunnyClient) Put(key string, in io.Reader) error { // We implement a conditional retry here to try deleting the directory if the delete for a key of the passed name fails. // Deleting keys that do not exist are expected to not throw an error func (b *bunnyClient) Delete(key string) error { + logger.Warnf("Delete: %v", key) if err := b.client.Delete(key, false); err != nil { if err.Error() == "Not Found" { - // Retry delete the directory with the same name - if errDirectoryDelete := b.client.Delete(key, true); err != nil { - if err.Error() == "Not Found" { - return nil - } else { - return errDirectoryDelete - } - } + logger.Warnf("Failed to delete %v, %v", key, err) + return nil } return err }