diff --git a/docs/en/reference/how_to_set_up_object_storage.md b/docs/en/reference/how_to_set_up_object_storage.md index 9ea36e12a5e1..a89dfbe9072e 100644 --- a/docs/en/reference/how_to_set_up_object_storage.md +++ b/docs/en/reference/how_to_set_up_object_storage.md @@ -160,6 +160,7 @@ If you wish to use a storage system that is not listed, feel free to submit a re | [Storj DCS](#storj-dcs) | `s3` | | [Vultr Object Storage](#vultr-object-storage) | `s3` | | [Cloudflare R2](#r2) | `s3` | +| [Bunny Storage](#bunny) | `bunny` | | [Alibaba Cloud OSS](#alibaba-cloud-oss) | `oss` | | [Tencent Cloud COS](#tencent-cloud-cos) | `cos` | | [Huawei Cloud OBS](#huawei-cloud-obs) | `obs` | @@ -524,6 +525,22 @@ juicefs format \ Cloudflare R2 `ListObjects` API is not fully S3 compatible (result list is not sorted), so some features of JuiceFS do not work. For example, `juicefs gc`, `juicefs fsck`, `juicefs sync`, `juicefs destroy`. And when using `juicefs mount`, you need to disable [automatic-backup](../administration/metadata_dump_load.md#backup-automatically) function by adding `--backup-meta 0`. ::: +### Bunny Storage {#bunny} + +Bunny Storage offers a non-S3 compatible object storage with multiple performance tiers and many storage regions. It uses [it uses a custom API](https://docs.bunny.net/reference/storage-api). + +#### Usage + +Create a Storage Zone and use the Zone Name with the Hostname of the Location seperated by a dot as Bucket name and the `Write Password` as Secret Key. + +```shell +juicefs format \ + --storage bunny \ + --secret-key "write-password" \ + --bucket "https://uk.storage.bunnycdn.com/myzone" \ # https:/// + myjfs +``` + ### Alibaba Cloud OSS Please follow [this document](https://www.alibabacloud.com/help/doc-detail/125558.htm) to learn how to get access key and secret key. If you have already created [RAM role](https://www.alibabacloud.com/help/doc-detail/110376.htm) and assigned it to a VM instance, you could omit the options `--access-key` and `--secret-key`. diff --git a/go.mod b/go.mod index bdcc28661311..23c4b16ea580 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/goccy/go-json v0.10.2 github.com/gofrs/flock v0.8.1 github.com/google/btree v1.1.2 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.5.0 github.com/grafana/pyroscope-go v1.0.4 github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 github.com/hashicorp/consul/api v1.15.2 @@ -39,6 +39,7 @@ require ( github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 + github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10 github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/cli v1.24.2 @@ -55,7 +56,7 @@ require ( github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 github.com/qiniu/go-sdk/v7 v7.15.0 github.com/redis/go-redis/v9 v9.0.2 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/smartystreets/goconvey v1.7.2 github.com/stretchr/testify v1.8.4 github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 @@ -72,12 +73,12 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.uber.org/automaxprocs v1.5.2 go.uber.org/zap v1.20.0 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.17.0 + golang.org/x/crypto v0.19.0 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.2.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/sys v0.17.0 + golang.org/x/term v0.17.0 golang.org/x/text v0.14.0 google.golang.org/api v0.114.0 google.golang.org/protobuf v1.30.0 @@ -86,6 +87,13 @@ require ( xorm.io/xorm v1.0.7 ) +require ( + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/go-resty/resty/v2 v2.11.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.52.0 // indirect +) + require ( cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.19.1 // indirect @@ -161,9 +169,9 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid v1.3.1 // indirect - github.com/klauspost/cpuid/v2 v2.0.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/readahead v1.3.1 // indirect github.com/klauspost/reedsolomon v1.9.11 // indirect @@ -176,7 +184,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/md5-simd v1.1.1 // indirect github.com/minio/selfupdate v0.3.1 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/minio/simdjson-go v0.2.1 // indirect github.com/minio/sio v0.2.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index 15769cfb130e..9db75c476034 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee/go.mod github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -271,6 +273,8 @@ github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -358,8 +362,12 @@ github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -519,14 +527,19 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= @@ -555,6 +568,12 @@ github.com/ks3sdklib/aws-sdk-go v1.2.2 h1:kykh38h9jQ+NGRsn0lm8NAXn2RKB+DotZIyvJ5 github.com/ks3sdklib/aws-sdk-go v1.2.2/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240222214432-60c84bce3bf9 h1:D0B0I47sgXkXsYMVorBT04CgUVLXYS1xm9rEGL9nnTk= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240222214432-60c84bce3bf9/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240223162144-5045a4b99139 h1:02FHEfQzys+SrCsx/rtOcvo5c7hLhd7aCeB/OAcBrDg= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10-0.20240223162144-5045a4b99139/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10 h1:Vy8I4nGazW1QvwdIR3b/viHmBVFBf2i4RgR0dV0wJ/c= +github.com/l0wl3vel/bunny-storage-go-sdk v0.0.10/go.mod h1:2kvY9oZnsZR4QAvtkj8s7MuEl37dTARhQz7ICLpyD2M= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -614,6 +633,8 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/simdjson-go v0.2.1 h1:nxYlp4Qd0w2pwLlif00l5vTFL6PcNAKpyHq27/pageg= @@ -809,6 +830,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -885,8 +908,12 @@ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0= +github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/volcengine/ve-tos-golang-sdk/v2 v2.5.3 h1:sc7EfqfTjMJtPtx8vYUDIL9WmmJtmamMFYxWF467IGw= @@ -974,8 +1001,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1034,8 +1062,10 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1123,13 +1153,20 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1142,6 +1179,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go new file mode 100644 index 000000000000..8abf61124ca0 --- /dev/null +++ b/pkg/object/bunny.go @@ -0,0 +1,186 @@ +/* + * JuiceFS, Copyright 2024 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "bytes" + "fmt" + "io" + "math" + "net/url" + "os" + "path" + "strings" + "time" + + bunnystorage "github.com/l0wl3vel/bunny-storage-go-sdk" +) + +type bunnyClient struct { + DefaultObjectStorage + client *bunnystorage.Client + endpoint string +} + +// Description of the object storage. +func (b *bunnyClient) String() string { + return fmt.Sprintf("bunny://%v", b.endpoint) +} + +// Limits of the object storage. +func (b *bunnyClient) Limits() Limits { + return Limits{ + IsSupportMultipartUpload: false, + IsSupportUploadPartCopy: false, + } +} + +// Get the data for the given object specified by key. +func (b *bunnyClient) Get(key string, off int64, limit int64) (io.ReadCloser, error) { + if limit == -1 { + limit = math.MaxInt64 + } + body, err := b.client.DownloadPartial(key, off, limit+off-1) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(body)), nil +} + +// Put data read from a reader to an object specified by key. +func (b *bunnyClient) Put(key string, in io.Reader) error { + content, readErr := io.ReadAll(in) + if readErr != nil { + return readErr + } + return b.client.Upload(key, content, true) +} + +// Delete a object. +// Requires a conditional retry, since deleting a directory or file called foo/bar requires two different calls to the Bunny API, which JuiceFS does not do +// Deleting a directory requires a trailing slash in the key to delete, which JuiceFS does not add to the path, leading to test case failures. +// We implement a conditional retry here to try deleting the directory if the delete for a key of the passed name fails. +// Deleting keys that do not exist are expected to not throw an error +func (b *bunnyClient) Delete(key string) error { + logger.Warnf("Delete: %v", key) + if err := b.client.Delete(key, false); err != nil { + if err.Error() == "Not Found" { + logger.Warnf("Failed to delete %v, %v", key, err) + return nil + } + return err + } + return nil +} + +func (b *bunnyClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) { + if delimiter != "/" { + return nil, notSupported + } + var output []Object + var dir = prefix + if !strings.HasSuffix(dir, dirSuffix) { // If no Directory list in parent directory + dir = path.Dir(dir) + if !strings.HasSuffix(dir, dirSuffix) { + dir += dirSuffix + } + } else if marker == "" { // If Directory && no marker: Return prefix directory as well + parentPath := path.Dir(path.Dir(prefix)) + objects, err := b.client.List(parentPath+dirSuffix) + if err == nil { + for _, o := range objects { + logger.Warnf("%v == %v", normalizedObjectNameWithinZone(o), path.Dir(prefix)+dirSuffix) + if normalizedObjectNameWithinZone(o) == path.Dir(prefix) { + output = append(output, parseObjectMetadata(o)) + } + } + } + } + + listedObjects, err := b.client.List(dir) + if err != nil { + logger.Errorf("Unable to list objects in path %v with prefix %v", dir, prefix) + return nil, err + } + + logger.Debugf("List: %v %v", prefix, marker) + for _, o := range listedObjects { + normalizedPath := normalizedObjectNameWithinZone(o) + if !strings.HasPrefix(normalizedPath, prefix) || (marker != "" && normalizedPath <= marker) { + continue + } + output = append(output, parseObjectMetadata(o)) + if len(output) == int(limit) { + break + } + } + + return output, nil +} + +// The Object Path returned by the Bunny API contains the Storage Zone Name, which this function removes +func normalizedObjectNameWithinZone(o bunnystorage.Object) string { + normalizedPath := path.Join(o.Path, o.ObjectName) + if o.IsDirectory { + normalizedPath = normalizedPath + "/" // Append a trailing slash to allow deletion of directories + } + return strings.TrimPrefix(normalizedPath, "/"+o.StorageZoneName+"/") +} + +// Parse Bunnystorage API Object to JuiceFS Object +func parseObjectMetadata(object bunnystorage.Object) Object { + lastChanged, _ := time.Parse("2006-01-02T15:04:05", object.LastChanged) + + key := normalizedObjectNameWithinZone(object) + if object.IsDirectory && !strings.HasSuffix(key, "/") { + key = key+"/" + } + return &obj{ + key, + int64(object.Length), + lastChanged, + object.IsDirectory, + "", + } +} + +func (b *bunnyClient) Head(key string) (Object, error) { + object, err := b.client.Describe(key) + if err != nil { + if err.Error() == "Not Found" { + return nil, os.ErrNotExist + } + return nil, err + } + return parseObjectMetadata(object), nil +} + +func newBunny(endpoint, accessKey, password, token string) (ObjectStorage, error) { + endpoint_url, err := url.Parse(endpoint) + + if err != nil { + return nil, err + } + + client := bunnystorage.NewClient(*endpoint_url, password) + + return &bunnyClient{client: &client, endpoint: endpoint}, nil +} + +func init() { + Register("bunny", newBunny) +} diff --git a/pkg/object/object_storage_test.go b/pkg/object/object_storage_test.go index 68c9f44659f7..0fe3893e9340 100644 --- a/pkg/object/object_storage_test.go +++ b/pkg/object/object_storage_test.go @@ -1020,6 +1020,17 @@ func TestDragonfly(t *testing.T) { //skip mutate testStorage(t, dragonfly) } +func TestBunny(t *testing.T) { //skip mutate + if os.Getenv("BUNNY_ENDPOINT") == "" { + t.SkipNow() + } + bunny, err := newBunny(os.Getenv("BUNNY_ENDPOINT"), "", os.Getenv("BUNNY_SECRET_KEY"), "") + if err != nil { + t.Fatalf("create: %s", err) + } + testStorage(t, bunny) +} + func TestMain(m *testing.M) { if envFile := os.Getenv("JUICEFS_ENV_FILE_FOR_TEST"); envFile != "" { // schema: S3 AWS_ENDPOINT=xxxxx