From 036bc28ae066e13b914b8fad5b9c99cc7d68488a Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 17 Oct 2023 17:00:49 +0800 Subject: [PATCH 01/15] chore: unify json http client implementation --- internal/utils/api.go | 54 +++++---------------- internal/utils/deno.go | 2 +- internal/utils/http.go | 85 +++++++++++++++++++++++++++++++++ internal/utils/tenant/client.go | 61 +++++------------------ 4 files changed, 110 insertions(+), 92 deletions(-) create mode 100644 internal/utils/http.go diff --git a/internal/utils/api.go b/internal/utils/api.go index 4d046b098..534bf1d09 100644 --- a/internal/utils/api.go +++ b/internal/utils/api.go @@ -4,16 +4,13 @@ import ( "context" "crypto/tls" "encoding/json" - "errors" "fmt" - "io" "log" "net" "net/http" "net/http/httptrace" "net/textproto" "sync" - "time" "github.com/spf13/viper" supabase "github.com/supabase/cli/pkg/api" @@ -27,7 +24,6 @@ const ( var ( clientOnce sync.Once apiClient *supabase.ClientWithResponses - httpClient = http.Client{Timeout: 10 * time.Second} DNSResolver = EnumFlag{ Allowed: []string{DNS_GO_NATIVE, DNS_OVER_HTTPS}, @@ -57,30 +53,14 @@ func FallbackLookupIP(ctx context.Context, host string) ([]string, error) { return []string{host}, nil } // Ref: https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json - req, err := http.NewRequestWithContext(ctx, "GET", "https://1.1.1.1/dns-query?name="+host, nil) - if err != nil { - return nil, err - } - req.Header.Add("accept", "application/dns-json") - // Sends request - resp, err := httpClient.Do(req) + url := "https://1.1.1.1/dns-query?name=" + host + data, err := JsonResponse[dnsResponse](ctx, http.MethodGet, url, nil, func(ctx context.Context, req *http.Request) error { + req.Header.Add("accept", "application/dns-json") + return nil + }) if err != nil { return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) - if err != nil || len(body) == 0 { - body = []byte(fmt.Sprintf("status %d", resp.StatusCode)) - } - return nil, errors.New(string(body)) - } - // Parses response - var data dnsResponse - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&data); err != nil { - return nil, err - } // Look for first valid IP var resolved []string for _, answer := range data.Answer { @@ -96,25 +76,13 @@ func FallbackLookupIP(ctx context.Context, host string) ([]string, error) { func ResolveCNAME(ctx context.Context, host string) (string, error) { // Ref: https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://1.1.1.1/dns-query?name=%s&type=CNAME", host), nil) - if err != nil { - return "", fmt.Errorf("failed to initialize request: %w", err) - } - req.Header.Add("accept", "application/dns-json") - // Sends request - resp, err := httpClient.Do(req) + url := fmt.Sprintf("https://1.1.1.1/dns-query?name=%s&type=CNAME", host) + data, err := JsonResponse[dnsResponse](ctx, http.MethodGet, url, nil, func(ctx context.Context, req *http.Request) error { + req.Header.Add("accept", "application/dns-json") + return nil + }) if err != nil { - return "", fmt.Errorf("failed to execute resolution request: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("resolution response code was not 200: %s", resp.Status) - } - // Parses response - var data dnsResponse - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&data); err != nil { - return "", fmt.Errorf("failed to decode response: %w", err) + return "", err } // Look for first valid IP for _, answer := range data.Answer { diff --git a/internal/utils/deno.go b/internal/utils/deno.go index 2970b0dc9..157edc6b8 100644 --- a/internal/utils/deno.go +++ b/internal/utils/deno.go @@ -81,7 +81,7 @@ func InstallOrUpgradeDeno(ctx context.Context, fsys afero.Fs) error { // 2. Download & install Deno binary. { assetUrl := fmt.Sprintf("https://github.com/%s/releases/download/v%s/%s", assetRepo, DenoVersion, assetFilename) - req, err := http.NewRequestWithContext(ctx, "GET", assetUrl, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetUrl, nil) if err != nil { return err } diff --git a/internal/utils/http.go b/internal/utils/http.go new file mode 100644 index 000000000..91fa0270c --- /dev/null +++ b/internal/utils/http.go @@ -0,0 +1,85 @@ +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + openapi "github.com/supabase/cli/pkg/api" +) + +var httpClient = http.Client{Timeout: 10 * time.Second} + +func JsonResponse[T any](ctx context.Context, method, url string, reqBody any, reqEditors ...openapi.RequestEditorFn) (*T, error) { + var body bytes.Buffer + if reqBody != nil { + enc := json.NewEncoder(&body) + if err := enc.Encode(reqBody); err != nil { + return nil, err + } + reqEditors = append(reqEditors, func(ctx context.Context, req *http.Request) error { + req.Header.Set("Content-Type", "application/json") + return nil + }) + } + // Creates request + req, err := http.NewRequestWithContext(ctx, method, url, &body) + if err != nil { + return nil, err + } + for _, edit := range reqEditors { + if err := edit(ctx, req); err != nil { + return nil, err + } + } + // Sends request + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("Error status %d: %s", resp.StatusCode, data) + } + // Parses response + var data T + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&data); err != nil { + return nil, err + } + return &data, nil +} + +func TextResponse(ctx context.Context, method, url string, body io.Reader, reqEditors ...openapi.RequestEditorFn) (string, error) { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return "", err + } + for _, edit := range reqEditors { + if err := edit(ctx, req); err != nil { + return "", err + } + } + // Sends request + resp, err := httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("Error status %d: %s", resp.StatusCode, body) + } + return string(data), nil +} diff --git a/internal/utils/tenant/client.go b/internal/utils/tenant/client.go index fea6dc2e8..0b7d24813 100644 --- a/internal/utils/tenant/client.go +++ b/internal/utils/tenant/client.go @@ -2,10 +2,8 @@ package tenant import ( "context" - "encoding/json" "errors" "fmt" - "io" "net/http" "sync" @@ -57,55 +55,22 @@ func GetApiKeys(ctx context.Context, projectRef string) (ApiKey, error) { } func GetJsonResponse[T any](ctx context.Context, url, apiKey string) (*T, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - if len(apiKey) > 0 { + return utils.JsonResponse[T](ctx, http.MethodGet, url, nil, func(ctx context.Context, req *http.Request) error { req.Header.Add("apikey", apiKey) - } - // Sends request - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return nil, fmt.Errorf("Error status %d: %s", resp.StatusCode, body) - } - // Parses response - var data T - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&data); err != nil { - return nil, err - } - return &data, nil + return nil + }) +} + +func JsonResponseWithBearer[T any](ctx context.Context, method, url, token string, reqBody any) (*T, error) { + return utils.JsonResponse[T](ctx, method, url, reqBody, func(ctx context.Context, req *http.Request) error { + req.Header.Add("Authorization", "Bearer "+token) + return nil + }) } func GetTextResponse(ctx context.Context, url, apiKey string) (string, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return "", err - } - if len(apiKey) > 0 { + return utils.TextResponse(ctx, http.MethodGet, url, nil, func(ctx context.Context, req *http.Request) error { req.Header.Add("apikey", apiKey) - } - // Sends request - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("Error status %d: %s", resp.StatusCode, body) - } - return string(body), nil + return nil + }) } From 765bb6732b31b91528234a96c46ceccdfc344646 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 17 Oct 2023 17:04:16 +0800 Subject: [PATCH 02/15] feat: implement storage api client --- internal/storage/client/buckets.go | 72 +++++++++++ internal/storage/client/objects.go | 201 +++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 internal/storage/client/buckets.go create mode 100644 internal/storage/client/objects.go diff --git a/internal/storage/client/buckets.go b/internal/storage/client/buckets.go new file mode 100644 index 000000000..2a1703bfc --- /dev/null +++ b/internal/storage/client/buckets.go @@ -0,0 +1,72 @@ +package client + +import ( + "context" + "fmt" + "net/http" + + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/internal/utils/tenant" +) + +type BucketResponse struct { + Id string `json:"id"` // "test" + Name string `json:"name"` // "test" + Owner string `json:"owner"` // "" + Public bool `json:"public"` // true + FileSizeLimit *int `json:"file_size_limit"` // null + AllowedMimeTypes *string `json:"allowed_mime_types"` // null + CreatedAt string `json:"created_at"` // "2023-10-13T17:48:58.491Z" + UpdatedAt string `json:"updated_at"` // "2023-10-13T17:48:58.491Z" +} + +func ListStorageBuckets(ctx context.Context, projectRef string) ([]BucketResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/bucket", utils.GetSupabaseHost(projectRef)) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + data, err := tenant.JsonResponseWithBearer[[]BucketResponse](ctx, http.MethodGet, url, apiKey.ServiceRole, nil) + if err != nil { + return nil, err + } + return *data, nil +} + +type CreateBucketRequest struct { + Id string `json:"id"` // "string", + Name string `json:"name"` // "string", + Public bool `json:"public,omitempty"` // false, + FileSizeLimit int `json:"file_size_limit,omitempty"` // 0, + AllowedMimeTypes []string `json:"allowed_mime_types,omitempty"` // ["string"] +} + +type CreateBucketResponse struct { + Name string `json:"name"` +} + +func CreateStorageBucket(ctx context.Context, projectRef, bucketName string) (*CreateBucketResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/bucket", utils.GetSupabaseHost(projectRef)) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + body := CreateBucketRequest{ + Id: bucketName, + Name: bucketName, + } + return tenant.JsonResponseWithBearer[CreateBucketResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, body) +} + +type DeleteBucketResponse struct { + Message string `json:"message"` +} + +func DeleteStorageBucket(ctx context.Context, projectRef, bucketId string) (*DeleteBucketResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/bucket/%s", utils.GetSupabaseHost(projectRef), bucketId) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + return tenant.JsonResponseWithBearer[DeleteBucketResponse](ctx, http.MethodDelete, url, apiKey.ServiceRole, nil) +} diff --git a/internal/storage/client/objects.go b/internal/storage/client/objects.go new file mode 100644 index 000000000..626b2d7cc --- /dev/null +++ b/internal/storage/client/objects.go @@ -0,0 +1,201 @@ +package client + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/internal/utils/tenant" +) + +const PAGE_LIMIT = 100 + +type ListObjectsQuery struct { + Prefix string `json:"prefix"` + Search string `json:"search"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type ObjectResponse struct { + Name string `json:"name"` // "abstract.pdf" + Id *string `json:"id"` // "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9" + UpdatedAt *string `json:"updated_at"` // "2023-10-13T18:08:22.068Z" + CreatedAt *string `json:"created_at"` // "2023-10-13T18:08:22.068Z" + LastAccessedAt *string `json:"last_accessed_at"` // "2023-10-13T18:08:22.068Z" + Metadata *ObjectMetadata `json:"metadata"` // null +} + +type ObjectMetadata struct { + ETag string `json:"eTag"` // "\"887ea9be3c68e6f2fca7fd2d7c77d8fe\"" + Size int `json:"size"` // 82702 + Mimetype string `json:"mimetype"` // "application/pdf" + CacheControl string `json:"cacheControl"` // "max-age=3600" + LastModified string `json:"lastModified"` // "2023-10-13T18:08:22.000Z" + ContentLength int `json:"contentLength"` // 82702 + HttpStatusCode int `json:"httpStatusCode"` // 200 +} + +func ListStorageObjects(ctx context.Context, projectRef, bucket, prefix string, page int) ([]ObjectResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/object/list/%s", utils.GetSupabaseHost(projectRef), bucket) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + dir, name := path.Split(prefix) + query := ListObjectsQuery{ + Prefix: dir, + Search: name, + Limit: PAGE_LIMIT, + Offset: PAGE_LIMIT * page, + } + data, err := tenant.JsonResponseWithBearer[[]ObjectResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, query) + if err != nil { + return nil, err + } + return *data, nil +} + +func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return err + } + url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) + f, err := fsys.Open(localPath) + if err != nil { + return err + } + defer f.Close() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, f) + if err != nil { + return err + } + req.Header.Add("Authorization", "Bearer "+apiKey.ServiceRole) + // Sends request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("Error status %d: %s", resp.StatusCode, body) + } + return nil +} + +func DownloadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return err + } + url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.Header.Add("Authorization", "Bearer "+apiKey.ServiceRole) + // Sends request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("Error status %d: %s", resp.StatusCode, body) + } + // Streams to file + f, err := fsys.OpenFile(localPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, resp.Body) + return err +} + +type MoveObjectRequest struct { + BucketId string `json:"bucketId"` + SourceKey string `json:"sourceKey"` + DestinationKey string `json:"destinationKey"` +} + +type MoveObjectResponse = DeleteBucketResponse + +func MoveStorageObject(ctx context.Context, projectRef, bucketId, srcPath, dstPath string) (*MoveObjectResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/object/move", utils.GetSupabaseHost(projectRef)) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + body := MoveObjectRequest{ + BucketId: bucketId, + SourceKey: srcPath, + DestinationKey: dstPath, + } + return tenant.JsonResponseWithBearer[MoveObjectResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, body) +} + +type CopyObjectRequest = MoveObjectRequest + +type CopyObjectResponse struct { + Key string `json:"key"` +} + +func CopyStorageObject(ctx context.Context, projectRef, bucketId, srcPath, dstPath string) (*CopyObjectResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/object/copy", utils.GetSupabaseHost(projectRef)) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + body := CopyObjectRequest{ + BucketId: bucketId, + SourceKey: srcPath, + DestinationKey: dstPath, + } + return tenant.JsonResponseWithBearer[CopyObjectResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, body) +} + +type DeleteObjectsRequest struct { + Prefixes []string `json:"prefixes"` +} + +type DeleteObjectsResponse struct { + BucketId string `json:"bucket_id"` // "private" + Owner string `json:"owner"` // "" + OwnerId string `json:"owner_id"` // "" + Version string `json:"version"` // "cf5c5c53-ee73-4806-84e3-7d92c954b436" + Name string `json:"name"` // "abstract.pdf" + Id string `json:"id"` // "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9" + UpdatedAt string `json:"updated_at"` // "2023-10-13T18:08:22.068Z" + CreatedAt string `json:"created_at"` // "2023-10-13T18:08:22.068Z" + LastAccessedAt string `json:"last_accessed_at"` // "2023-10-13T18:08:22.068Z" + Metadata ObjectMetadata `json:"metadata"` // null +} + +func DeleteStorageObjects(ctx context.Context, projectRef, bucket string, prefixes []string) ([]DeleteObjectsResponse, error) { + url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), bucket) + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return nil, err + } + body := DeleteObjectsRequest{Prefixes: prefixes} + data, err := tenant.JsonResponseWithBearer[[]DeleteObjectsResponse](ctx, http.MethodDelete, url, apiKey.ServiceRole, body) + if err != nil { + return nil, err + } + return *data, nil +} From d44445cc72e7ba856f650b94b9faad484f5dc816 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 17 Oct 2023 23:25:48 +0800 Subject: [PATCH 03/15] feat: implement storage management commands --- cmd/storage.go | 81 ++++++++++++++++++++ internal/storage/cp/cp.go | 156 ++++++++++++++++++++++++++++++++++++++ internal/storage/ls/ls.go | 143 ++++++++++++++++++++++++++++++++++ internal/storage/mv/mv.go | 73 ++++++++++++++++++ internal/storage/rm/rm.go | 117 ++++++++++++++++++++++++++++ 5 files changed, 570 insertions(+) create mode 100644 cmd/storage.go create mode 100644 internal/storage/cp/cp.go create mode 100644 internal/storage/ls/ls.go create mode 100644 internal/storage/mv/mv.go create mode 100644 internal/storage/rm/rm.go diff --git a/cmd/storage.go b/cmd/storage.go new file mode 100644 index 000000000..43359bdb5 --- /dev/null +++ b/cmd/storage.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "github.com/spf13/afero" + "github.com/spf13/cobra" + "github.com/supabase/cli/internal/storage/cp" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/storage/mv" + "github.com/supabase/cli/internal/storage/rm" +) + +var ( + storageCmd = &cobra.Command{ + GroupID: groupManagementAPI, + Use: "storage", + Short: "Manage Supabase Storage objects", + } + + recursive bool + + lsCmd = &cobra.Command{ + Use: "ls [prefix]", + Example: "ls ss:///bucket/docs", + Short: "List objects by path prefix", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + objectPath := ls.STORAGE_SCHEME + ":///" + if len(args) > 0 { + objectPath = args[0] + } + return ls.Run(cmd.Context(), objectPath, recursive, afero.NewOsFs()) + }, + } + + cpCmd = &cobra.Command{ + Use: "cp ", + Example: `cp readme.md ss:///bucket +cp -r docs ss:///bucket/docs +cp -r ss:///bucket/docs . +`, + Short: "Copy objects by from src to dst path", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return cp.Run(cmd.Context(), args[0], args[1], recursive, afero.NewOsFs()) + }, + } + + mvCmd = &cobra.Command{ + Use: "mv ", + Short: "Move objects by from src to dst path", + Example: "mv -r ss:///bucket/docs ss:///bucket/www/docs", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return mv.Run(cmd.Context(), args[0], args[1], recursive, afero.NewOsFs()) + }, + } + + rmCmd = &cobra.Command{ + Use: "rm ...", + Short: "Remove objects by file path", + Example: `rm -r ss:///bucket/docs +rm ss:///bucket/docs/example.md ss:///bucket/readme.md +`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return rm.Run(cmd.Context(), args, recursive, afero.NewOsFs()) + }, + } +) + +func init() { + lsCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively list a directory.") + storageCmd.AddCommand(lsCmd) + cpCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively copy a directory.") + storageCmd.AddCommand(cpCmd) + rmCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively move a directory.") + storageCmd.AddCommand(rmCmd) + mvCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively remove a directory.") + storageCmd.AddCommand(mvCmd) + rootCmd.AddCommand(storageCmd) +} diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go new file mode 100644 index 000000000..4dcb6e6ba --- /dev/null +++ b/internal/storage/cp/cp.go @@ -0,0 +1,156 @@ +package cp + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { + srcParsed, err := url.Parse(src) + if err != nil { + return err + } + dstParsed, err := url.Parse(dst) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + if strings.ToLower(srcParsed.Scheme) == ls.STORAGE_SCHEME && dstParsed.Scheme == "" { + if recursive { + return DownloadStorageObjectAll(ctx, projectRef, srcParsed.Path, dst, fsys) + } + // TODO: Check if destination is a directory + return client.DownloadStorageObject(ctx, projectRef, srcParsed.Path, dst, fsys) + } else if srcParsed.Scheme == "" && strings.ToLower(dstParsed.Scheme) == ls.STORAGE_SCHEME { + if recursive { + return UploadStorageObjectAll(ctx, projectRef, dstParsed.Path, src, fsys) + } + // TODO: Check if destination is a directory + return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys) + } + return errors.New("Unsupported operation") +} + +func DownloadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + remotePath = path.Join("/", remotePath) + if fi, err := fsys.Stat(localPath); err == nil && fi.IsDir() { + localPath = filepath.Join(localPath, path.Base(remotePath)) + } + if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(localPath)); err != nil { + return err + } + if !IsDir(remotePath) { + fmt.Fprintln(os.Stderr, "Downloading:", remotePath, "=>", localPath) + if err := client.DownloadStorageObject(ctx, projectRef, remotePath, localPath, fsys); err != nil && strings.Contains(err.Error(), `"error":"Not Found"`) { + // Retry downloading as directory + remotePath += "/" + } else { + return err + } + } + queue := make([]string, 0) + queue = append(queue, remotePath) + for len(queue) > 0 { + dirPath := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ls.ListStoragePaths(ctx, projectRef, dirPath) + if err != nil { + return err + } + if strings.Count(dirPath, "/") > 2 && len(paths) == 0 { + return errors.New("Object not found: " + dirPath) + } + for _, objectName := range paths { + objectPath := dirPath + objectName + relPath := strings.TrimPrefix(objectPath, remotePath) + dstPath := filepath.Join(localPath, filepath.FromSlash(relPath)) + fmt.Fprintln(os.Stderr, "Downloading:", objectPath, "=>", dstPath) + if strings.HasSuffix(objectName, "/") { + if err := utils.MkdirIfNotExistFS(fsys, dstPath); err != nil { + return err + } + queue = append(queue, objectPath) + continue + } + if err := client.DownloadStorageObject(ctx, projectRef, objectPath, dstPath, fsys); err != nil { + return err + } + } + } + return nil +} + +func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + noSlash := strings.TrimSuffix(remotePath, "/") + paths, err := ls.ListStoragePaths(ctx, projectRef, noSlash) + if err != nil { + return err + } + // Check if directory exists on remote + dirExists := false + fileExists := false + for _, p := range paths { + if p == path.Base(noSlash) { + fileExists = true + } + if p == path.Base(noSlash)+"/" { + dirExists = true + } + } + baseName := filepath.Base(localPath) + return afero.Walk(fsys, localPath, func(filePath string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + relPath, err := filepath.Rel(localPath, filePath) + if err != nil { + return err + } + dstPath := remotePath + // Copying single file + if relPath == "." { + if IsDir(dstPath) || (dirExists && !fileExists) { + dstPath = path.Join(dstPath, info.Name()) + } + } else { + if baseName != "." && (dirExists || len(noSlash) == 0) { + dstPath = path.Join(dstPath, baseName) + } + dstPath = path.Join(dstPath, relPath) + } + fmt.Fprintln(os.Stderr, "Uploading:", filePath, "=>", dstPath) + err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys) + if err != nil && strings.Contains(err.Error(), `"error":"Bucket not found"`) { + // Retry after creating bucket + if bucket, prefix := ls.SplitBucketPrefix(dstPath); len(prefix) > 0 { + if _, err := client.CreateStorageBucket(ctx, projectRef, bucket); err != nil { + return err + } + err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys) + } + } + return err + }) +} + +func IsDir(objectPath string) bool { + return len(objectPath) == 0 || strings.HasSuffix(objectPath, "/") +} diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go new file mode 100644 index 000000000..7d3b776d8 --- /dev/null +++ b/internal/storage/ls/ls.go @@ -0,0 +1,143 @@ +package ls + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/utils" +) + +const STORAGE_SCHEME = "ss" + +func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) error { + remotePath, err := ParseStorageURL(objectPath) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + paths, err := ListStoragePaths(ctx, projectRef, remotePath) + if err != nil { + return err + } + if recursive { + basePath := remotePath + if !strings.HasSuffix(remotePath, "/") { + basePath, _ = path.Split(remotePath) + } + var result []string + for i := len(paths) - 1; i >= 0; i-- { + name := paths[i] + if !strings.HasSuffix(name, "/") { + result = append(result, name) + continue + } + dirPath := basePath + name + children, err := ListStoragePathsAll(ctx, projectRef, dirPath) + if err != nil { + return err + } + result = append(result, children...) + } + paths = result + } + if len(paths) > 0 { + fmt.Println(strings.Join(paths, "\n")) + } + return nil +} + +func ParseStorageURL(objectPath string) (string, error) { + parsed, err := url.Parse(objectPath) + if err != nil { + return "", err + } + if strings.ToLower(parsed.Scheme) != STORAGE_SCHEME || len(parsed.Path) == 0 || len(parsed.Host) > 0 { + return "", errors.New("URL must match pattern ss:///bucket/prefix") + } + return parsed.Path, nil +} + +func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]string, error) { + var result []string + bucket, prefix := SplitBucketPrefix(remotePath) + if len(bucket) == 0 || (len(prefix) == 0 && !strings.HasSuffix(remotePath, "/")) { + buckets, err := client.ListStorageBuckets(ctx, projectRef) + if err != nil { + return nil, err + } + for _, b := range buckets { + if strings.HasPrefix(b.Name, bucket) { + result = append(result, b.Name+"/") + } + } + } else { + pages := 1 + for i := 0; i < pages; i++ { + objects, err := client.ListStorageObjects(ctx, projectRef, bucket, prefix, i) + if err != nil { + return nil, err + } + for _, o := range objects { + name := o.Name + if o.Id == nil { + name += "/" + } + result = append(result, name) + } + if len(objects) == client.PAGE_LIMIT { + // TODO: show interactive prompt? + fmt.Fprintln(os.Stderr, "Loading page:", pages) + pages++ + } + } + } + return result, nil +} + +func SplitBucketPrefix(objectPath string) (string, string) { + if objectPath == "" || objectPath == "/" { + return "", "" + } + sep := strings.IndexByte(objectPath[1:], '/') + if sep < 0 { + return objectPath[1:], "" + } + return objectPath[1 : sep+1], objectPath[sep+2:] +} + +// Expects remotePath to be terminated by "/" +func ListStoragePathsAll(ctx context.Context, projectRef, remotePath string) ([]string, error) { + var result []string + queue := make([]string, 0) + queue = append(queue, remotePath) + for len(queue) > 0 { + dirPath := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ListStoragePaths(ctx, projectRef, dirPath) + if err != nil { + return result, err + } + if len(paths) == 0 { + result = append(result, dirPath) + } + for _, objectName := range paths { + objectPath := dirPath + objectName + if strings.HasSuffix(objectName, "/") { + queue = append(queue, objectPath) + } else { + result = append(result, objectPath) + } + } + } + return result, nil +} diff --git a/internal/storage/mv/mv.go b/internal/storage/mv/mv.go new file mode 100644 index 000000000..910efead3 --- /dev/null +++ b/internal/storage/mv/mv.go @@ -0,0 +1,73 @@ +package mv + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { + srcParsed, err := ls.ParseStorageURL(src) + if err != nil { + return err + } + dstParsed, err := ls.ParseStorageURL(dst) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + srcBucket, srcPrefix := ls.SplitBucketPrefix(srcParsed) + dstBucket, dstPrefix := ls.SplitBucketPrefix(dstParsed) + if srcBucket != dstBucket { + return errors.New("Moving between buckets is unsupported") + } + fmt.Fprintln(os.Stderr, "Moving object:", srcParsed, "=>", dstParsed) + data, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, dstPrefix) + if err == nil { + fmt.Fprintln(os.Stderr, data.Message) + } else if strings.Contains(err.Error(), `"error":"not_found"`) && recursive { + return MoveStorageObjectAll(ctx, projectRef, srcParsed+"/", dstParsed) + } + return err +} + +// Expects srcPath to be terminated by "/" +func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath string) error { + _, dstPrefix := ls.SplitBucketPrefix(dstPath) + queue := make([]string, 0) + queue = append(queue, srcPath) + for len(queue) > 0 { + dirPath := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ls.ListStoragePaths(ctx, projectRef, dirPath) + if err != nil { + return err + } + for _, objectName := range paths { + objectPath := dirPath + objectName + if strings.HasSuffix(objectName, "/") { + queue = append(queue, objectPath) + continue + } + relPath := strings.TrimPrefix(objectPath, srcPath) + srcBucket, srcPrefix := ls.SplitBucketPrefix(objectPath) + absPath := path.Join(dstPrefix, relPath) + fmt.Fprintln(os.Stderr, "Moving object:", objectPath, "=>", path.Join(dstPath, relPath)) + if _, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, absPath); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/storage/rm/rm.go b/internal/storage/rm/rm.go new file mode 100644 index 000000000..7802e7f38 --- /dev/null +++ b/internal/storage/rm/rm.go @@ -0,0 +1,117 @@ +package rm + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/cp" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +type PrefixGroup struct { + Bucket string + Prefixes []string +} + +func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) error { + // Group paths by buckets + groups := map[string][]string{} + for _, objectPath := range paths { + remotePath, err := ls.ParseStorageURL(objectPath) + if err != nil { + return err + } + bucket, prefix := ls.SplitBucketPrefix(remotePath) + // Ignore attempts to delete all buckets + if len(bucket) == 0 { + return errors.New("You must specify a bucket to delete.") + } + if cp.IsDir(prefix) && !recursive { + return errors.New("You must specify -r flag to delete directories.") + } + groups[bucket] = append(groups[bucket], prefix) + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + for bucket, prefixes := range groups { + if utils.SliceContains(prefixes, "") { + fmt.Fprintln(os.Stderr, "Deleting bucket:", bucket) + if err := RemoveStoragePathAll(ctx, projectRef, bucket, ""); err != nil { + return err + } + if data, err := client.DeleteStorageBucket(ctx, projectRef, bucket); err == nil { + fmt.Fprintln(os.Stderr, data.Message) + } else if !strings.Contains(err.Error(), `"error":"Bucket not found"`) { + return err + } else { + fmt.Fprintln(os.Stderr, "Bucket not found") + } + continue + } + fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) + removed, err := client.DeleteStorageObjects(ctx, projectRef, bucket, prefixes) + if err != nil { + return err + } + if !recursive { + if len(removed) == 0 { + utils.CmdSuggestion = "You must specify -r flag to delete directories." + return errors.New("Object not found") + } + continue + } + set := map[string]struct{}{} + for _, object := range removed { + set[object.Name] = struct{}{} + } + for _, prefix := range prefixes { + if _, ok := set[prefix]; !ok { + if err := RemoveStoragePathAll(ctx, projectRef, bucket, prefix+"/"); err != nil { + return err + } + } + } + } + return nil +} + +// Expects prefix to be terminated by "/" +func RemoveStoragePathAll(ctx context.Context, projectRef, bucket, prefix string) error { + queue := make([]string, 0) + queue = append(queue, prefix) + for len(queue) > 0 { + dirPrefix := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ls.ListStoragePaths(ctx, projectRef, fmt.Sprintf("/%s/%s", bucket, dirPrefix)) + if err != nil { + return err + } + if len(paths) == 0 { + return errors.New("Object not found") + } + var files []string + for _, objectName := range paths { + objectPrefix := dirPrefix + objectName + if strings.HasSuffix(objectName, "/") { + queue = append(queue, objectPrefix) + } else { + files = append(files, objectPrefix) + } + } + if len(files) > 0 { + fmt.Fprintln(os.Stderr, "Deleting objects:", files) + if _, err := client.DeleteStorageObjects(ctx, projectRef, bucket, files); err != nil { + return err + } + } + } + return nil +} From 51c742bb4b98137682e683159010f57a9e9bf703 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 18 Oct 2023 17:32:32 +0800 Subject: [PATCH 04/15] fix: add storage ls tests --- internal/storage/ls/ls.go | 126 +++++---- internal/storage/ls/ls_test.go | 498 +++++++++++++++++++++++++++++++++ internal/utils/misc.go | 4 + 3 files changed, 577 insertions(+), 51 deletions(-) create mode 100644 internal/storage/ls/ls_test.go diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go index 7d3b776d8..5c4a1f7cd 100644 --- a/internal/storage/ls/ls.go +++ b/internal/storage/ls/ls.go @@ -25,59 +25,50 @@ func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) if err != nil { return err } - paths, err := ListStoragePaths(ctx, projectRef, remotePath) - if err != nil { - return err + callback := func(objectPath string) error { + fmt.Println(objectPath) + return nil } if recursive { - basePath := remotePath - if !strings.HasSuffix(remotePath, "/") { - basePath, _ = path.Split(remotePath) - } - var result []string - for i := len(paths) - 1; i >= 0; i-- { - name := paths[i] - if !strings.HasSuffix(name, "/") { - result = append(result, name) - continue - } - dirPath := basePath + name - children, err := ListStoragePathsAll(ctx, projectRef, dirPath) - if err != nil { - return err - } - result = append(result, children...) - } - paths = result + return IterateStoragePathsAll(ctx, projectRef, remotePath, callback) } - if len(paths) > 0 { - fmt.Println(strings.Join(paths, "\n")) - } - return nil + return IterateStoragePaths(ctx, projectRef, remotePath, callback) } +var errInvalidURL = errors.New("URL must match pattern ss:///bucket/prefix") + func ParseStorageURL(objectPath string) (string, error) { parsed, err := url.Parse(objectPath) if err != nil { return "", err } if strings.ToLower(parsed.Scheme) != STORAGE_SCHEME || len(parsed.Path) == 0 || len(parsed.Host) > 0 { - return "", errors.New("URL must match pattern ss:///bucket/prefix") + return "", errInvalidURL } return parsed.Path, nil } func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]string, error) { var result []string + err := IterateStoragePaths(ctx, projectRef, remotePath, func(objectName string) error { + result = append(result, objectName) + return nil + }) + return result, err +} + +func IterateStoragePaths(ctx context.Context, projectRef, remotePath string, callback func(objectName string) error) error { bucket, prefix := SplitBucketPrefix(remotePath) if len(bucket) == 0 || (len(prefix) == 0 && !strings.HasSuffix(remotePath, "/")) { buckets, err := client.ListStorageBuckets(ctx, projectRef) if err != nil { - return nil, err + return err } for _, b := range buckets { if strings.HasPrefix(b.Name, bucket) { - result = append(result, b.Name+"/") + if err := callback(b.Name + "/"); err != nil { + return err + } } } } else { @@ -85,14 +76,16 @@ func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]str for i := 0; i < pages; i++ { objects, err := client.ListStorageObjects(ctx, projectRef, bucket, prefix, i) if err != nil { - return nil, err + return err } for _, o := range objects { name := o.Name if o.Id == nil { name += "/" } - result = append(result, name) + if err := callback(name); err != nil { + return err + } } if len(objects) == client.PAGE_LIMIT { // TODO: show interactive prompt? @@ -101,43 +94,74 @@ func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]str } } } - return result, nil + return nil } func SplitBucketPrefix(objectPath string) (string, string) { if objectPath == "" || objectPath == "/" { return "", "" } - sep := strings.IndexByte(objectPath[1:], '/') + start := 0 + if objectPath[0] == '/' { + start = 1 + } + sep := strings.IndexByte(objectPath[start:], '/') if sep < 0 { - return objectPath[1:], "" + return objectPath[start:], "" } - return objectPath[1 : sep+1], objectPath[sep+2:] + return objectPath[start : sep+start], objectPath[sep+start+1:] } // Expects remotePath to be terminated by "/" func ListStoragePathsAll(ctx context.Context, projectRef, remotePath string) ([]string, error) { var result []string - queue := make([]string, 0) - queue = append(queue, remotePath) - for len(queue) > 0 { - dirPath := queue[len(queue)-1] - queue = queue[:len(queue)-1] - paths, err := ListStoragePaths(ctx, projectRef, dirPath) - if err != nil { - return result, err - } - if len(paths) == 0 { - result = append(result, dirPath) + err := IterateStoragePathsAll(ctx, projectRef, remotePath, func(objectPath string) error { + result = append(result, objectPath) + return nil + }) + return result, err +} + +func IterateStoragePathsAll(ctx context.Context, projectRef, remotePath string, callback func(objectPath string) error) error { + basePath := remotePath + if !strings.HasSuffix(remotePath, "/") { + basePath, _ = path.Split(remotePath) + } + // BFS so we can list paths in increasing depth + dirQueue := make([]string, 0) + // We don't know if user passed in a directory or file, so query storage first. + if err := IterateStoragePaths(ctx, projectRef, remotePath, func(objectName string) error { + objectPath := basePath + objectName + if strings.HasSuffix(objectName, "/") { + dirQueue = append(dirQueue, objectPath) + return nil } - for _, objectName := range paths { + return callback(objectPath) + }); err != nil { + return err + } + for len(dirQueue) > 0 { + dirPath := dirQueue[len(dirQueue)-1] + dirQueue = dirQueue[:len(dirQueue)-1] + empty := true + if err := IterateStoragePaths(ctx, projectRef, dirPath, func(objectName string) error { + empty = false objectPath := dirPath + objectName if strings.HasSuffix(objectName, "/") { - queue = append(queue, objectPath) - } else { - result = append(result, objectPath) + dirQueue = append(dirQueue, objectPath) + return nil + } + return callback(objectPath) + }); err != nil { + return err + } + // Also report empty buckets + bucket, prefix := SplitBucketPrefix(dirPath) + if empty && len(prefix) == 0 { + if err := callback(bucket + "/"); err != nil { + return err } } } - return result, nil + return nil } diff --git a/internal/storage/ls/ls_test.go b/internal/storage/ls/ls_test.go new file mode 100644 index 000000000..77148d733 --- /dev/null +++ b/internal/storage/ls/ls_test.go @@ -0,0 +1,498 @@ +package ls + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +func TestStorageLS(t *testing.T) { + t.Run("lists buckets", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "ss:///", false, fsys) + // Check error + assert.NoError(t, err) + }) + + t.Run("throws error on invalid URL", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "", false, fsys) + // Check error + assert.ErrorIs(t, err, errInvalidURL) + }) + + t.Run("throws error on invalid project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("lists objects recursive", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := Run(context.Background(), "ss:///", true, fsys) + // Check error + assert.NoError(t, err) + }) +} + +func TestListStoragePaths(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("lists bucket paths by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "te") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"test/"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on bucket service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("lists object paths by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "folder", + }, { + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "bucket/") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"folder/", "abstract.pdf"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on object service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "bucket/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("lists object paths with pagination", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + expected := make([]string, client.PAGE_LIMIT) + resp := make([]client.ObjectResponse, client.PAGE_LIMIT) + for i := 0; i < len(resp); i++ { + resp[i] = client.ObjectResponse{Name: fmt.Sprintf("dir_%d", i)} + expected[i] = resp[i].Name + "/" + } + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "dir", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON(resp) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "dir", + Limit: client.PAGE_LIMIT, + Offset: client.PAGE_LIMIT, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "/bucket/dir") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, expected, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestListStoragePathsAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("lists nested object paths", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List buckets + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + // List folders + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "folder", + }}) + // List files + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "folder/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + // Run test + paths, err := ListStoragePathsAll(context.Background(), projectRef, "") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"private/folder/abstract.pdf", "test/"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("returns partial result on error", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List folders + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "error", + }, { + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "empty/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "error/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePathsAll(context.Background(), projectRef, "private/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.ElementsMatch(t, []string{"private/abstract.pdf"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestSplitBucketPrefix(t *testing.T) { + t.Run("splits empty path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits root path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits no slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits prefix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits suffix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits file path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/name.png") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/name.png") + }) + + t.Run("splits dir path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/") + }) +} + +func TestParseStorageURL(t *testing.T) { + t.Run("parses valid url", func(t *testing.T) { + path, err := ParseStorageURL("ss:///bucket/folder/name.png") + assert.NoError(t, err) + assert.Equal(t, path, "/bucket/folder/name.png") + }) + + t.Run("throws error on invalid host", func(t *testing.T) { + path, err := ParseStorageURL("ss://bucket") + assert.ErrorIs(t, err, errInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on missing path", func(t *testing.T) { + path, err := ParseStorageURL("ss:") + assert.ErrorIs(t, err, errInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid scheme", func(t *testing.T) { + path, err := ParseStorageURL(".") + assert.ErrorIs(t, err, errInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid url", func(t *testing.T) { + path, err := ParseStorageURL(":") + assert.ErrorContains(t, err, "missing protocol scheme") + assert.Empty(t, path) + }) +} diff --git a/internal/utils/misc.go b/internal/utils/misc.go index b986c19e9..f64f708c5 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -326,3 +326,7 @@ func ValidateFunctionSlug(slug string) error { return nil } + +func Ptr[T any](v T) *T { + return &v +} From 4286d532a1470fc38ceda72cd262d12147605ca1 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Thu, 19 Oct 2023 21:08:51 +0800 Subject: [PATCH 05/15] fix: add storage cp tests --- internal/storage/client/objects.go | 11 +- internal/storage/cp/cp.go | 74 ++-- internal/storage/cp/cp_test.go | 623 +++++++++++++++++++++++++++++ 3 files changed, 658 insertions(+), 50 deletions(-) create mode 100644 internal/storage/cp/cp_test.go diff --git a/internal/storage/client/objects.go b/internal/storage/client/objects.go index 626b2d7cc..2dba7d3ed 100644 --- a/internal/storage/client/objects.go +++ b/internal/storage/client/objects.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path" + "strings" "github.com/spf13/afero" "github.com/supabase/cli/internal/utils" @@ -62,16 +63,17 @@ func ListStorageObjects(ctx context.Context, projectRef, bucket, prefix string, } func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { - apiKey, err := tenant.GetApiKeys(ctx, projectRef) + f, err := fsys.Open(localPath) if err != nil { return err } - url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) - f, err := fsys.Open(localPath) + defer f.Close() + apiKey, err := tenant.GetApiKeys(ctx, projectRef) if err != nil { return err } - defer f.Close() + remotePath = strings.TrimPrefix(remotePath, "/") + url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, f) if err != nil { return err @@ -98,6 +100,7 @@ func DownloadStorageObject(ctx context.Context, projectRef, remotePath, localPat if err != nil { return err } + remotePath = strings.TrimPrefix(remotePath, "/") url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go index 4dcb6e6ba..31422de9f 100644 --- a/internal/storage/cp/cp.go +++ b/internal/storage/cp/cp.go @@ -17,6 +17,8 @@ import ( "github.com/supabase/cli/internal/utils" ) +var errUnsupportedOperation = errors.New("Unsupported operation") + func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { srcParsed, err := url.Parse(src) if err != nil { @@ -42,75 +44,55 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er } // TODO: Check if destination is a directory return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys) + } else if strings.ToLower(srcParsed.Scheme) == ls.STORAGE_SCHEME && strings.ToLower(dstParsed.Scheme) == ls.STORAGE_SCHEME { + return errors.New("Copying between buckets is not supported") } - return errors.New("Unsupported operation") + utils.CmdSuggestion = fmt.Sprintf("Run %s to copy between local directories.", utils.Aqua("cp -r ")) + return errUnsupportedOperation } func DownloadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { - remotePath = path.Join("/", remotePath) + // Prepare local directory for download if fi, err := fsys.Stat(localPath); err == nil && fi.IsDir() { localPath = filepath.Join(localPath, path.Base(remotePath)) } - if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(localPath)); err != nil { - return err - } - if !IsDir(remotePath) { - fmt.Fprintln(os.Stderr, "Downloading:", remotePath, "=>", localPath) - if err := client.DownloadStorageObject(ctx, projectRef, remotePath, localPath, fsys); err != nil && strings.Contains(err.Error(), `"error":"Not Found"`) { - // Retry downloading as directory - remotePath += "/" - } else { - return err + count := 0 + if err := ls.IterateStoragePathsAll(ctx, projectRef, remotePath, func(objectPath string) error { + relPath := strings.TrimPrefix(objectPath, remotePath) + dstPath := filepath.Join(localPath, filepath.FromSlash(relPath)) + fmt.Fprintln(os.Stderr, "Downloading:", objectPath, "=>", dstPath) + count++ + if strings.HasSuffix(objectPath, "/") { + return utils.MkdirIfNotExistFS(fsys, dstPath) } - } - queue := make([]string, 0) - queue = append(queue, remotePath) - for len(queue) > 0 { - dirPath := queue[len(queue)-1] - queue = queue[:len(queue)-1] - paths, err := ls.ListStoragePaths(ctx, projectRef, dirPath) - if err != nil { + if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(dstPath)); err != nil { return err } - if strings.Count(dirPath, "/") > 2 && len(paths) == 0 { - return errors.New("Object not found: " + dirPath) - } - for _, objectName := range paths { - objectPath := dirPath + objectName - relPath := strings.TrimPrefix(objectPath, remotePath) - dstPath := filepath.Join(localPath, filepath.FromSlash(relPath)) - fmt.Fprintln(os.Stderr, "Downloading:", objectPath, "=>", dstPath) - if strings.HasSuffix(objectName, "/") { - if err := utils.MkdirIfNotExistFS(fsys, dstPath); err != nil { - return err - } - queue = append(queue, objectPath) - continue - } - if err := client.DownloadStorageObject(ctx, projectRef, objectPath, dstPath, fsys); err != nil { - return err - } - } + return client.DownloadStorageObject(ctx, projectRef, objectPath, dstPath, fsys) + }); err != nil { + return err + } + if count == 0 { + return errors.New("Object not found: " + remotePath) } return nil } func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { noSlash := strings.TrimSuffix(remotePath, "/") - paths, err := ls.ListStoragePaths(ctx, projectRef, noSlash) - if err != nil { - return err - } // Check if directory exists on remote dirExists := false fileExists := false - for _, p := range paths { - if p == path.Base(noSlash) { + if err := ls.IterateStoragePaths(ctx, projectRef, noSlash, func(objectName string) error { + if objectName == path.Base(noSlash) { fileExists = true } - if p == path.Base(noSlash)+"/" { + if objectName == path.Base(noSlash)+"/" { dirExists = true } + return nil + }); err != nil { + return err } baseName := filepath.Base(localPath) return afero.Walk(fsys, localPath, func(filePath string, info fs.FileInfo, err error) error { diff --git a/internal/storage/cp/cp_test.go b/internal/storage/cp/cp_test.go new file mode 100644 index 000000000..151a0ea49 --- /dev/null +++ b/internal/storage/cp/cp_test.go @@ -0,0 +1,623 @@ +package cp + +import ( + "context" + "io/fs" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +func TestStorageCP(t *testing.T) { + t.Run("copy local to remote", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := Run(context.Background(), utils.ProjectRefPath, "ss:///private/file", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on missing file", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "abstract.pdf", "ss:///private", true, fsys) + // Check error + assert.ErrorIs(t, err, fs.ErrNotExist) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("copy remote to local", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := Run(context.Background(), "ss:///private/file", "abstract.pdf", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "abstract.pdf") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("throws error on missing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "ss:///private", ".", true, fsys) + // Check error + assert.ErrorContains(t, err, "Object not found: /private") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on invalid src", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ":", ".", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on invalid dst", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ".", ":", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ".", ".", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), ".", ".", false, fsys) + // Check error + assert.ErrorIs(t, err, errUnsupportedOperation) + }) +} + +func TestUploadAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("uploads directory to new bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON(client.CreateBucketResponse{Name: "tmp"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on failure to create bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", "/tmp", fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads directory to existing prefix", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + require.NoError(t, afero.WriteFile(fsys, "/tmp/docs/api.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "dir", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/dir/tmp/readme.md"). + Reply(http.StatusOK) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/dir/tmp/docs/api.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "/private/dir/", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads file to existing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/readme.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "private", "/tmp/readme.md", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads file to existing object", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "file", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "private/file", "/tmp/readme.md", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", ".", fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestDownloadAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("downloads buckets to existing directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "", "/", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.True(t, exists) + exists, err = afero.DirExists(fsys, "/test") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("downloads empty bucket to new directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Not Found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "/private", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + exists, err = afero.DirExists(fsys, "/tmp") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("throws error on empty directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "private/dir/", "/", fsys) + // Check error + assert.ErrorContains(t, err, "Object not found: private/dir/") + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("downloads objects to existing directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }, { + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/tmp/abstract.pdf"). + Reply(http.StatusOK) + // Lists /private/tmp/docs directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "readme.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/tmp/docs/readme.pdf"). + Reply(http.StatusOK) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "private/tmp/", "/", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "/tmp/abstract.pdf") + assert.NoError(t, err) + assert.True(t, exists) + exists, err = afero.Exists(fsys, "/tmp/docs/readme.pdf") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("downloads object to existing file", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/abstract.pdf"). + Reply(http.StatusOK) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "/private/abstract.pdf", "/tmp/file", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + exists, err = afero.Exists(fsys, "/tmp/file") + assert.NoError(t, err) + assert.True(t, exists) + }) +} From 9b14b1f7669bb1ca25d0c5a6cca470e1002b4c29 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Fri, 20 Oct 2023 14:49:09 +0800 Subject: [PATCH 06/15] fix: add storage mv tests --- internal/storage/mv/mv.go | 16 +- internal/storage/mv/mv_test.go | 374 +++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 internal/storage/mv/mv_test.go diff --git a/internal/storage/mv/mv.go b/internal/storage/mv/mv.go index 910efead3..11e402f61 100644 --- a/internal/storage/mv/mv.go +++ b/internal/storage/mv/mv.go @@ -14,6 +14,11 @@ import ( "github.com/supabase/cli/internal/utils" ) +var ( + errUnsupportedMove = errors.New("Moving between buckets is unsupported") + errMissingPath = errors.New("You must specify an object path") +) + func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { srcParsed, err := ls.ParseStorageURL(src) if err != nil { @@ -29,8 +34,11 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er } srcBucket, srcPrefix := ls.SplitBucketPrefix(srcParsed) dstBucket, dstPrefix := ls.SplitBucketPrefix(dstParsed) + if len(srcPrefix) == 0 && len(dstPrefix) == 0 { + return errMissingPath + } if srcBucket != dstBucket { - return errors.New("Moving between buckets is unsupported") + return errUnsupportedMove } fmt.Fprintln(os.Stderr, "Moving object:", srcParsed, "=>", dstParsed) data, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, dstPrefix) @@ -45,6 +53,8 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er // Expects srcPath to be terminated by "/" func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath string) error { _, dstPrefix := ls.SplitBucketPrefix(dstPath) + // Cannot iterate because pagination result may be updated during move + count := 0 queue := make([]string, 0) queue = append(queue, srcPath) for len(queue) > 0 { @@ -60,6 +70,7 @@ func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath stri queue = append(queue, objectPath) continue } + count++ relPath := strings.TrimPrefix(objectPath, srcPath) srcBucket, srcPrefix := ls.SplitBucketPrefix(objectPath) absPath := path.Join(dstPrefix, relPath) @@ -69,5 +80,8 @@ func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath stri } } } + if count == 0 { + return errors.New("Object not found: " + srcPath) + } return nil } diff --git a/internal/storage/mv/mv_test.go b/internal/storage/mv/mv_test.go new file mode 100644 index 000000000..609bca21e --- /dev/null +++ b/internal/storage/mv/mv_test.go @@ -0,0 +1,374 @@ +package mv + +import ( + "context" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageMV(t *testing.T) { + t.Run("moves single object", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "readme.md", + DestinationKey: "docs/file", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := Run(context.Background(), "ss:///private/readme.md", "ss:///private/docs/file", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves directory when recursive", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "", + DestinationKey: "docs", + }). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "not_found"}) + // List bucket /private/ + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "abstract.pdf", + DestinationKey: "docs/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := Run(context.Background(), "ss:///private", "ss:///private/docs", true, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on invalid src", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ":", "ss:///", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on invalid dst", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", ":", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("throws error on missing object path", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), "ss:///", "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingPath) + }) + + t.Run("throws error on bucket mismatch", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), "ss:///bucket/docs", "ss:///private", false, fsys) + // Check error + assert.ErrorIs(t, err, errUnsupportedMove) + }) +} + +func TestMoveAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("rename directory within bucket", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }, mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/abstract.pdf", + DestinationKey: "dir/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Lists /private/tmp/docs directory + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/docs/readme.md", + DestinationKey: "dir/docs/readme.md", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private/dir") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves object into directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/ bucket + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "abstract.pdf", + DestinationKey: "dir/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/", "private/dir") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves object out of directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp/ directory + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/readme.md", + DestinationKey: "readme.md", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on move failure", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + Reply(http.StatusServiceUnavailable) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on missing object", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Object not found: private/tmp/") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} From c1932a745b9c19aaa86a3b220ae681c1c8b19db1 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Fri, 20 Oct 2023 17:15:47 +0800 Subject: [PATCH 07/15] fix: add storage rm tests --- internal/storage/rm/rm.go | 65 +++--- internal/storage/rm/rm_test.go | 376 +++++++++++++++++++++++++++++++++ 2 files changed, 411 insertions(+), 30 deletions(-) create mode 100644 internal/storage/rm/rm_test.go diff --git a/internal/storage/rm/rm.go b/internal/storage/rm/rm.go index 7802e7f38..6eb7c9ace 100644 --- a/internal/storage/rm/rm.go +++ b/internal/storage/rm/rm.go @@ -14,6 +14,12 @@ import ( "github.com/supabase/cli/internal/utils" ) +var ( + errMissingObject = errors.New("Object not found") + errMissingBucket = errors.New("You must specify a bucket to delete.") + errMissingFlag = errors.New("You must specify -r flag to delete directories.") +) + type PrefixGroup struct { Bucket string Prefixes []string @@ -30,10 +36,10 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err bucket, prefix := ls.SplitBucketPrefix(remotePath) // Ignore attempts to delete all buckets if len(bucket) == 0 { - return errors.New("You must specify a bucket to delete.") + return errMissingBucket } if cp.IsDir(prefix) && !recursive { - return errors.New("You must specify -r flag to delete directories.") + return errMissingFlag } groups[bucket] = append(groups[bucket], prefix) } @@ -42,49 +48,38 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err return err } for bucket, prefixes := range groups { - if utils.SliceContains(prefixes, "") { - fmt.Fprintln(os.Stderr, "Deleting bucket:", bucket) - if err := RemoveStoragePathAll(ctx, projectRef, bucket, ""); err != nil { - return err - } - if data, err := client.DeleteStorageBucket(ctx, projectRef, bucket); err == nil { - fmt.Fprintln(os.Stderr, data.Message) - } else if !strings.Contains(err.Error(), `"error":"Bucket not found"`) { - return err - } else { - fmt.Fprintln(os.Stderr, "Bucket not found") - } - continue - } + // Always try deleting first in case the paths resolve to extensionless files fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) removed, err := client.DeleteStorageObjects(ctx, projectRef, bucket, prefixes) if err != nil { return err } - if !recursive { - if len(removed) == 0 { - utils.CmdSuggestion = "You must specify -r flag to delete directories." - return errors.New("Object not found") - } - continue - } set := map[string]struct{}{} for _, object := range removed { set[object.Name] = struct{}{} } for _, prefix := range prefixes { - if _, ok := set[prefix]; !ok { - if err := RemoveStoragePathAll(ctx, projectRef, bucket, prefix+"/"); err != nil { - return err - } + if _, ok := set[prefix]; ok { + continue + } + if !recursive { + fmt.Fprintln(os.Stderr, "Object not found:", prefix) + continue + } + if len(prefix) > 0 { + prefix += "/" + } + if err := RemoveStoragePathAll(ctx, projectRef, bucket, prefix); err != nil { + return err } } } return nil } -// Expects prefix to be terminated by "/" +// Expects prefix to be terminated by "/" or "" func RemoveStoragePathAll(ctx context.Context, projectRef, bucket, prefix string) error { + // We must remove one directory at a time to avoid breaking pagination result queue := make([]string, 0) queue = append(queue, prefix) for len(queue) > 0 { @@ -94,8 +89,8 @@ func RemoveStoragePathAll(ctx context.Context, projectRef, bucket, prefix string if err != nil { return err } - if len(paths) == 0 { - return errors.New("Object not found") + if len(paths) == 0 && len(prefix) > 0 { + return fmt.Errorf("%w: %s/%s", errMissingObject, bucket, prefix) } var files []string for _, objectName := range paths { @@ -113,5 +108,15 @@ func RemoveStoragePathAll(ctx context.Context, projectRef, bucket, prefix string } } } + if len(prefix) == 0 { + fmt.Fprintln(os.Stderr, "Deleting bucket:", bucket) + if data, err := client.DeleteStorageBucket(ctx, projectRef, bucket); err == nil { + fmt.Fprintln(os.Stderr, data.Message) + } else if strings.Contains(err.Error(), `"error":"Bucket not found"`) { + fmt.Fprintln(os.Stderr, "Bucket not found:", bucket) + } else { + return err + } + } return nil } diff --git a/internal/storage/rm/rm_test.go b/internal/storage/rm/rm_test.go new file mode 100644 index 000000000..49d4e3da2 --- /dev/null +++ b/internal/storage/rm/rm_test.go @@ -0,0 +1,376 @@ +package rm + +import ( + "context" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageRM(t *testing.T) { + t.Run("throws error on invalid url", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{":"}, false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{"ss:///"}, false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingBucket) + }) + + t.Run("throws error on missing flag", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{"ss:///private/"}, false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingFlag) + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{}, false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("removes multiple objects", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "abstract.pdf", + "docs/readme.md", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := Run(context.Background(), []string{ + "ss:///private/abstract.pdf", + "ss:///private/docs/readme.md", + }, false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("removes buckets and directories", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Delete /test/ bucket + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/test"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/bucket/test"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + // Delete /private/docs/ directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "docs", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "docs/abstract.pdf", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := Run(context.Background(), []string{ + "ss:///test", + "ss:///private/docs", + }, true, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on delete failure", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := Run(context.Background(), []string{"ss:///private"}, true, fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestRemoveAll(t *testing.T) { + projectRef := apitest.RandomProjectRef() + + t.Run("removes objects by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List /private/tmp/ + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }}) + // List /private/docs/ + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile, readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "tmp/docs/abstract.pdf", + "tmp/docs/readme.md", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }, { + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "readme.md", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "tmp/") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("removes empty bucket", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/bucket/private"). + Reply(http.StatusOK). + JSON(client.DeleteBucketResponse{Message: "Successfully deleted"}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on empty directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "dir") + // Check error + assert.ErrorContains(t, err, "Object not found: private/dir") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on delete failure", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} From e2f9cb69c9977273eb4ceba83ba3f3d2ecebe7b4 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 15:14:17 +0800 Subject: [PATCH 08/15] chore: refactor mock objects for unit tests --- internal/storage/cp/cp_test.go | 93 ++++++++++------------------------ internal/storage/ls/ls_test.go | 68 ++++++++----------------- 2 files changed, 47 insertions(+), 114 deletions(-) diff --git a/internal/storage/cp/cp_test.go b/internal/storage/cp/cp_test.go index 151a0ea49..00de4b9e3 100644 --- a/internal/storage/cp/cp_test.go +++ b/internal/storage/cp/cp_test.go @@ -16,6 +16,23 @@ import ( "gopkg.in/h2non/gock.v1" ) +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + func TestStorageCP(t *testing.T) { t.Run("copy local to remote", func(t *testing.T) { // Setup in-memory fs @@ -314,25 +331,12 @@ func TestUploadAll(t *testing.T) { Name: "service_role", ApiKey: "service-key", }}) + fileObject := mockFile + fileObject.Name = "file" gock.New("https://" + utils.GetSupabaseHost(projectRef)). Post("/storage/v1/object/list/private"). Reply(http.StatusOK). - JSON([]client.ObjectResponse{{ - Name: "file", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + JSON([]client.ObjectResponse{fileObject}) gock.New("https://" + utils.GetSupabaseHost(projectRef)). Post("/storage/v1/object/private/file"). Reply(http.StatusOK) @@ -513,26 +517,13 @@ func TestDownloadAll(t *testing.T) { Reply(http.StatusOK). JSON([]client.ObjectResponse{{ Name: "docs", - }, { - Name: "abstract.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + }, mockFile}) gock.New("https://" + utils.GetSupabaseHost(projectRef)). Get("/storage/v1/object/private/tmp/abstract.pdf"). Reply(http.StatusOK) // Lists /private/tmp/docs directory + readme := mockFile + readme.Name = "readme.md" gock.New("https://" + utils.GetSupabaseHost(projectRef)). Post("/storage/v1/object/list/private"). JSON(client.ListObjectsQuery{ @@ -542,24 +533,9 @@ func TestDownloadAll(t *testing.T) { Offset: 0, }). Reply(http.StatusOK). - JSON([]client.ObjectResponse{{ - Name: "readme.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + JSON([]client.ObjectResponse{readme}) gock.New("https://" + utils.GetSupabaseHost(projectRef)). - Get("/storage/v1/object/private/tmp/docs/readme.pdf"). + Get("/storage/v1/object/private/tmp/docs/readme.md"). Reply(http.StatusOK) // Run test err := DownloadStorageObjectAll(context.Background(), projectRef, "private/tmp/", "/", fsys) @@ -569,7 +545,7 @@ func TestDownloadAll(t *testing.T) { exists, err := afero.Exists(fsys, "/tmp/abstract.pdf") assert.NoError(t, err) assert.True(t, exists) - exists, err = afero.Exists(fsys, "/tmp/docs/readme.pdf") + exists, err = afero.Exists(fsys, "/tmp/docs/readme.md") assert.NoError(t, err) assert.True(t, exists) }) @@ -589,22 +565,7 @@ func TestDownloadAll(t *testing.T) { gock.New("https://" + utils.GetSupabaseHost(projectRef)). Post("/storage/v1/object/list/private"). Reply(http.StatusOK). - JSON([]client.ObjectResponse{{ - Name: "abstract.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + JSON([]client.ObjectResponse{mockFile}) gock.New("https://" + utils.GetSupabaseHost(projectRef)). Get("/storage/v1/object/private/abstract.pdf"). Reply(http.StatusOK) diff --git a/internal/storage/ls/ls_test.go b/internal/storage/ls/ls_test.go index 77148d733..c82de6b5a 100644 --- a/internal/storage/ls/ls_test.go +++ b/internal/storage/ls/ls_test.go @@ -16,6 +16,23 @@ import ( "gopkg.in/h2non/gock.v1" ) +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + func TestStorageLS(t *testing.T) { t.Run("lists buckets", func(t *testing.T) { // Setup in-memory fs @@ -175,22 +192,7 @@ func TestListStoragePaths(t *testing.T) { Reply(http.StatusOK). JSON([]client.ObjectResponse{{ Name: "folder", - }, { - Name: "abstract.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + }, mockFile}) // Run test paths, err := ListStoragePaths(context.Background(), projectRef, "bucket/") // Check error @@ -331,22 +333,7 @@ func TestListStoragePathsAll(t *testing.T) { Offset: 0, }). Reply(http.StatusOK). - JSON([]client.ObjectResponse{{ - Name: "abstract.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + JSON([]client.ObjectResponse{mockFile}) // Run test paths, err := ListStoragePathsAll(context.Background(), projectRef, "") // Check error @@ -377,22 +364,7 @@ func TestListStoragePathsAll(t *testing.T) { Reply(http.StatusOK). JSON([]client.ObjectResponse{{ Name: "error", - }, { - Name: "abstract.pdf", - Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), - UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), - Metadata: &client.ObjectMetadata{ - ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, - Size: 82702, - Mimetype: "application/pdf", - CacheControl: "max-age=3600", - LastModified: "2023-10-13T18:08:22.000Z", - ContentLength: 82702, - HttpStatusCode: 200, - }, - }}) + }, mockFile}) gock.New("https://" + utils.GetSupabaseHost(projectRef)). Post("/storage/v1/object/list/private"). JSON(client.ListObjectsQuery{ From 3e7a8fc9896a8a78a12ebf718f7d15c40cf16535 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 18:52:51 +0800 Subject: [PATCH 09/15] chore: address comments --- cmd/storage.go | 6 +++--- internal/storage/cp/cp_test.go | 2 +- internal/storage/ls/ls.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/storage.go b/cmd/storage.go index 43359bdb5..ce524b7f5 100644 --- a/cmd/storage.go +++ b/cmd/storage.go @@ -19,7 +19,7 @@ var ( recursive bool lsCmd = &cobra.Command{ - Use: "ls [prefix]", + Use: "ls [path]", Example: "ls ss:///bucket/docs", Short: "List objects by path prefix", Args: cobra.MaximumNArgs(1), @@ -38,7 +38,7 @@ var ( cp -r docs ss:///bucket/docs cp -r ss:///bucket/docs . `, - Short: "Copy objects by from src to dst path", + Short: "Copy objects from src to dst path", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { return cp.Run(cmd.Context(), args[0], args[1], recursive, afero.NewOsFs()) @@ -47,7 +47,7 @@ cp -r ss:///bucket/docs . mvCmd = &cobra.Command{ Use: "mv ", - Short: "Move objects by from src to dst path", + Short: "Move objects from src to dst path", Example: "mv -r ss:///bucket/docs ss:///bucket/www/docs", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { diff --git a/internal/storage/cp/cp_test.go b/internal/storage/cp/cp_test.go index 00de4b9e3..aad264c57 100644 --- a/internal/storage/cp/cp_test.go +++ b/internal/storage/cp/cp_test.go @@ -168,7 +168,7 @@ func TestStorageCP(t *testing.T) { assert.ErrorIs(t, err, utils.ErrNotLinked) }) - t.Run("throws error on missing project", func(t *testing.T) { + t.Run("throws error on unsupported operation", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() projectRef := apitest.RandomProjectRef() diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go index 5c4a1f7cd..5cb5963c9 100644 --- a/internal/storage/ls/ls.go +++ b/internal/storage/ls/ls.go @@ -35,7 +35,7 @@ func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) return IterateStoragePaths(ctx, projectRef, remotePath, callback) } -var errInvalidURL = errors.New("URL must match pattern ss:///bucket/prefix") +var errInvalidURL = errors.New("URL must match pattern ss:///bucket/[prefix]") func ParseStorageURL(objectPath string) (string, error) { parsed, err := url.Parse(objectPath) From f91b745f932d1e4d80c575d14cd181b93859f1e6 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 18:53:36 +0800 Subject: [PATCH 10/15] chore: move common functions to shared package --- internal/storage/cp/cp.go | 3 +- internal/storage/ls/ls.go | 37 ++------------- internal/storage/ls/ls_test.go | 79 +------------------------------ internal/storage/mv/mv.go | 13 +++--- internal/storage/rm/rm.go | 5 +- internal/storage/scheme.go | 37 +++++++++++++++ internal/storage/scheme_test.go | 83 +++++++++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 119 deletions(-) create mode 100644 internal/storage/scheme.go create mode 100644 internal/storage/scheme_test.go diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go index 31422de9f..5ff84dd78 100644 --- a/internal/storage/cp/cp.go +++ b/internal/storage/cp/cp.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/client" "github.com/supabase/cli/internal/storage/ls" "github.com/supabase/cli/internal/utils" @@ -122,7 +123,7 @@ func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPa err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys) if err != nil && strings.Contains(err.Error(), `"error":"Bucket not found"`) { // Retry after creating bucket - if bucket, prefix := ls.SplitBucketPrefix(dstPath); len(prefix) > 0 { + if bucket, prefix := storage.SplitBucketPrefix(dstPath); len(prefix) > 0 { if _, err := client.CreateStorageBucket(ctx, projectRef, bucket); err != nil { return err } diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go index 5cb5963c9..77a069a78 100644 --- a/internal/storage/ls/ls.go +++ b/internal/storage/ls/ls.go @@ -2,14 +2,13 @@ package ls import ( "context" - "errors" "fmt" - "net/url" "os" "path" "strings" "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/client" "github.com/supabase/cli/internal/utils" ) @@ -17,7 +16,7 @@ import ( const STORAGE_SCHEME = "ss" func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) error { - remotePath, err := ParseStorageURL(objectPath) + remotePath, err := storage.ParseStorageURL(objectPath) if err != nil { return err } @@ -35,19 +34,6 @@ func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) return IterateStoragePaths(ctx, projectRef, remotePath, callback) } -var errInvalidURL = errors.New("URL must match pattern ss:///bucket/[prefix]") - -func ParseStorageURL(objectPath string) (string, error) { - parsed, err := url.Parse(objectPath) - if err != nil { - return "", err - } - if strings.ToLower(parsed.Scheme) != STORAGE_SCHEME || len(parsed.Path) == 0 || len(parsed.Host) > 0 { - return "", errInvalidURL - } - return parsed.Path, nil -} - func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]string, error) { var result []string err := IterateStoragePaths(ctx, projectRef, remotePath, func(objectName string) error { @@ -58,7 +44,7 @@ func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]str } func IterateStoragePaths(ctx context.Context, projectRef, remotePath string, callback func(objectName string) error) error { - bucket, prefix := SplitBucketPrefix(remotePath) + bucket, prefix := storage.SplitBucketPrefix(remotePath) if len(bucket) == 0 || (len(prefix) == 0 && !strings.HasSuffix(remotePath, "/")) { buckets, err := client.ListStorageBuckets(ctx, projectRef) if err != nil { @@ -97,21 +83,6 @@ func IterateStoragePaths(ctx context.Context, projectRef, remotePath string, cal return nil } -func SplitBucketPrefix(objectPath string) (string, string) { - if objectPath == "" || objectPath == "/" { - return "", "" - } - start := 0 - if objectPath[0] == '/' { - start = 1 - } - sep := strings.IndexByte(objectPath[start:], '/') - if sep < 0 { - return objectPath[start:], "" - } - return objectPath[start : sep+start], objectPath[sep+start+1:] -} - // Expects remotePath to be terminated by "/" func ListStoragePathsAll(ctx context.Context, projectRef, remotePath string) ([]string, error) { var result []string @@ -156,7 +127,7 @@ func IterateStoragePathsAll(ctx context.Context, projectRef, remotePath string, return err } // Also report empty buckets - bucket, prefix := SplitBucketPrefix(dirPath) + bucket, prefix := storage.SplitBucketPrefix(dirPath) if empty && len(prefix) == 0 { if err := callback(bucket + "/"); err != nil { return err diff --git a/internal/storage/ls/ls_test.go b/internal/storage/ls/ls_test.go index c82de6b5a..cae6542d3 100644 --- a/internal/storage/ls/ls_test.go +++ b/internal/storage/ls/ls_test.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/client" "github.com/supabase/cli/internal/testing/apitest" "github.com/supabase/cli/internal/utils" @@ -67,7 +68,7 @@ func TestStorageLS(t *testing.T) { // Run test err := Run(context.Background(), "", false, fsys) // Check error - assert.ErrorIs(t, err, errInvalidURL) + assert.ErrorIs(t, err, storage.ErrInvalidURL) }) t.Run("throws error on invalid project", func(t *testing.T) { @@ -392,79 +393,3 @@ func TestListStoragePathsAll(t *testing.T) { assert.Empty(t, apitest.ListUnmatchedRequests()) }) } - -func TestSplitBucketPrefix(t *testing.T) { - t.Run("splits empty path", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("") - assert.Equal(t, bucket, "") - assert.Equal(t, prefix, "") - }) - - t.Run("splits root path", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("/") - assert.Equal(t, bucket, "") - assert.Equal(t, prefix, "") - }) - - t.Run("splits no slash", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("bucket") - assert.Equal(t, bucket, "bucket") - assert.Equal(t, prefix, "") - }) - - t.Run("splits prefix slash", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("/bucket") - assert.Equal(t, bucket, "bucket") - assert.Equal(t, prefix, "") - }) - - t.Run("splits suffix slash", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("bucket/") - assert.Equal(t, bucket, "bucket") - assert.Equal(t, prefix, "") - }) - - t.Run("splits file path", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("/bucket/folder/name.png") - assert.Equal(t, bucket, "bucket") - assert.Equal(t, prefix, "folder/name.png") - }) - - t.Run("splits dir path", func(t *testing.T) { - bucket, prefix := SplitBucketPrefix("/bucket/folder/") - assert.Equal(t, bucket, "bucket") - assert.Equal(t, prefix, "folder/") - }) -} - -func TestParseStorageURL(t *testing.T) { - t.Run("parses valid url", func(t *testing.T) { - path, err := ParseStorageURL("ss:///bucket/folder/name.png") - assert.NoError(t, err) - assert.Equal(t, path, "/bucket/folder/name.png") - }) - - t.Run("throws error on invalid host", func(t *testing.T) { - path, err := ParseStorageURL("ss://bucket") - assert.ErrorIs(t, err, errInvalidURL) - assert.Empty(t, path) - }) - - t.Run("throws error on missing path", func(t *testing.T) { - path, err := ParseStorageURL("ss:") - assert.ErrorIs(t, err, errInvalidURL) - assert.Empty(t, path) - }) - - t.Run("throws error on invalid scheme", func(t *testing.T) { - path, err := ParseStorageURL(".") - assert.ErrorIs(t, err, errInvalidURL) - assert.Empty(t, path) - }) - - t.Run("throws error on invalid url", func(t *testing.T) { - path, err := ParseStorageURL(":") - assert.ErrorContains(t, err, "missing protocol scheme") - assert.Empty(t, path) - }) -} diff --git a/internal/storage/mv/mv.go b/internal/storage/mv/mv.go index 11e402f61..1fe1b440f 100644 --- a/internal/storage/mv/mv.go +++ b/internal/storage/mv/mv.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/client" "github.com/supabase/cli/internal/storage/ls" "github.com/supabase/cli/internal/utils" @@ -20,11 +21,11 @@ var ( ) func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { - srcParsed, err := ls.ParseStorageURL(src) + srcParsed, err := storage.ParseStorageURL(src) if err != nil { return err } - dstParsed, err := ls.ParseStorageURL(dst) + dstParsed, err := storage.ParseStorageURL(dst) if err != nil { return err } @@ -32,8 +33,8 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er if err != nil { return err } - srcBucket, srcPrefix := ls.SplitBucketPrefix(srcParsed) - dstBucket, dstPrefix := ls.SplitBucketPrefix(dstParsed) + srcBucket, srcPrefix := storage.SplitBucketPrefix(srcParsed) + dstBucket, dstPrefix := storage.SplitBucketPrefix(dstParsed) if len(srcPrefix) == 0 && len(dstPrefix) == 0 { return errMissingPath } @@ -52,7 +53,7 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er // Expects srcPath to be terminated by "/" func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath string) error { - _, dstPrefix := ls.SplitBucketPrefix(dstPath) + _, dstPrefix := storage.SplitBucketPrefix(dstPath) // Cannot iterate because pagination result may be updated during move count := 0 queue := make([]string, 0) @@ -72,7 +73,7 @@ func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath stri } count++ relPath := strings.TrimPrefix(objectPath, srcPath) - srcBucket, srcPrefix := ls.SplitBucketPrefix(objectPath) + srcBucket, srcPrefix := storage.SplitBucketPrefix(objectPath) absPath := path.Join(dstPrefix, relPath) fmt.Fprintln(os.Stderr, "Moving object:", objectPath, "=>", path.Join(dstPath, relPath)) if _, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, absPath); err != nil { diff --git a/internal/storage/rm/rm.go b/internal/storage/rm/rm.go index 6eb7c9ace..51a21544c 100644 --- a/internal/storage/rm/rm.go +++ b/internal/storage/rm/rm.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/client" "github.com/supabase/cli/internal/storage/cp" "github.com/supabase/cli/internal/storage/ls" @@ -29,11 +30,11 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err // Group paths by buckets groups := map[string][]string{} for _, objectPath := range paths { - remotePath, err := ls.ParseStorageURL(objectPath) + remotePath, err := storage.ParseStorageURL(objectPath) if err != nil { return err } - bucket, prefix := ls.SplitBucketPrefix(remotePath) + bucket, prefix := storage.SplitBucketPrefix(remotePath) // Ignore attempts to delete all buckets if len(bucket) == 0 { return errMissingBucket diff --git a/internal/storage/scheme.go b/internal/storage/scheme.go new file mode 100644 index 000000000..aabf23017 --- /dev/null +++ b/internal/storage/scheme.go @@ -0,0 +1,37 @@ +package storage + +import ( + "errors" + "net/url" + "strings" +) + +const STORAGE_SCHEME = "ss" + +var ErrInvalidURL = errors.New("URL must match pattern ss:///bucket/[prefix]") + +func ParseStorageURL(objectURL string) (string, error) { + parsed, err := url.Parse(objectURL) + if err != nil { + return "", err + } + if strings.ToLower(parsed.Scheme) != STORAGE_SCHEME || len(parsed.Path) == 0 || len(parsed.Host) > 0 { + return "", ErrInvalidURL + } + return parsed.Path, nil +} + +func SplitBucketPrefix(objectPath string) (string, string) { + if objectPath == "" || objectPath == "/" { + return "", "" + } + start := 0 + if objectPath[0] == '/' { + start = 1 + } + sep := strings.IndexByte(objectPath[start:], '/') + if sep < 0 { + return objectPath[start:], "" + } + return objectPath[start : sep+start], objectPath[sep+start+1:] +} diff --git a/internal/storage/scheme_test.go b/internal/storage/scheme_test.go new file mode 100644 index 000000000..9427ffcb6 --- /dev/null +++ b/internal/storage/scheme_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseStorageURL(t *testing.T) { + t.Run("parses valid url", func(t *testing.T) { + path, err := ParseStorageURL("ss:///bucket/folder/name.png") + assert.NoError(t, err) + assert.Equal(t, path, "/bucket/folder/name.png") + }) + + t.Run("throws error on invalid host", func(t *testing.T) { + path, err := ParseStorageURL("ss://bucket") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on missing path", func(t *testing.T) { + path, err := ParseStorageURL("ss:") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid scheme", func(t *testing.T) { + path, err := ParseStorageURL(".") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid url", func(t *testing.T) { + path, err := ParseStorageURL(":") + assert.ErrorContains(t, err, "missing protocol scheme") + assert.Empty(t, path) + }) +} + +func TestSplitBucketPrefix(t *testing.T) { + t.Run("splits empty path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits root path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits no slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits prefix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits suffix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits file path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/name.png") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/name.png") + }) + + t.Run("splits dir path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/") + }) +} From a7e473a66e6d437cb0acbc38848d9f1225da18bc Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 20:45:28 +0800 Subject: [PATCH 11/15] chore: remove unused constant --- cmd/storage.go | 3 ++- internal/storage/cp/cp.go | 8 +++----- internal/storage/ls/ls.go | 2 -- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/storage.go b/cmd/storage.go index ce524b7f5..fefdc87f5 100644 --- a/cmd/storage.go +++ b/cmd/storage.go @@ -3,6 +3,7 @@ package cmd import ( "github.com/spf13/afero" "github.com/spf13/cobra" + "github.com/supabase/cli/internal/storage" "github.com/supabase/cli/internal/storage/cp" "github.com/supabase/cli/internal/storage/ls" "github.com/supabase/cli/internal/storage/mv" @@ -24,7 +25,7 @@ var ( Short: "List objects by path prefix", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - objectPath := ls.STORAGE_SCHEME + ":///" + objectPath := storage.STORAGE_SCHEME + ":///" if len(args) > 0 { objectPath = args[0] } diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go index 5ff84dd78..328e40c15 100644 --- a/internal/storage/cp/cp.go +++ b/internal/storage/cp/cp.go @@ -33,19 +33,17 @@ func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) er if err != nil { return err } - if strings.ToLower(srcParsed.Scheme) == ls.STORAGE_SCHEME && dstParsed.Scheme == "" { + if strings.ToLower(srcParsed.Scheme) == storage.STORAGE_SCHEME && dstParsed.Scheme == "" { if recursive { return DownloadStorageObjectAll(ctx, projectRef, srcParsed.Path, dst, fsys) } - // TODO: Check if destination is a directory return client.DownloadStorageObject(ctx, projectRef, srcParsed.Path, dst, fsys) - } else if srcParsed.Scheme == "" && strings.ToLower(dstParsed.Scheme) == ls.STORAGE_SCHEME { + } else if srcParsed.Scheme == "" && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME { if recursive { return UploadStorageObjectAll(ctx, projectRef, dstParsed.Path, src, fsys) } - // TODO: Check if destination is a directory return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys) - } else if strings.ToLower(srcParsed.Scheme) == ls.STORAGE_SCHEME && strings.ToLower(dstParsed.Scheme) == ls.STORAGE_SCHEME { + } else if strings.ToLower(srcParsed.Scheme) == storage.STORAGE_SCHEME && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME { return errors.New("Copying between buckets is not supported") } utils.CmdSuggestion = fmt.Sprintf("Run %s to copy between local directories.", utils.Aqua("cp -r ")) diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go index 77a069a78..079dd74b5 100644 --- a/internal/storage/ls/ls.go +++ b/internal/storage/ls/ls.go @@ -13,8 +13,6 @@ import ( "github.com/supabase/cli/internal/utils" ) -const STORAGE_SCHEME = "ss" - func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) error { remotePath, err := storage.ParseStorageURL(objectPath) if err != nil { From 86d691b23d7cadd1a31242bef37813b332ea1cc5 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 21:15:22 +0800 Subject: [PATCH 12/15] chore: add delete confirmation --- internal/storage/rm/rm.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/storage/rm/rm.go b/internal/storage/rm/rm.go index 51a21544c..ed0cf6e74 100644 --- a/internal/storage/rm/rm.go +++ b/internal/storage/rm/rm.go @@ -49,6 +49,10 @@ func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) err return err } for bucket, prefixes := range groups { + confirm := fmt.Sprintf("Confirm deleting files in bucket %v?", utils.Bold(bucket)) + if shouldDelete := utils.PromptYesNo(confirm, true, os.Stdin); !shouldDelete { + continue + } // Always try deleting first in case the paths resolve to extensionless files fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) removed, err := client.DeleteStorageObjects(ctx, projectRef, bucket, prefixes) From 7d25ae5244878c8b4e1b0314976cc3a1eeda0937 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 23 Oct 2023 21:32:06 +0800 Subject: [PATCH 13/15] chore: update storage client json properties --- internal/storage/client/buckets.go | 7 ++----- internal/storage/client/objects.go | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/storage/client/buckets.go b/internal/storage/client/buckets.go index 2a1703bfc..ef3947d28 100644 --- a/internal/storage/client/buckets.go +++ b/internal/storage/client/buckets.go @@ -34,8 +34,8 @@ func ListStorageBuckets(ctx context.Context, projectRef string) ([]BucketRespons } type CreateBucketRequest struct { - Id string `json:"id"` // "string", Name string `json:"name"` // "string", + Id string `json:"id,omitempty"` // "string", Public bool `json:"public,omitempty"` // false, FileSizeLimit int `json:"file_size_limit,omitempty"` // 0, AllowedMimeTypes []string `json:"allowed_mime_types,omitempty"` // ["string"] @@ -51,10 +51,7 @@ func CreateStorageBucket(ctx context.Context, projectRef, bucketName string) (*C if err != nil { return nil, err } - body := CreateBucketRequest{ - Id: bucketName, - Name: bucketName, - } + body := CreateBucketRequest{Name: bucketName} return tenant.JsonResponseWithBearer[CreateBucketResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, body) } diff --git a/internal/storage/client/objects.go b/internal/storage/client/objects.go index 2dba7d3ed..7f4433122 100644 --- a/internal/storage/client/objects.go +++ b/internal/storage/client/objects.go @@ -18,9 +18,9 @@ const PAGE_LIMIT = 100 type ListObjectsQuery struct { Prefix string `json:"prefix"` - Search string `json:"search"` - Limit int `json:"limit"` - Offset int `json:"offset"` + Search string `json:"search,omitempty"` + Limit int `json:"limit,omitempty"` + Offset int `json:"offset,omitempty"` } type ObjectResponse struct { From 61d6c060cbfd54432eead61d579e9a5485177e1d Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 24 Oct 2023 19:33:17 +0800 Subject: [PATCH 14/15] fix: add mimetype and cache-control headers for upload --- internal/storage/client/objects.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/storage/client/objects.go b/internal/storage/client/objects.go index 7f4433122..12ef1a3bb 100644 --- a/internal/storage/client/objects.go +++ b/internal/storage/client/objects.go @@ -68,6 +68,18 @@ func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath return err } defer f.Close() + // Decode mimetype + header := io.LimitReader(f, 512) + buf, err := io.ReadAll(header) + if err != nil { + return err + } + mimetype := http.DetectContentType(buf) + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return err + } + // Prepare request apiKey, err := tenant.GetApiKeys(ctx, projectRef) if err != nil { return err @@ -79,6 +91,9 @@ func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath return err } req.Header.Add("Authorization", "Bearer "+apiKey.ServiceRole) + req.Header.Add("Content-Type", mimetype) + // Use default value of storage-js: https://github.com/supabase/storage-js/blob/main/src/packages/StorageFileApi.ts#L22 + req.Header.Add("Cache-Control", "max-age=3600") // Sends request resp, err := http.DefaultClient.Do(req) if err != nil { From dfdd02ac58d049538bdac6b23221ec646bd5f0e2 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 24 Oct 2023 19:33:53 +0800 Subject: [PATCH 15/15] fix: auto create bucket when uploading file recursively --- cmd/storage.go | 2 +- internal/storage/cp/cp.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/storage.go b/cmd/storage.go index fefdc87f5..c161c4845 100644 --- a/cmd/storage.go +++ b/cmd/storage.go @@ -35,7 +35,7 @@ var ( cpCmd = &cobra.Command{ Use: "cp ", - Example: `cp readme.md ss:///bucket + Example: `cp readme.md ss:///bucket/readme.md cp -r docs ss:///bucket/docs cp -r ss:///bucket/docs . `, diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go index 328e40c15..eaf972e9c 100644 --- a/internal/storage/cp/cp.go +++ b/internal/storage/cp/cp.go @@ -108,7 +108,8 @@ func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPa dstPath := remotePath // Copying single file if relPath == "." { - if IsDir(dstPath) || (dirExists && !fileExists) { + _, prefix := storage.SplitBucketPrefix(dstPath) + if IsDir(prefix) || (dirExists && !fileExists) { dstPath = path.Join(dstPath, info.Name()) } } else { @@ -132,6 +133,6 @@ func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPa }) } -func IsDir(objectPath string) bool { - return len(objectPath) == 0 || strings.HasSuffix(objectPath, "/") +func IsDir(objectPrefix string) bool { + return len(objectPrefix) == 0 || strings.HasSuffix(objectPrefix, "/") }