diff --git a/cmd/storage.go b/cmd/storage.go new file mode 100644 index 000000000..c161c4845 --- /dev/null +++ b/cmd/storage.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "github.com/spf13/afero" + "github.com/spf13/cobra" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/cp" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/storage/mv" + "github.com/supabase/cli/internal/storage/rm" +) + +var ( + storageCmd = &cobra.Command{ + GroupID: groupManagementAPI, + Use: "storage", + Short: "Manage Supabase Storage objects", + } + + recursive bool + + lsCmd = &cobra.Command{ + Use: "ls [path]", + Example: "ls ss:///bucket/docs", + Short: "List objects by path prefix", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + objectPath := storage.STORAGE_SCHEME + ":///" + if len(args) > 0 { + objectPath = args[0] + } + return ls.Run(cmd.Context(), objectPath, recursive, afero.NewOsFs()) + }, + } + + cpCmd = &cobra.Command{ + Use: "cp ", + Example: `cp readme.md ss:///bucket/readme.md +cp -r docs ss:///bucket/docs +cp -r ss:///bucket/docs . +`, + Short: "Copy objects from src to dst path", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return cp.Run(cmd.Context(), args[0], args[1], recursive, afero.NewOsFs()) + }, + } + + mvCmd = &cobra.Command{ + Use: "mv ", + Short: "Move objects from src to dst path", + Example: "mv -r ss:///bucket/docs ss:///bucket/www/docs", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return mv.Run(cmd.Context(), args[0], args[1], recursive, afero.NewOsFs()) + }, + } + + rmCmd = &cobra.Command{ + Use: "rm ...", + Short: "Remove objects by file path", + Example: `rm -r ss:///bucket/docs +rm ss:///bucket/docs/example.md ss:///bucket/readme.md +`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return rm.Run(cmd.Context(), args, recursive, afero.NewOsFs()) + }, + } +) + +func init() { + lsCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively list a directory.") + storageCmd.AddCommand(lsCmd) + cpCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively copy a directory.") + storageCmd.AddCommand(cpCmd) + rmCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively move a directory.") + storageCmd.AddCommand(rmCmd) + mvCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively remove a directory.") + storageCmd.AddCommand(mvCmd) + rootCmd.AddCommand(storageCmd) +} diff --git a/internal/storage/client/buckets.go b/internal/storage/client/buckets.go index 2a1703bfc..ef3947d28 100644 --- a/internal/storage/client/buckets.go +++ b/internal/storage/client/buckets.go @@ -34,8 +34,8 @@ func ListStorageBuckets(ctx context.Context, projectRef string) ([]BucketRespons } type CreateBucketRequest struct { - Id string `json:"id"` // "string", Name string `json:"name"` // "string", + Id string `json:"id,omitempty"` // "string", Public bool `json:"public,omitempty"` // false, FileSizeLimit int `json:"file_size_limit,omitempty"` // 0, AllowedMimeTypes []string `json:"allowed_mime_types,omitempty"` // ["string"] @@ -51,10 +51,7 @@ func CreateStorageBucket(ctx context.Context, projectRef, bucketName string) (*C if err != nil { return nil, err } - body := CreateBucketRequest{ - Id: bucketName, - Name: bucketName, - } + body := CreateBucketRequest{Name: bucketName} return tenant.JsonResponseWithBearer[CreateBucketResponse](ctx, http.MethodPost, url, apiKey.ServiceRole, body) } diff --git a/internal/storage/client/objects.go b/internal/storage/client/objects.go index 626b2d7cc..12ef1a3bb 100644 --- a/internal/storage/client/objects.go +++ b/internal/storage/client/objects.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path" + "strings" "github.com/spf13/afero" "github.com/supabase/cli/internal/utils" @@ -17,9 +18,9 @@ const PAGE_LIMIT = 100 type ListObjectsQuery struct { Prefix string `json:"prefix"` - Search string `json:"search"` - Limit int `json:"limit"` - Offset int `json:"offset"` + Search string `json:"search,omitempty"` + Limit int `json:"limit,omitempty"` + Offset int `json:"offset,omitempty"` } type ObjectResponse struct { @@ -62,21 +63,37 @@ func ListStorageObjects(ctx context.Context, projectRef, bucket, prefix string, } func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { - apiKey, err := tenant.GetApiKeys(ctx, projectRef) + f, err := fsys.Open(localPath) if err != nil { return err } - url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) - f, err := fsys.Open(localPath) + defer f.Close() + // Decode mimetype + header := io.LimitReader(f, 512) + buf, err := io.ReadAll(header) if err != nil { return err } - defer f.Close() + mimetype := http.DetectContentType(buf) + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return err + } + // Prepare request + apiKey, err := tenant.GetApiKeys(ctx, projectRef) + if err != nil { + return err + } + remotePath = strings.TrimPrefix(remotePath, "/") + url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, f) if err != nil { return err } req.Header.Add("Authorization", "Bearer "+apiKey.ServiceRole) + req.Header.Add("Content-Type", mimetype) + // Use default value of storage-js: https://github.com/supabase/storage-js/blob/main/src/packages/StorageFileApi.ts#L22 + req.Header.Add("Cache-Control", "max-age=3600") // Sends request resp, err := http.DefaultClient.Do(req) if err != nil { @@ -98,6 +115,7 @@ func DownloadStorageObject(ctx context.Context, projectRef, remotePath, localPat if err != nil { return err } + remotePath = strings.TrimPrefix(remotePath, "/") url := fmt.Sprintf("https://%s/storage/v1/object/%s", utils.GetSupabaseHost(projectRef), remotePath) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { diff --git a/internal/storage/cp/cp.go b/internal/storage/cp/cp.go new file mode 100644 index 000000000..eaf972e9c --- /dev/null +++ b/internal/storage/cp/cp.go @@ -0,0 +1,138 @@ +package cp + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +var errUnsupportedOperation = errors.New("Unsupported operation") + +func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { + srcParsed, err := url.Parse(src) + if err != nil { + return err + } + dstParsed, err := url.Parse(dst) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + if strings.ToLower(srcParsed.Scheme) == storage.STORAGE_SCHEME && dstParsed.Scheme == "" { + if recursive { + return DownloadStorageObjectAll(ctx, projectRef, srcParsed.Path, dst, fsys) + } + return client.DownloadStorageObject(ctx, projectRef, srcParsed.Path, dst, fsys) + } else if srcParsed.Scheme == "" && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME { + if recursive { + return UploadStorageObjectAll(ctx, projectRef, dstParsed.Path, src, fsys) + } + return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys) + } else if strings.ToLower(srcParsed.Scheme) == storage.STORAGE_SCHEME && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME { + return errors.New("Copying between buckets is not supported") + } + utils.CmdSuggestion = fmt.Sprintf("Run %s to copy between local directories.", utils.Aqua("cp -r ")) + return errUnsupportedOperation +} + +func DownloadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + // Prepare local directory for download + if fi, err := fsys.Stat(localPath); err == nil && fi.IsDir() { + localPath = filepath.Join(localPath, path.Base(remotePath)) + } + count := 0 + if err := ls.IterateStoragePathsAll(ctx, projectRef, remotePath, func(objectPath string) error { + relPath := strings.TrimPrefix(objectPath, remotePath) + dstPath := filepath.Join(localPath, filepath.FromSlash(relPath)) + fmt.Fprintln(os.Stderr, "Downloading:", objectPath, "=>", dstPath) + count++ + if strings.HasSuffix(objectPath, "/") { + return utils.MkdirIfNotExistFS(fsys, dstPath) + } + if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(dstPath)); err != nil { + return err + } + return client.DownloadStorageObject(ctx, projectRef, objectPath, dstPath, fsys) + }); err != nil { + return err + } + if count == 0 { + return errors.New("Object not found: " + remotePath) + } + return nil +} + +func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error { + noSlash := strings.TrimSuffix(remotePath, "/") + // Check if directory exists on remote + dirExists := false + fileExists := false + if err := ls.IterateStoragePaths(ctx, projectRef, noSlash, func(objectName string) error { + if objectName == path.Base(noSlash) { + fileExists = true + } + if objectName == path.Base(noSlash)+"/" { + dirExists = true + } + return nil + }); err != nil { + return err + } + baseName := filepath.Base(localPath) + return afero.Walk(fsys, localPath, func(filePath string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + relPath, err := filepath.Rel(localPath, filePath) + if err != nil { + return err + } + dstPath := remotePath + // Copying single file + if relPath == "." { + _, prefix := storage.SplitBucketPrefix(dstPath) + if IsDir(prefix) || (dirExists && !fileExists) { + dstPath = path.Join(dstPath, info.Name()) + } + } else { + if baseName != "." && (dirExists || len(noSlash) == 0) { + dstPath = path.Join(dstPath, baseName) + } + dstPath = path.Join(dstPath, relPath) + } + fmt.Fprintln(os.Stderr, "Uploading:", filePath, "=>", dstPath) + err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys) + if err != nil && strings.Contains(err.Error(), `"error":"Bucket not found"`) { + // Retry after creating bucket + if bucket, prefix := storage.SplitBucketPrefix(dstPath); len(prefix) > 0 { + if _, err := client.CreateStorageBucket(ctx, projectRef, bucket); err != nil { + return err + } + err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys) + } + } + return err + }) +} + +func IsDir(objectPrefix string) bool { + return len(objectPrefix) == 0 || strings.HasSuffix(objectPrefix, "/") +} diff --git a/internal/storage/cp/cp_test.go b/internal/storage/cp/cp_test.go new file mode 100644 index 000000000..aad264c57 --- /dev/null +++ b/internal/storage/cp/cp_test.go @@ -0,0 +1,584 @@ +package cp + +import ( + "context" + "io/fs" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageCP(t *testing.T) { + t.Run("copy local to remote", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := Run(context.Background(), utils.ProjectRefPath, "ss:///private/file", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on missing file", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "abstract.pdf", "ss:///private", true, fsys) + // Check error + assert.ErrorIs(t, err, fs.ErrNotExist) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("copy remote to local", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := Run(context.Background(), "ss:///private/file", "abstract.pdf", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "abstract.pdf") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("throws error on missing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "ss:///private", ".", true, fsys) + // Check error + assert.ErrorContains(t, err, "Object not found: /private") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on invalid src", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ":", ".", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on invalid dst", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ".", ":", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ".", ".", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("throws error on unsupported operation", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), ".", ".", false, fsys) + // Check error + assert.ErrorIs(t, err, errUnsupportedOperation) + }) +} + +func TestUploadAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("uploads directory to new bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON(client.CreateBucketResponse{Name: "tmp"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on failure to create bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/tmp/readme.md"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", "/tmp", fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads directory to existing prefix", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + require.NoError(t, afero.WriteFile(fsys, "/tmp/docs/api.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "dir", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/dir/tmp/readme.md"). + Reply(http.StatusOK) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/dir/tmp/docs/api.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "/private/dir/", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads file to existing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/readme.md"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "private", "/tmp/readme.md", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("uploads file to existing object", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "/tmp/readme.md", []byte{}, 0644)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + fileObject := mockFile + fileObject.Name = "file" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{fileObject}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/private/file"). + Reply(http.StatusOK) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "private/file", "/tmp/readme.md", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + err := UploadStorageObjectAll(context.Background(), projectRef, "", ".", fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestDownloadAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("downloads buckets to existing directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "", "/", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.True(t, exists) + exists, err = afero.DirExists(fsys, "/test") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("downloads empty bucket to new directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Not Found"}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "/private", "/tmp", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + exists, err = afero.DirExists(fsys, "/tmp") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("throws error on empty directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "private/dir/", "/", fsys) + // Check error + assert.ErrorContains(t, err, "Object not found: private/dir/") + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("downloads objects to existing directory", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }, mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/tmp/abstract.pdf"). + Reply(http.StatusOK) + // Lists /private/tmp/docs directory + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/tmp/docs/readme.md"). + Reply(http.StatusOK) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "private/tmp/", "/", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "/tmp/abstract.pdf") + assert.NoError(t, err) + assert.True(t, exists) + exists, err = afero.Exists(fsys, "/tmp/docs/readme.md") + assert.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("downloads object to existing file", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/object/private/abstract.pdf"). + Reply(http.StatusOK) + // Run test + err := DownloadStorageObjectAll(context.Background(), projectRef, "/private/abstract.pdf", "/tmp/file", fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.DirExists(fsys, "/private") + assert.NoError(t, err) + assert.False(t, exists) + exists, err = afero.Exists(fsys, "/tmp/file") + assert.NoError(t, err) + assert.True(t, exists) + }) +} diff --git a/internal/storage/ls/ls.go b/internal/storage/ls/ls.go new file mode 100644 index 000000000..079dd74b5 --- /dev/null +++ b/internal/storage/ls/ls.go @@ -0,0 +1,136 @@ +package ls + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/utils" +) + +func Run(ctx context.Context, objectPath string, recursive bool, fsys afero.Fs) error { + remotePath, err := storage.ParseStorageURL(objectPath) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + callback := func(objectPath string) error { + fmt.Println(objectPath) + return nil + } + if recursive { + return IterateStoragePathsAll(ctx, projectRef, remotePath, callback) + } + return IterateStoragePaths(ctx, projectRef, remotePath, callback) +} + +func ListStoragePaths(ctx context.Context, projectRef, remotePath string) ([]string, error) { + var result []string + err := IterateStoragePaths(ctx, projectRef, remotePath, func(objectName string) error { + result = append(result, objectName) + return nil + }) + return result, err +} + +func IterateStoragePaths(ctx context.Context, projectRef, remotePath string, callback func(objectName string) error) error { + bucket, prefix := storage.SplitBucketPrefix(remotePath) + if len(bucket) == 0 || (len(prefix) == 0 && !strings.HasSuffix(remotePath, "/")) { + buckets, err := client.ListStorageBuckets(ctx, projectRef) + if err != nil { + return err + } + for _, b := range buckets { + if strings.HasPrefix(b.Name, bucket) { + if err := callback(b.Name + "/"); err != nil { + return err + } + } + } + } else { + pages := 1 + for i := 0; i < pages; i++ { + objects, err := client.ListStorageObjects(ctx, projectRef, bucket, prefix, i) + if err != nil { + return err + } + for _, o := range objects { + name := o.Name + if o.Id == nil { + name += "/" + } + if err := callback(name); err != nil { + return err + } + } + if len(objects) == client.PAGE_LIMIT { + // TODO: show interactive prompt? + fmt.Fprintln(os.Stderr, "Loading page:", pages) + pages++ + } + } + } + return nil +} + +// Expects remotePath to be terminated by "/" +func ListStoragePathsAll(ctx context.Context, projectRef, remotePath string) ([]string, error) { + var result []string + err := IterateStoragePathsAll(ctx, projectRef, remotePath, func(objectPath string) error { + result = append(result, objectPath) + return nil + }) + return result, err +} + +func IterateStoragePathsAll(ctx context.Context, projectRef, remotePath string, callback func(objectPath string) error) error { + basePath := remotePath + if !strings.HasSuffix(remotePath, "/") { + basePath, _ = path.Split(remotePath) + } + // BFS so we can list paths in increasing depth + dirQueue := make([]string, 0) + // We don't know if user passed in a directory or file, so query storage first. + if err := IterateStoragePaths(ctx, projectRef, remotePath, func(objectName string) error { + objectPath := basePath + objectName + if strings.HasSuffix(objectName, "/") { + dirQueue = append(dirQueue, objectPath) + return nil + } + return callback(objectPath) + }); err != nil { + return err + } + for len(dirQueue) > 0 { + dirPath := dirQueue[len(dirQueue)-1] + dirQueue = dirQueue[:len(dirQueue)-1] + empty := true + if err := IterateStoragePaths(ctx, projectRef, dirPath, func(objectName string) error { + empty = false + objectPath := dirPath + objectName + if strings.HasSuffix(objectName, "/") { + dirQueue = append(dirQueue, objectPath) + return nil + } + return callback(objectPath) + }); err != nil { + return err + } + // Also report empty buckets + bucket, prefix := storage.SplitBucketPrefix(dirPath) + if empty && len(prefix) == 0 { + if err := callback(bucket + "/"); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/storage/ls/ls_test.go b/internal/storage/ls/ls_test.go new file mode 100644 index 000000000..cae6542d3 --- /dev/null +++ b/internal/storage/ls/ls_test.go @@ -0,0 +1,395 @@ +package ls + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageLS(t *testing.T) { + t.Run("lists buckets", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{}) + // Run test + err := Run(context.Background(), "ss:///", false, fsys) + // Check error + assert.NoError(t, err) + }) + + t.Run("throws error on invalid URL", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "", false, fsys) + // Check error + assert.ErrorIs(t, err, storage.ErrInvalidURL) + }) + + t.Run("throws error on invalid project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("lists objects recursive", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := Run(context.Background(), "ss:///", true, fsys) + // Check error + assert.NoError(t, err) + }) +} + +func TestListStoragePaths(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("lists bucket paths by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "te") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"test/"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on bucket service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("lists object paths by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "folder", + }, mockFile}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "bucket/") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"folder/", "abstract.pdf"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on object service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "bucket/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("lists object paths with pagination", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + expected := make([]string, client.PAGE_LIMIT) + resp := make([]client.ObjectResponse, client.PAGE_LIMIT) + for i := 0; i < len(resp); i++ { + resp[i] = client.ObjectResponse{Name: fmt.Sprintf("dir_%d", i)} + expected[i] = resp[i].Name + "/" + } + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "dir", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON(resp) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/bucket"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "dir", + Limit: client.PAGE_LIMIT, + Offset: client.PAGE_LIMIT, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + paths, err := ListStoragePaths(context.Background(), projectRef, "/bucket/dir") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, expected, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestListStoragePathsAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("lists nested object paths", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List buckets + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Get("/storage/v1/bucket"). + Reply(http.StatusOK). + JSON([]client.BucketResponse{{ + Id: "test", + Name: "test", + Public: true, + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }, { + Id: "private", + Name: "private", + CreatedAt: "2023-10-13T17:48:58.491Z", + UpdatedAt: "2023-10-13T17:48:58.491Z", + }}) + // List folders + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "folder", + }}) + // List files + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "folder/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + // Run test + paths, err := ListStoragePathsAll(context.Background(), projectRef, "") + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"private/folder/abstract.pdf", "test/"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("returns partial result on error", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List folders + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "error", + }, mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "empty/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "error/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusServiceUnavailable) + // Run test + paths, err := ListStoragePathsAll(context.Background(), projectRef, "private/") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.ElementsMatch(t, []string{"private/abstract.pdf"}, paths) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} diff --git a/internal/storage/mv/mv.go b/internal/storage/mv/mv.go new file mode 100644 index 000000000..1fe1b440f --- /dev/null +++ b/internal/storage/mv/mv.go @@ -0,0 +1,88 @@ +package mv + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +var ( + errUnsupportedMove = errors.New("Moving between buckets is unsupported") + errMissingPath = errors.New("You must specify an object path") +) + +func Run(ctx context.Context, src, dst string, recursive bool, fsys afero.Fs) error { + srcParsed, err := storage.ParseStorageURL(src) + if err != nil { + return err + } + dstParsed, err := storage.ParseStorageURL(dst) + if err != nil { + return err + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + srcBucket, srcPrefix := storage.SplitBucketPrefix(srcParsed) + dstBucket, dstPrefix := storage.SplitBucketPrefix(dstParsed) + if len(srcPrefix) == 0 && len(dstPrefix) == 0 { + return errMissingPath + } + if srcBucket != dstBucket { + return errUnsupportedMove + } + fmt.Fprintln(os.Stderr, "Moving object:", srcParsed, "=>", dstParsed) + data, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, dstPrefix) + if err == nil { + fmt.Fprintln(os.Stderr, data.Message) + } else if strings.Contains(err.Error(), `"error":"not_found"`) && recursive { + return MoveStorageObjectAll(ctx, projectRef, srcParsed+"/", dstParsed) + } + return err +} + +// Expects srcPath to be terminated by "/" +func MoveStorageObjectAll(ctx context.Context, projectRef, srcPath, dstPath string) error { + _, dstPrefix := storage.SplitBucketPrefix(dstPath) + // Cannot iterate because pagination result may be updated during move + count := 0 + queue := make([]string, 0) + queue = append(queue, srcPath) + for len(queue) > 0 { + dirPath := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ls.ListStoragePaths(ctx, projectRef, dirPath) + if err != nil { + return err + } + for _, objectName := range paths { + objectPath := dirPath + objectName + if strings.HasSuffix(objectName, "/") { + queue = append(queue, objectPath) + continue + } + count++ + relPath := strings.TrimPrefix(objectPath, srcPath) + srcBucket, srcPrefix := storage.SplitBucketPrefix(objectPath) + absPath := path.Join(dstPrefix, relPath) + fmt.Fprintln(os.Stderr, "Moving object:", objectPath, "=>", path.Join(dstPath, relPath)) + if _, err := client.MoveStorageObject(ctx, projectRef, srcBucket, srcPrefix, absPath); err != nil { + return err + } + } + } + if count == 0 { + return errors.New("Object not found: " + srcPath) + } + return nil +} diff --git a/internal/storage/mv/mv_test.go b/internal/storage/mv/mv_test.go new file mode 100644 index 000000000..609bca21e --- /dev/null +++ b/internal/storage/mv/mv_test.go @@ -0,0 +1,374 @@ +package mv + +import ( + "context" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageMV(t *testing.T) { + t.Run("moves single object", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "readme.md", + DestinationKey: "docs/file", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := Run(context.Background(), "ss:///private/readme.md", "ss:///private/docs/file", false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves directory when recursive", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "", + DestinationKey: "docs", + }). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "not_found"}) + // List bucket /private/ + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "abstract.pdf", + DestinationKey: "docs/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := Run(context.Background(), "ss:///private", "ss:///private/docs", true, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on invalid src", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), ":", "ss:///", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on invalid dst", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", ":", false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), "ss:///", "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("throws error on missing object path", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), "ss:///", "ss:///", false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingPath) + }) + + t.Run("throws error on bucket mismatch", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Run test + err := Run(context.Background(), "ss:///bucket/docs", "ss:///private", false, fsys) + // Check error + assert.ErrorIs(t, err, errUnsupportedMove) + }) +} + +func TestMoveAll(t *testing.T) { + // Setup valid project ref + projectRef := apitest.RandomProjectRef() + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + + t.Run("rename directory within bucket", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }, mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/abstract.pdf", + DestinationKey: "dir/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Lists /private/tmp/docs directory + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/docs/readme.md", + DestinationKey: "dir/docs/readme.md", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private/dir") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves object into directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/ bucket + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "abstract.pdf", + DestinationKey: "dir/abstract.pdf", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/", "private/dir") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("moves object out of directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Lists /private/tmp/ directory + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + JSON(client.MoveObjectRequest{ + BucketId: "private", + SourceKey: "tmp/readme.md", + DestinationKey: "readme.md", + }). + Reply(http.StatusOK). + JSON(client.MoveObjectResponse{Message: "Successfully moved"}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on move failure", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/move"). + Reply(http.StatusServiceUnavailable) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on missing object", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := MoveStorageObjectAll(context.Background(), projectRef, "private/tmp/", "private") + // Check error + assert.ErrorContains(t, err, "Object not found: private/tmp/") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} diff --git a/internal/storage/rm/rm.go b/internal/storage/rm/rm.go new file mode 100644 index 000000000..ed0cf6e74 --- /dev/null +++ b/internal/storage/rm/rm.go @@ -0,0 +1,127 @@ +package rm + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/afero" + "github.com/supabase/cli/internal/storage" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/storage/cp" + "github.com/supabase/cli/internal/storage/ls" + "github.com/supabase/cli/internal/utils" +) + +var ( + errMissingObject = errors.New("Object not found") + errMissingBucket = errors.New("You must specify a bucket to delete.") + errMissingFlag = errors.New("You must specify -r flag to delete directories.") +) + +type PrefixGroup struct { + Bucket string + Prefixes []string +} + +func Run(ctx context.Context, paths []string, recursive bool, fsys afero.Fs) error { + // Group paths by buckets + groups := map[string][]string{} + for _, objectPath := range paths { + remotePath, err := storage.ParseStorageURL(objectPath) + if err != nil { + return err + } + bucket, prefix := storage.SplitBucketPrefix(remotePath) + // Ignore attempts to delete all buckets + if len(bucket) == 0 { + return errMissingBucket + } + if cp.IsDir(prefix) && !recursive { + return errMissingFlag + } + groups[bucket] = append(groups[bucket], prefix) + } + projectRef, err := utils.LoadProjectRef(fsys) + if err != nil { + return err + } + for bucket, prefixes := range groups { + confirm := fmt.Sprintf("Confirm deleting files in bucket %v?", utils.Bold(bucket)) + if shouldDelete := utils.PromptYesNo(confirm, true, os.Stdin); !shouldDelete { + continue + } + // Always try deleting first in case the paths resolve to extensionless files + fmt.Fprintln(os.Stderr, "Deleting objects:", prefixes) + removed, err := client.DeleteStorageObjects(ctx, projectRef, bucket, prefixes) + if err != nil { + return err + } + set := map[string]struct{}{} + for _, object := range removed { + set[object.Name] = struct{}{} + } + for _, prefix := range prefixes { + if _, ok := set[prefix]; ok { + continue + } + if !recursive { + fmt.Fprintln(os.Stderr, "Object not found:", prefix) + continue + } + if len(prefix) > 0 { + prefix += "/" + } + if err := RemoveStoragePathAll(ctx, projectRef, bucket, prefix); err != nil { + return err + } + } + } + return nil +} + +// Expects prefix to be terminated by "/" or "" +func RemoveStoragePathAll(ctx context.Context, projectRef, bucket, prefix string) error { + // We must remove one directory at a time to avoid breaking pagination result + queue := make([]string, 0) + queue = append(queue, prefix) + for len(queue) > 0 { + dirPrefix := queue[len(queue)-1] + queue = queue[:len(queue)-1] + paths, err := ls.ListStoragePaths(ctx, projectRef, fmt.Sprintf("/%s/%s", bucket, dirPrefix)) + if err != nil { + return err + } + if len(paths) == 0 && len(prefix) > 0 { + return fmt.Errorf("%w: %s/%s", errMissingObject, bucket, prefix) + } + var files []string + for _, objectName := range paths { + objectPrefix := dirPrefix + objectName + if strings.HasSuffix(objectName, "/") { + queue = append(queue, objectPrefix) + } else { + files = append(files, objectPrefix) + } + } + if len(files) > 0 { + fmt.Fprintln(os.Stderr, "Deleting objects:", files) + if _, err := client.DeleteStorageObjects(ctx, projectRef, bucket, files); err != nil { + return err + } + } + } + if len(prefix) == 0 { + fmt.Fprintln(os.Stderr, "Deleting bucket:", bucket) + if data, err := client.DeleteStorageBucket(ctx, projectRef, bucket); err == nil { + fmt.Fprintln(os.Stderr, data.Message) + } else if strings.Contains(err.Error(), `"error":"Bucket not found"`) { + fmt.Fprintln(os.Stderr, "Bucket not found:", bucket) + } else { + return err + } + } + return nil +} diff --git a/internal/storage/rm/rm_test.go b/internal/storage/rm/rm_test.go new file mode 100644 index 000000000..49d4e3da2 --- /dev/null +++ b/internal/storage/rm/rm_test.go @@ -0,0 +1,376 @@ +package rm + +import ( + "context" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/storage/client" + "github.com/supabase/cli/internal/testing/apitest" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/api" + "gopkg.in/h2non/gock.v1" +) + +var mockFile = client.ObjectResponse{ + Name: "abstract.pdf", + Id: utils.Ptr("9b7f9f48-17a6-4ca8-b14a-39b0205a63e9"), + UpdatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + CreatedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + LastAccessedAt: utils.Ptr("2023-10-13T18:08:22.068Z"), + Metadata: &client.ObjectMetadata{ + ETag: `"887ea9be3c68e6f2fca7fd2d7c77d8fe"`, + Size: 82702, + Mimetype: "application/pdf", + CacheControl: "max-age=3600", + LastModified: "2023-10-13T18:08:22.000Z", + ContentLength: 82702, + HttpStatusCode: 200, + }, +} + +func TestStorageRM(t *testing.T) { + t.Run("throws error on invalid url", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{":"}, false, fsys) + // Check error + assert.ErrorContains(t, err, "missing protocol scheme") + }) + + t.Run("throws error on missing bucket", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{"ss:///"}, false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingBucket) + }) + + t.Run("throws error on missing flag", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{"ss:///private/"}, false, fsys) + // Check error + assert.ErrorIs(t, err, errMissingFlag) + }) + + t.Run("throws error on missing project", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Run test + err := Run(context.Background(), []string{}, false, fsys) + // Check error + assert.ErrorIs(t, err, utils.ErrNotLinked) + }) + + t.Run("removes multiple objects", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "abstract.pdf", + "docs/readme.md", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := Run(context.Background(), []string{ + "ss:///private/abstract.pdf", + "ss:///private/docs/readme.md", + }, false, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("removes buckets and directories", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // Delete /test/ bucket + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/test"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/test"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/bucket/test"). + Reply(http.StatusNotFound). + JSON(map[string]string{"error": "Bucket not found"}) + // Delete /private/docs/ directory + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "docs", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "docs/abstract.pdf", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := Run(context.Background(), []string{ + "ss:///test", + "ss:///private/docs", + }, true, fsys) + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on delete failure", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + projectRef := apitest.RandomProjectRef() + require.NoError(t, afero.WriteFile(fsys, utils.ProjectRefPath, []byte(projectRef), 0644)) + // Setup valid access token + token := apitest.RandomAccessToken(t) + t.Setenv("SUPABASE_ACCESS_TOKEN", string(token)) + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := Run(context.Background(), []string{"ss:///private"}, true, fsys) + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} + +func TestRemoveAll(t *testing.T) { + projectRef := apitest.RandomProjectRef() + + t.Run("removes objects by prefix", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + // List /private/tmp/ + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{{ + Name: "docs", + }}) + // List /private/docs/ + readme := mockFile + readme.Name = "readme.md" + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + JSON(client.ListObjectsQuery{ + Prefix: "tmp/docs/", + Search: "", + Limit: client.PAGE_LIMIT, + Offset: 0, + }). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile, readme}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + JSON(client.DeleteObjectsRequest{Prefixes: []string{ + "tmp/docs/abstract.pdf", + "tmp/docs/readme.md", + }}). + Reply(http.StatusOK). + JSON([]client.DeleteObjectsResponse{{ + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "abstract.pdf", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }, { + BucketId: "private", + Version: "cf5c5c53-ee73-4806-84e3-7d92c954b436", + Name: "readme.md", + Id: "9b7f9f48-17a6-4ca8-b14a-39b0205a63e9", + UpdatedAt: "2023-10-13T18:08:22.068Z", + CreatedAt: "2023-10-13T18:08:22.068Z", + LastAccessedAt: "2023-10-13T18:08:22.068Z", + }}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "tmp/") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("removes empty bucket", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/bucket/private"). + Reply(http.StatusOK). + JSON(client.DeleteBucketResponse{Message: "Successfully deleted"}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.NoError(t, err) + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on empty directory", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{}) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "dir") + // Check error + assert.ErrorContains(t, err, "Object not found: private/dir") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on service unavailable", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) + + t.Run("throws error on delete failure", func(t *testing.T) { + // Setup mock api + defer gock.OffAll() + gock.New(utils.DefaultApiHost). + Get("/v1/projects/" + projectRef + "/api-keys"). + Reply(http.StatusOK). + JSON([]api.ApiKeyResponse{{ + Name: "service_role", + ApiKey: "service-key", + }}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Post("/storage/v1/object/list/private"). + Reply(http.StatusOK). + JSON([]client.ObjectResponse{mockFile}) + gock.New("https://" + utils.GetSupabaseHost(projectRef)). + Delete("/storage/v1/object/private"). + Reply(http.StatusServiceUnavailable) + // Run test + err := RemoveStoragePathAll(context.Background(), projectRef, "private", "") + // Check error + assert.ErrorContains(t, err, "Error status 503:") + assert.Empty(t, apitest.ListUnmatchedRequests()) + }) +} diff --git a/internal/storage/scheme.go b/internal/storage/scheme.go new file mode 100644 index 000000000..aabf23017 --- /dev/null +++ b/internal/storage/scheme.go @@ -0,0 +1,37 @@ +package storage + +import ( + "errors" + "net/url" + "strings" +) + +const STORAGE_SCHEME = "ss" + +var ErrInvalidURL = errors.New("URL must match pattern ss:///bucket/[prefix]") + +func ParseStorageURL(objectURL string) (string, error) { + parsed, err := url.Parse(objectURL) + if err != nil { + return "", err + } + if strings.ToLower(parsed.Scheme) != STORAGE_SCHEME || len(parsed.Path) == 0 || len(parsed.Host) > 0 { + return "", ErrInvalidURL + } + return parsed.Path, nil +} + +func SplitBucketPrefix(objectPath string) (string, string) { + if objectPath == "" || objectPath == "/" { + return "", "" + } + start := 0 + if objectPath[0] == '/' { + start = 1 + } + sep := strings.IndexByte(objectPath[start:], '/') + if sep < 0 { + return objectPath[start:], "" + } + return objectPath[start : sep+start], objectPath[sep+start+1:] +} diff --git a/internal/storage/scheme_test.go b/internal/storage/scheme_test.go new file mode 100644 index 000000000..9427ffcb6 --- /dev/null +++ b/internal/storage/scheme_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseStorageURL(t *testing.T) { + t.Run("parses valid url", func(t *testing.T) { + path, err := ParseStorageURL("ss:///bucket/folder/name.png") + assert.NoError(t, err) + assert.Equal(t, path, "/bucket/folder/name.png") + }) + + t.Run("throws error on invalid host", func(t *testing.T) { + path, err := ParseStorageURL("ss://bucket") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on missing path", func(t *testing.T) { + path, err := ParseStorageURL("ss:") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid scheme", func(t *testing.T) { + path, err := ParseStorageURL(".") + assert.ErrorIs(t, err, ErrInvalidURL) + assert.Empty(t, path) + }) + + t.Run("throws error on invalid url", func(t *testing.T) { + path, err := ParseStorageURL(":") + assert.ErrorContains(t, err, "missing protocol scheme") + assert.Empty(t, path) + }) +} + +func TestSplitBucketPrefix(t *testing.T) { + t.Run("splits empty path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits root path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/") + assert.Equal(t, bucket, "") + assert.Equal(t, prefix, "") + }) + + t.Run("splits no slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits prefix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits suffix slash", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("bucket/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "") + }) + + t.Run("splits file path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/name.png") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/name.png") + }) + + t.Run("splits dir path", func(t *testing.T) { + bucket, prefix := SplitBucketPrefix("/bucket/folder/") + assert.Equal(t, bucket, "bucket") + assert.Equal(t, prefix, "folder/") + }) +} diff --git a/internal/utils/misc.go b/internal/utils/misc.go index b986c19e9..f64f708c5 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -326,3 +326,7 @@ func ValidateFunctionSlug(slug string) error { return nil } + +func Ptr[T any](v T) *T { + return &v +}