From 95e433e0562d50bdec8064b011bd90c74deec210 Mon Sep 17 00:00:00 2001 From: chaitu Date: Wed, 3 Jan 2024 20:45:54 +0530 Subject: [PATCH] fix: added auto detect content type Signed-off-by: chaitu --- cmd/lakectl/cmd/fs_download.go | 2 +- cmd/lakectl/cmd/fs_upload.go | 2 +- cmd/lakectl/cmd/local_checkout.go | 2 +- cmd/lakectl/cmd/local_clone.go | 2 +- cmd/lakectl/cmd/local_commit.go | 5 +++-- cmd/lakectl/cmd/local_pull.go | 2 +- cmd/lakectl/cmd/root.go | 9 +++++++++ pkg/local/sync.go | 19 ++++++++++++------- 8 files changed, 29 insertions(+), 14 deletions(-) diff --git a/cmd/lakectl/cmd/fs_download.go b/cmd/lakectl/cmd/fs_download.go index 8cd6255512b..9f07c1e1528 100644 --- a/cmd/lakectl/cmd/fs_download.go +++ b/cmd/lakectl/cmd/fs_download.go @@ -88,7 +88,7 @@ var fsDownloadCmd = &cobra.Command{ } s := local.NewSyncManager(ctx, client, syncFlags) - err := s.Sync(dest, remote, ch) + err := s.Sync(dest, remote, ch, false) if err != nil { DieErr(err) } diff --git a/cmd/lakectl/cmd/fs_upload.go b/cmd/lakectl/cmd/fs_upload.go index 03e429b37c5..36fff01e6da 100644 --- a/cmd/lakectl/cmd/fs_upload.go +++ b/cmd/lakectl/cmd/fs_upload.go @@ -61,7 +61,7 @@ var fsUploadCmd = &cobra.Command{ if err != nil { DieErr(err) } - err = s.Sync(fullPath, pathURI, c) + err = s.Sync(fullPath, pathURI, c, false) if err != nil { DieErr(err) } diff --git a/cmd/lakectl/cmd/local_checkout.go b/cmd/lakectl/cmd/local_checkout.go index b37f92ab724..947159198a0 100644 --- a/cmd/lakectl/cmd/local_checkout.go +++ b/cmd/lakectl/cmd/local_checkout.go @@ -100,7 +100,7 @@ func localCheckout(cmd *cobra.Command, localPath string, specifiedRef string, co } } }() - err = syncMgr.Sync(idx.LocalPath(), currentBase, c) + err = syncMgr.Sync(idx.LocalPath(), currentBase, c, false) if err != nil { DieErr(err) } diff --git a/cmd/lakectl/cmd/local_clone.go b/cmd/lakectl/cmd/local_clone.go index b1afac88993..66b2afa14a1 100644 --- a/cmd/lakectl/cmd/local_clone.go +++ b/cmd/lakectl/cmd/local_clone.go @@ -88,7 +88,7 @@ var localCloneCmd = &cobra.Command{ } sigCtx := localHandleSyncInterrupt(ctx, idx, string(cloneOperation)) s := local.NewSyncManager(sigCtx, client, syncFlags) - err = s.Sync(localPath, stableRemote, ch) + err = s.Sync(localPath, stableRemote, ch, false) if err != nil { DieErr(err) } diff --git a/cmd/lakectl/cmd/local_commit.go b/cmd/lakectl/cmd/local_commit.go index 0bcd4f627fb..e7d3ee57ceb 100644 --- a/cmd/lakectl/cmd/local_commit.go +++ b/cmd/lakectl/cmd/local_commit.go @@ -33,7 +33,7 @@ var localCommitCmd = &cobra.Command{ _, localPath := getSyncArgs(args, false, false) syncFlags := getSyncFlags(cmd, client) message, kvPairs := getCommitFlags(cmd) - + detectContentTypeFlag := getDetectContentTypeFlag(cmd) idx, err := local.ReadIndex(localPath) if err != nil { DieErr(err) @@ -112,7 +112,7 @@ var localCommitCmd = &cobra.Command{ }() sigCtx := localHandleSyncInterrupt(cmd.Context(), idx, string(commitOperation)) s := local.NewSyncManager(sigCtx, client, syncFlags) - err = s.Sync(idx.LocalPath(), remote, c) + err = s.Sync(idx.LocalPath(), remote, c, detectContentTypeFlag) if err != nil { DieErr(err) } @@ -172,5 +172,6 @@ var localCommitCmd = &cobra.Command{ func init() { withCommitFlags(localCommitCmd, false) withSyncFlags(localCommitCmd) + withDetectContentFlag(localCommitCmd, false) localCmd.AddCommand(localCommitCmd) } diff --git a/cmd/lakectl/cmd/local_pull.go b/cmd/lakectl/cmd/local_pull.go index ff661428cfe..b00547c4e26 100644 --- a/cmd/lakectl/cmd/local_pull.go +++ b/cmd/lakectl/cmd/local_pull.go @@ -67,7 +67,7 @@ var localPullCmd = &cobra.Command{ }) sigCtx := localHandleSyncInterrupt(cmd.Context(), idx, string(pullOperation)) s := local.NewSyncManager(sigCtx, client, syncFlags) - err = s.Sync(idx.LocalPath(), newBase, c) + err = s.Sync(idx.LocalPath(), newBase, c, false) if err != nil { DieErr(err) } diff --git a/cmd/lakectl/cmd/root.go b/cmd/lakectl/cmd/root.go index b5c41302337..0bd3227c2b3 100644 --- a/cmd/lakectl/cmd/root.go +++ b/cmd/lakectl/cmd/root.go @@ -140,6 +140,7 @@ const ( allowEmptyMsgFlagName = "allow-empty-message" fmtErrEmptyMsg = `commit with no message without specifying the "--allow-empty-message" flag` metaFlagName = "meta" + detectContentType = "detect-content-type" ) func withRecursiveFlag(cmd *cobra.Command, usage string) { @@ -222,6 +223,10 @@ func withMetadataFlag(cmd *cobra.Command) { cmd.Flags().StringSlice(metaFlagName, []string{}, "key value pair in the form of key=value") } +func withDetectContentFlag(cmd *cobra.Command, detectContentFlag bool) { + cmd.Flags().Bool(detectContentType, detectContentFlag, "detect content type") +} + func withCommitFlags(cmd *cobra.Command, allowEmptyMessage bool) { withMessageFlags(cmd, allowEmptyMessage) withMetadataFlag(cmd) @@ -242,6 +247,10 @@ func getCommitFlags(cmd *cobra.Command) (string, map[string]string) { return message, kvPairs } +func getDetectContentTypeFlag(cmd *cobra.Command) bool { + return Must(cmd.Flags().GetBool(detectContentType)) +} + func getKV(cmd *cobra.Command, name string) (map[string]string, error) { //nolint:unparam kvList, err := cmd.Flags().GetStringSlice(name) if err != nil { diff --git a/pkg/local/sync.go b/pkg/local/sync.go index 76e19a66577..5233baf87cf 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "mime" "net/http" "os" "path/filepath" @@ -73,7 +74,7 @@ func NewSyncManager(ctx context.Context, client *apigen.ClientWithResponses, fla // Sync - sync changes between remote and local directory given the Changes channel. // For each change, will apply download, upload or delete according to the change type and change source -func (s *SyncManager) Sync(rootPath string, remote *uri.URI, changeSet <-chan *Change) error { +func (s *SyncManager) Sync(rootPath string, remote *uri.URI, changeSet <-chan *Change, detectContentType bool) error { s.progressBar.Start() defer s.progressBar.Stop() @@ -81,7 +82,7 @@ func (s *SyncManager) Sync(rootPath string, remote *uri.URI, changeSet <-chan *C for i := 0; i < s.flags.Parallelism; i++ { wg.Go(func() error { for change := range changeSet { - if err := s.apply(ctx, rootPath, remote, change); err != nil { + if err := s.apply(ctx, rootPath, remote, change, detectContentType); err != nil { return err } } @@ -95,7 +96,7 @@ func (s *SyncManager) Sync(rootPath string, remote *uri.URI, changeSet <-chan *C return err } -func (s *SyncManager) apply(ctx context.Context, rootPath string, remote *uri.URI, change *Change) error { +func (s *SyncManager) apply(ctx context.Context, rootPath string, remote *uri.URI, change *Change, detectContentType bool) error { switch change.Type { case ChangeTypeAdded, ChangeTypeModified: switch change.Source { @@ -106,7 +107,7 @@ func (s *SyncManager) apply(ctx context.Context, rootPath string, remote *uri.UR } case ChangeSourceLocal: // we wrote something, upload it! - if err := s.upload(ctx, rootPath, remote, change.Path); err != nil { + if err := s.upload(ctx, rootPath, remote, change.Path, detectContentType); err != nil { return fmt.Errorf("upload %s failed: %w", change.Path, err) } default: @@ -237,7 +238,11 @@ func (s *SyncManager) download(ctx context.Context, rootPath string, remote *uri return err } -func (s *SyncManager) upload(ctx context.Context, rootPath string, remote *uri.URI, path string) error { +func (s *SyncManager) upload(ctx context.Context, rootPath string, remote *uri.URI, path string, detectContentType bool) error { + var contentType string + if detectContentType { + contentType = mime.TypeByExtension(filepath.Ext(path)) + } source := filepath.Join(rootPath, path) if err := fileutil.VerifySafeFilename(source); err != nil { return err @@ -276,12 +281,12 @@ func (s *SyncManager) upload(ctx context.Context, rootPath string, remote *uri.U } if s.flags.Presign { _, err = helpers.ClientUploadPreSign( - ctx, s.client, remote.Repository, remote.Ref, dest, metadata, "", reader) + ctx, s.client, remote.Repository, remote.Ref, dest, metadata, contentType, reader) return err } // not pre-signed _, err = helpers.ClientUpload( - ctx, s.client, remote.Repository, remote.Ref, dest, metadata, "", reader) + ctx, s.client, remote.Repository, remote.Ref, dest, metadata, contentType, reader) return err }