From bf5a544c522a7e4e39cb9fec554f4921e516188c Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Wed, 18 Dec 2024 10:10:45 +0100 Subject: [PATCH 1/2] Allow to specify cluster for resource download/upload --- cmd/beach/cmd/helpers.go | 8 +++++++- cmd/beach/cmd/resource-download.go | 1 + cmd/beach/cmd/resource-upload.go | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/beach/cmd/helpers.go b/cmd/beach/cmd/helpers.go index 0646859..c85e53a 100644 --- a/cmd/beach/cmd/helpers.go +++ b/cmd/beach/cmd/helpers.go @@ -94,8 +94,14 @@ func retrieveCloudStorageCredentials(instanceIdentifier string, projectNamespace log.Info("Retrieving cloud storage access data from instance") internalHost := "beach@" + instanceIdentifier + "." + projectNamespace + jumpHost := "" + if projectCluster != "" { + jumpHost = "beach@ssh." + projectCluster + ".flownative.cloud" + } else { + jumpHost = "beach@ssh.flownative.cloud" + } output, err := exec.RunCommand("ssh", []string{ - "-J", "beach@ssh.flownative.cloud", internalHost, + "-J", jumpHost, internalHost, "/bin/bash", "-c", "env | grep BEACH_GOOGLE_CLOUD_STORAGE_", }) if err != nil { diff --git a/cmd/beach/cmd/resource-download.go b/cmd/beach/cmd/resource-download.go index a976413..5c7c706 100644 --- a/cmd/beach/cmd/resource-download.go +++ b/cmd/beach/cmd/resource-download.go @@ -59,6 +59,7 @@ Notes: func init() { resourceDownloadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to download from, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'") resourceDownloadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to download from, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'") + resourceDownloadCmd.Flags().StringVar(&projectCluster, "cluster", "", "The cluster name of the Beach instance to download from, eg. 'h9acc4'") resourceDownloadCmd.Flags().StringVar(&bucketName, "bucket", "", "name of the bucket to download resources from") resourceDownloadCmd.Flags().StringVar(&resourcesPath, "resources-path", "", "custom path where to store the downloaded resources, e.g. 'Data/Persistent/Protected'") _ = resourceDownloadCmd.MarkFlagRequired("instance") diff --git a/cmd/beach/cmd/resource-upload.go b/cmd/beach/cmd/resource-upload.go index 9b2d09b..0715e20 100644 --- a/cmd/beach/cmd/resource-upload.go +++ b/cmd/beach/cmd/resource-upload.go @@ -29,7 +29,7 @@ import ( "google.golang.org/api/option" ) -var instanceIdentifier, projectNamespace, bucketName, resourcesPath, resumeWithFile string +var instanceIdentifier, projectNamespace, projectCluster, bucketName, resourcesPath, resumeWithFile string var force bool // resourceUploadCmd represents the resource-upload command @@ -62,6 +62,7 @@ Notes: func init() { resourceUploadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to upload to, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'") resourceUploadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to upload to, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'") + resourceUploadCmd.Flags().StringVar(&projectCluster, "cluster", "", "The cluster name of the Beach instance to upload to, eg. 'h9acc4'") resourceUploadCmd.Flags().BoolVar(&force, "force", false, "Force uploading resources which already exist in the target bucket") resourceUploadCmd.Flags().StringVar(&resumeWithFile, "resume-with-file", "", "If specified, resume uploading resources starting with the given filename, eg. '12dcde4c13142942288c5a973caf0fa720ed2794'") _ = resourceUploadCmd.MarkFlagRequired("instance") From 5fb20b1887cc4080088c921b9739773ab393f58e Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Wed, 18 Dec 2024 10:59:19 +0100 Subject: [PATCH 2/2] Tweak variable naming & declaration Moves variables from non-intuitive places, renames `clusterName` to `clusterIdentifier`. --- cmd/beach/cmd/helpers.go | 8 +++++--- cmd/beach/cmd/resource-download.go | 28 +++++++++++++++------------- cmd/beach/cmd/resource-upload.go | 24 ++++++++++++------------ 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cmd/beach/cmd/helpers.go b/cmd/beach/cmd/helpers.go index c85e53a..e6fa984 100644 --- a/cmd/beach/cmd/helpers.go +++ b/cmd/beach/cmd/helpers.go @@ -33,6 +33,8 @@ import ( asset "github.com/flownative/localbeach/assets" ) +var instanceIdentifier, projectNamespace, clusterIdentifier string + func copyFileFromAssets(src, dst string) (int64, error) { source, err := asset.Assets.Open(src) if err != nil { @@ -90,13 +92,13 @@ func getRelativePersistentResourcePathByHash(hash string) string { } } -func retrieveCloudStorageCredentials(instanceIdentifier string, projectNamespace string) (err error, bucketName string, privateKey []byte) { +func retrieveCloudStorageCredentials(instanceIdentifier string, projectNamespace string, clusterIdentifier string) (err error, bucketName string, privateKey []byte) { log.Info("Retrieving cloud storage access data from instance") internalHost := "beach@" + instanceIdentifier + "." + projectNamespace jumpHost := "" - if projectCluster != "" { - jumpHost = "beach@ssh." + projectCluster + ".flownative.cloud" + if clusterIdentifier != "" { + jumpHost = "beach@ssh." + clusterIdentifier + ".flownative.cloud" } else { jumpHost = "beach@ssh.flownative.cloud" } diff --git a/cmd/beach/cmd/resource-download.go b/cmd/beach/cmd/resource-download.go index 5c7c706..8f9f0f4 100644 --- a/cmd/beach/cmd/resource-download.go +++ b/cmd/beach/cmd/resource-download.go @@ -29,6 +29,8 @@ import ( "path/filepath" ) +var sourceBucketName, targetResourcesPath string + // resourceDownloadCmd represents the resource-download command var resourceDownloadCmd = &cobra.Command{ Use: "resource-download", @@ -59,9 +61,9 @@ Notes: func init() { resourceDownloadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to download from, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'") resourceDownloadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to download from, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'") - resourceDownloadCmd.Flags().StringVar(&projectCluster, "cluster", "", "The cluster name of the Beach instance to download from, eg. 'h9acc4'") - resourceDownloadCmd.Flags().StringVar(&bucketName, "bucket", "", "name of the bucket to download resources from") - resourceDownloadCmd.Flags().StringVar(&resourcesPath, "resources-path", "", "custom path where to store the downloaded resources, e.g. 'Data/Persistent/Protected'") + resourceDownloadCmd.Flags().StringVar(&clusterIdentifier, "cluster", "", "The cluster identifier of the Beach instance to download from, eg. 'h9acc4'") + resourceDownloadCmd.Flags().StringVar(&sourceBucketName, "bucket", "", "name of the bucket to download resources from") + resourceDownloadCmd.Flags().StringVar(&targetResourcesPath, "resources-path", "", "custom path where to store the downloaded resources, e.g. 'Data/Persistent/Protected'") _ = resourceDownloadCmd.MarkFlagRequired("instance") _ = resourceDownloadCmd.MarkFlagRequired("namespace") rootCmd.AddCommand(resourceDownloadCmd) @@ -74,24 +76,24 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) { return } - if resourcesPath == "" { - resourcesPath = sandbox.ProjectDataPersistentResourcesPath + if targetResourcesPath == "" { + targetResourcesPath = sandbox.ProjectDataPersistentResourcesPath } - _, err = os.Stat(resourcesPath) + _, err = os.Stat(targetResourcesPath) if err != nil { - log.Fatal(fmt.Sprintf("The path %v does not exist", resourcesPath)) + log.Fatal(fmt.Sprintf("The path %v does not exist", targetResourcesPath)) return } - err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace) + err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace, clusterIdentifier) if err != nil { log.Fatal(err) return } - if bucketName == "" { - bucketName = bucketNameFromCredentials + if sourceBucketName == "" { + sourceBucketName = bucketNameFromCredentials } ctx := context.Background() @@ -101,9 +103,9 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) { return } - log.Info(fmt.Sprintf("Downloading resources from bucket %v to local directory %v ...", bucketName, resourcesPath)) + log.Info(fmt.Sprintf("Downloading resources from bucket %v to local directory %v ...", sourceBucketName, targetResourcesPath)) - bucket := client.Bucket(bucketName) + bucket := client.Bucket(sourceBucketName) it := bucket.Objects(ctx, nil) for { attributes, err := it.Next() @@ -114,7 +116,7 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) { log.Error(err) } else { source := bucket.Object(attributes.Name) - targetPathAndFilename := filepath.Join(resourcesPath, getRelativePersistentResourcePathByHash(attributes.Name), filepath.Base(attributes.Name)) + targetPathAndFilename := filepath.Join(targetResourcesPath, getRelativePersistentResourcePathByHash(attributes.Name), filepath.Base(attributes.Name)) err = os.MkdirAll(filepath.Dir(targetPathAndFilename), 0755) if err != nil { diff --git a/cmd/beach/cmd/resource-upload.go b/cmd/beach/cmd/resource-upload.go index 0715e20..dbc5132 100644 --- a/cmd/beach/cmd/resource-upload.go +++ b/cmd/beach/cmd/resource-upload.go @@ -29,7 +29,7 @@ import ( "google.golang.org/api/option" ) -var instanceIdentifier, projectNamespace, projectCluster, bucketName, resourcesPath, resumeWithFile string +var targetBucketName, sourceResourcesPath, resumeWithFile string var force bool // resourceUploadCmd represents the resource-upload command @@ -62,7 +62,7 @@ Notes: func init() { resourceUploadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to upload to, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'") resourceUploadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to upload to, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'") - resourceUploadCmd.Flags().StringVar(&projectCluster, "cluster", "", "The cluster name of the Beach instance to upload to, eg. 'h9acc4'") + resourceUploadCmd.Flags().StringVar(&clusterIdentifier, "cluster", "", "The cluster identifier of the Beach instance to upload to, eg. 'h9acc4'") resourceUploadCmd.Flags().BoolVar(&force, "force", false, "Force uploading resources which already exist in the target bucket") resourceUploadCmd.Flags().StringVar(&resumeWithFile, "resume-with-file", "", "If specified, resume uploading resources starting with the given filename, eg. '12dcde4c13142942288c5a973caf0fa720ed2794'") _ = resourceUploadCmd.MarkFlagRequired("instance") @@ -76,23 +76,23 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) { log.Fatal("Could not activate sandbox: ", err) return } - if resourcesPath == "" { - resourcesPath = sandbox.ProjectDataPersistentResourcesPath + if sourceResourcesPath == "" { + sourceResourcesPath = sandbox.ProjectDataPersistentResourcesPath } - _, err = os.Stat(resourcesPath) + _, err = os.Stat(sourceResourcesPath) if err != nil { - log.Fatal("The path %v does not exist", resourcesPath) + log.Fatal("The path %v does not exist", sourceResourcesPath) return } - err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace) + err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace, clusterIdentifier) if err != nil { log.Fatal(err) return } - if bucketName == "" { - bucketName = bucketNameFromCredentials + if targetBucketName == "" { + targetBucketName = bucketNameFromCredentials } ctx := context.Background() @@ -102,10 +102,10 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) { return } - log.Info(fmt.Sprintf("Uploading resources from local directory %v to bucket %v...", resourcesPath, bucketName)) + log.Info(fmt.Sprintf("Uploading resources from local directory %v to bucket %v...", sourceResourcesPath, targetBucketName)) var fileList []string - err = filepath.Walk(resourcesPath, func(path string, f os.FileInfo, err error) error { + err = filepath.Walk(sourceResourcesPath, func(path string, f os.FileInfo, err error) error { if !f.IsDir() { fileList = append(fileList, path) } @@ -116,7 +116,7 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) { return } - bucket := client.Bucket(bucketName) + bucket := client.Bucket(targetBucketName) for _, pathAndFilename := range fileList { filename := filepath.Base(pathAndFilename)