diff --git a/cmd/mindthegap/create/bundle/bundle.go b/cmd/mindthegap/create/bundle/bundle.go new file mode 100644 index 00000000..8f55bb2e --- /dev/null +++ b/cmd/mindthegap/create/bundle/bundle.go @@ -0,0 +1,482 @@ +// Copyright 2021 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" + "helm.sh/helm/v3/pkg/action" + "k8s.io/utils/ptr" + + "github.com/mesosphere/dkp-cli-runtime/core/output" + + "github.com/mesosphere/mindthegap/archive" + "github.com/mesosphere/mindthegap/cleanup" + "github.com/mesosphere/mindthegap/cmd/mindthegap/flags" + "github.com/mesosphere/mindthegap/cmd/mindthegap/utils" + "github.com/mesosphere/mindthegap/config" + "github.com/mesosphere/mindthegap/docker/registry" + "github.com/mesosphere/mindthegap/helm" + "github.com/mesosphere/mindthegap/images" + "github.com/mesosphere/mindthegap/images/authnhelpers" + "github.com/mesosphere/mindthegap/images/httputils" +) + +func NewCommand(out output.Output) *cobra.Command { + var ( + imagesConfigFile string + helmChartsConfigFile string + platforms = flags.NewPlatformsValue("linux/amd64") + outputFile string + overwrite bool + imagePullConcurrency int + ) + + cmd := &cobra.Command{ + Use: "bundle", + Short: "Create a bundle containing container images and/or Helm charts", + PreRunE: func(cmd *cobra.Command, args []string) error { + return cmd.ValidateRequiredFlags() + }, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + helmChartsConfig config.HelmChartsConfig + imagesConfig config.ImagesConfig + ) + if imagesConfigFile != "" { + out.StartOperation("Parsing image bundle config") + cfg, err := config.ParseImagesConfigFile(imagesConfigFile) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return err + } + out.EndOperationWithStatus(output.Success()) + out.V(4).Infof("Images config: %+v", cfg) + imagesConfig = cfg + } + + if helmChartsConfigFile != "" { + out.StartOperation("Parsing Helm chart bundle config") + cfg, err := config.ParseHelmChartsConfigFile(helmChartsConfigFile) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return err + } + out.EndOperationWithStatus(output.Success()) + out.V(4).Infof("Helm charts config: %+v", cfg) + helmChartsConfig = cfg + } + + if !overwrite { + out.StartOperation("Checking if output file already exists") + _, err := os.Stat(outputFile) + switch { + case err == nil: + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "%s already exists: specify --overwrite to overwrite existing file", + outputFile, + ) + case !errors.Is(err, os.ErrNotExist): + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "failed to check if output file %s already exists: %w", + outputFile, + err, + ) + default: + out.EndOperationWithStatus(output.Success()) + } + } + + out.StartOperation("Creating temporary directory") + outputFileAbs, err := filepath.Abs(outputFile) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "failed to determine where to create temporary directory: %w", + err, + ) + } + + cleaner := cleanup.NewCleaner() + defer cleaner.Cleanup() + + tempDir, err := os.MkdirTemp(filepath.Dir(outputFileAbs), ".bundle-*") + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to create temporary directory: %w", err) + } + cleaner.AddCleanupFn(func() { _ = os.RemoveAll(tempDir) }) + + out.EndOperationWithStatus(output.Success()) + + out.StartOperation("Starting temporary Docker registry") + reg, err := registry.NewRegistry(registry.Config{StorageDirectory: tempDir}) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to create local Docker registry: %w", err) + } + go func() { + if err := reg.ListenAndServe(); err != nil { + out.Error(err, "error serving Docker registry") + os.Exit(2) + } + }() + out.EndOperationWithStatus(output.Success()) + + logs.Debug.SetOutput(out.V(4).InfoWriter()) + logs.Warn.SetOutput(out.V(2).InfoWriter()) + + if imagesConfigFile != "" { + if err := pullImages( + imagesConfig, + platforms, + imagePullConcurrency, + reg, + tempDir, + out, + ); err != nil { + return err + } + } + + if helmChartsConfigFile != "" { + helmChartsConfigFileAbs, err := filepath.Abs(helmChartsConfigFile) + if err != nil { + return err + } + + if err := pullCharts( + helmChartsConfig, + helmChartsConfigFileAbs, + reg, + tempDir, + cleaner, + out, + ); err != nil { + return err + } + } + + out.StartOperation(fmt.Sprintf("Archiving bundle to %s", outputFile)) + if err := archive.ArchiveDirectory(tempDir, outputFile); err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to create bundle tarball: %w", err) + } + out.EndOperationWithStatus(output.Success()) + + return nil + }, + } + + cmd.Flags().StringVar(&imagesConfigFile, "images-file", "", + "File containing list of images to create bundle from, either as YAML configuration or a simple list of images") + cmd.Flags().StringVar(&helmChartsConfigFile, "helm-charts-file", "", + "YAML file containing configuration of Helm charts to create bundle from") + cmd.MarkFlagsOneRequired("images-file", "helm-charts-file") + cmd.Flags().Var(&platforms, "platform", "platforms to download images for (required format: /[/])") + cmd.Flags(). + StringVar(&outputFile, "output-file", "bundle.tar", "Output file to write bundle to") + cmd.Flags(). + BoolVar(&overwrite, "overwrite", false, "Overwrite bundle file if it already exists") + cmd.Flags(). + IntVar(&imagePullConcurrency, "image-pull-concurrency", 1, "Image pull concurrency") + + return cmd +} + +func pullImages( + cfg config.ImagesConfig, + platforms flags.Platforms, + imagePullConcurrency int, + reg *registry.Registry, + outputDir string, + out output.Output, +) error { + // Sort registries for deterministic ordering. + regNames := cfg.SortedRegistryNames() + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.SetLimit(imagePullConcurrency) + + pullGauge := &output.ProgressGauge{} + pullGauge.SetCapacity(cfg.TotalImages()) + pullGauge.SetStatus("Pulling requested images") + + destTLSRoundTripper, err := httputils.InsecureTLSRoundTripper(remote.DefaultTransport) + if err != nil { + return fmt.Errorf("error configuring TLS for destination registry: %w", err) + } + defer func() { + if tr, ok := destTLSRoundTripper.(*http.Transport); ok { + tr.CloseIdleConnections() + } + }() + destRemoteOpts := []remote.Option{ + remote.WithTransport(destTLSRoundTripper), + remote.WithContext(egCtx), + remote.WithUserAgent(utils.Useragent()), + } + + out.StartOperationWithProgress(pullGauge) + + for registryIdx := range regNames { + registryName := regNames[registryIdx] + + registryConfig := cfg[registryName] + + sourceTLSRoundTripper, err := httputils.TLSConfiguredRoundTripper( + remote.DefaultTransport, + registryName, + registryConfig.TLSVerify != nil && !*registryConfig.TLSVerify, + "", + ) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("error configuring TLS for source registry: %w", err) + } + + keychain := authn.NewMultiKeychain( + authn.NewKeychainFromHelper( + authnhelpers.NewStaticHelper(registryName, registryConfig.Credentials), + ), + authn.DefaultKeychain, + ) + + sourceRemoteOpts := []remote.Option{ + remote.WithTransport(sourceTLSRoundTripper), + remote.WithAuthFromKeychain(keychain), + remote.WithContext(egCtx), + remote.WithUserAgent(utils.Useragent()), + } + + platformsStrings := platforms.GetSlice() + + // Sort images for deterministic ordering. + imageNames := registryConfig.SortedImageNames() + + wg := new(sync.WaitGroup) + + for imageIdx := range imageNames { + imageName := imageNames[imageIdx] + imageTags := registryConfig.Images[imageName] + + wg.Add(len(imageTags)) + for j := range imageTags { + imageTag := imageTags[j] + + eg.Go(func() error { + defer wg.Done() + + srcImageName := fmt.Sprintf( + "%s/%s:%s", + registryName, + imageName, + imageTag, + ) + + imageIndex, err := images.ManifestListForImage( + srcImageName, + platformsStrings, + sourceRemoteOpts..., + ) + if err != nil { + return err + } + + destImageName := fmt.Sprintf( + "%s/%s:%s", + reg.Address(), + imageName, + imageTag, + ) + ref, err := name.ParseReference(destImageName, name.StrictValidation) + if err != nil { + return err + } + + if err := remote.WriteIndex(ref, imageIndex, destRemoteOpts...); err != nil { + return err + } + + pullGauge.Inc() + + return nil + }) + } + } + + go func() { + wg.Wait() + + if tr, ok := sourceTLSRoundTripper.(*http.Transport); ok { + tr.CloseIdleConnections() + } + }() + } + + if err := eg.Wait(); err != nil { + out.EndOperationWithStatus(output.Failure()) + return err + } + + out.EndOperationWithStatus(output.Success()) + + if err := config.WriteSanitizedImagesConfig(cfg, filepath.Join(outputDir, "images.yaml")); err != nil { + return err + } + + return nil +} + +func pullCharts( + cfg config.HelmChartsConfig, + helmChartsConfigFileAbs string, + reg *registry.Registry, + outputDir string, + cleaner cleanup.Cleaner, + out output.Output, +) error { + out.StartOperation("Creating temporary chart storage directory") + + tempHelmChartStorageDir, err := os.MkdirTemp("", ".helm-bundle-temp-storage-*") + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "failed to create temporary directory for Helm chart storage: %w", + err, + ) + } + cleaner.AddCleanupFn(func() { _ = os.RemoveAll(tempHelmChartStorageDir) }) + out.EndOperationWithStatus(output.Success()) + + helmClient, helmCleanup := helm.NewClient(out) + cleaner.AddCleanupFn(func() { _ = helmCleanup() }) + + ociAddress := fmt.Sprintf("%s://%s/charts", helm.OCIScheme, reg.Address()) + + for repoName, repoConfig := range cfg.Repositories { + for chartName, chartVersions := range repoConfig.Charts { + sort.Strings(chartVersions) + + out.StartOperation( + fmt.Sprintf( + "Fetching Helm chart %s (versions %v) from %s (%s)", + chartName, + chartVersions, + repoName, + repoConfig.RepoURL, + ), + ) + var opts []action.PullOpt + if repoConfig.Username != "" { + opts = append( + opts, + helm.UsernamePasswordOpt(repoConfig.Username, repoConfig.Password), + ) + } + if !ptr.Deref(repoConfig.TLSVerify, true) { + opts = append(opts, helm.InsecureSkipTLSverifyOpt()) + } + for _, chartVersion := range chartVersions { + downloaded, err := helmClient.GetChartFromRepo( + tempHelmChartStorageDir, + repoConfig.RepoURL, + chartName, + chartVersion, + []helm.ConfigOpt{helm.RegistryClientConfigOpt()}, + opts..., + ) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to create Helm chart bundle: %w", err) + } + + if err := helmClient.PushHelmChartToOCIRegistry( + downloaded, ociAddress, + ); err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "failed to push Helm chart to temporary registry: %w", + err, + ) + } + + // Best effort cleanup of downloaded chart, will be cleaned up when the cleaner deletes the temporary + // directory anyway. + _ = os.Remove(downloaded) + } + out.EndOperationWithStatus(output.Success()) + } + } + for _, chartURL := range cfg.ChartURLs { + out.StartOperation(fmt.Sprintf("Fetching Helm chart from URL %s", chartURL)) + downloaded, err := helmClient.GetChartFromURL( + outputDir, + chartURL, + filepath.Dir(helmChartsConfigFileAbs), + ) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to create Helm chart bundle: %w", err) + } + + chrt, err := helm.LoadChart(downloaded) + if err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf( + "failed to extract Helm chart details from local chart: %w", + err, + ) + } + + _, ok := cfg.Repositories["local"] + if !ok { + cfg.Repositories["local"] = config.HelmRepositorySyncConfig{ + Charts: make(map[string][]string, 1), + } + } + _, ok = cfg.Repositories["local"].Charts[chrt.Name()] + if !ok { + cfg.Repositories["local"].Charts[chrt.Name()] = make([]string, 0, 1) + } + cfg.Repositories["local"].Charts[chrt.Name()] = append( + cfg.Repositories["local"].Charts[chrt.Name()], + chrt.Metadata.Version, + ) + + if err := helmClient.PushHelmChartToOCIRegistry( + downloaded, ociAddress, + ); err != nil { + out.EndOperationWithStatus(output.Failure()) + return fmt.Errorf("failed to push Helm chart to temporary registry: %w", err) + } + + // Best effort cleanup of downloaded chart, will be cleaned up when the cleaner deletes the temporary + // directory anyway. + _ = os.Remove(downloaded) + + out.EndOperationWithStatus(output.Success()) + } + + if err := config.WriteSanitizedHelmChartsConfig(cfg, filepath.Join(outputDir, "charts.yaml")); err != nil { + return err + } + + return nil +} diff --git a/cmd/mindthegap/create/create.go b/cmd/mindthegap/create/create.go index bd2e6594..a601c9e0 100644 --- a/cmd/mindthegap/create/create.go +++ b/cmd/mindthegap/create/create.go @@ -8,6 +8,7 @@ import ( "github.com/mesosphere/dkp-cli-runtime/core/output" + "github.com/mesosphere/mindthegap/cmd/mindthegap/create/bundle" "github.com/mesosphere/mindthegap/cmd/mindthegap/create/helmbundle" "github.com/mesosphere/mindthegap/cmd/mindthegap/create/imagebundle" ) @@ -20,5 +21,6 @@ func NewCommand(out output.Output) *cobra.Command { cmd.AddCommand(imagebundle.NewCommand(out)) cmd.AddCommand(helmbundle.NewCommand(out)) + cmd.AddCommand(bundle.NewCommand(out)) return cmd } diff --git a/cmd/mindthegap/create/helmbundle/helm_bundle.go b/cmd/mindthegap/create/helmbundle/helm_bundle.go index 24d7b701..3b8b0f4d 100644 --- a/cmd/mindthegap/create/helmbundle/helm_bundle.go +++ b/cmd/mindthegap/create/helmbundle/helm_bundle.go @@ -4,25 +4,15 @@ package helmbundle import ( - "errors" - "fmt" - "os" - "path/filepath" - "sort" + "strconv" "github.com/spf13/cobra" - "helm.sh/helm/v3/pkg/action" - "k8s.io/utils/ptr" "github.com/mesosphere/dkp-cli-runtime/core/output" - "github.com/mesosphere/mindthegap/archive" - "github.com/mesosphere/mindthegap/cleanup" + "github.com/mesosphere/mindthegap/cmd/mindthegap/create/bundle" "github.com/mesosphere/mindthegap/cmd/mindthegap/flags" "github.com/mesosphere/mindthegap/cmd/mindthegap/utils" - "github.com/mesosphere/mindthegap/config" - "github.com/mesosphere/mindthegap/docker/registry" - "github.com/mesosphere/mindthegap/helm" ) func NewCommand(out output.Output) *cobra.Command { @@ -47,211 +37,13 @@ func NewCommand(out output.Output) *cobra.Command { return nil }, RunE: func(cmd *cobra.Command, args []string) error { - if !overwrite { - out.StartOperation("Checking if output file already exists") - _, err := os.Stat(outputFile) - switch { - case err == nil: - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "%s already exists: specify --overwrite to overwrite existing file", - outputFile, - ) - case !errors.Is(err, os.ErrNotExist): - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to check if output file %s already exists: %w", - outputFile, - err, - ) - default: - out.EndOperationWithStatus(output.Success()) - } - } - - out.StartOperation("Parsing Helm chart bundle config") - cfg, err := config.ParseHelmChartsConfigFile(configFile) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return err - } - out.EndOperationWithStatus(output.Success()) - out.V(4).Infof("Helm charts config: %+v", cfg) - - configFileAbs, err := filepath.Abs(configFile) - if err != nil { - return err - } - - out.StartOperation("Creating temporary OCI registry directory") - outputFileAbs, err := filepath.Abs(outputFile) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to determine where to create temporary directory: %w", - err, - ) - } - - cleaner := cleanup.NewCleaner() - defer cleaner.Cleanup() - - tempRegistryDir, err := os.MkdirTemp(filepath.Dir(outputFileAbs), ".helm-bundle-*") - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create temporary directory for OCI registry: %w", err) - } - cleaner.AddCleanupFn(func() { _ = os.RemoveAll(tempRegistryDir) }) - out.EndOperationWithStatus(output.Success()) - - out.StartOperation("Starting temporary OCI registry") - reg, err := registry.NewRegistry(registry.Config{StorageDirectory: tempRegistryDir}) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create local OCI registry: %w", err) - } - go func() { - if err := reg.ListenAndServe(); err != nil { - out.Error(err, "error serving OCI registry") - os.Exit(2) - } - }() - out.EndOperationWithStatus(output.Success()) - - out.StartOperation("Creating temporary chart storage directory") - - tempHelmChartStorageDir, err := os.MkdirTemp("", ".helm-bundle-temp-storage-*") - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to create temporary directory for Helm chart storage: %w", - err, - ) - } - cleaner.AddCleanupFn(func() { _ = os.RemoveAll(tempHelmChartStorageDir) }) - out.EndOperationWithStatus(output.Success()) - - helmClient, helmCleanup := helm.NewClient(out) - cleaner.AddCleanupFn(func() { _ = helmCleanup() }) - - ociAddress := fmt.Sprintf("%s://%s/charts", helm.OCIScheme, reg.Address()) - - for repoName, repoConfig := range cfg.Repositories { - for chartName, chartVersions := range repoConfig.Charts { - sort.Strings(chartVersions) - - out.StartOperation( - fmt.Sprintf( - "Fetching Helm chart %s (versions %v) from %s (%s)", - chartName, - chartVersions, - repoName, - repoConfig.RepoURL, - ), - ) - var opts []action.PullOpt - if repoConfig.Username != "" { - opts = append( - opts, - helm.UsernamePasswordOpt(repoConfig.Username, repoConfig.Password), - ) - } - if !ptr.Deref(repoConfig.TLSVerify, true) { - opts = append(opts, helm.InsecureSkipTLSverifyOpt()) - } - for _, chartVersion := range chartVersions { - downloaded, err := helmClient.GetChartFromRepo( - tempHelmChartStorageDir, - repoConfig.RepoURL, - chartName, - chartVersion, - []helm.ConfigOpt{helm.RegistryClientConfigOpt()}, - opts..., - ) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create Helm chart bundle: %w", err) - } - - if err := helmClient.PushHelmChartToOCIRegistry( - downloaded, ociAddress, - ); err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to push Helm chart to temporary registry: %w", - err, - ) - } - - // Best effort cleanup of downloaded chart, will be cleaned up when the cleaner deletes the temporary - // directory anyway. - _ = os.Remove(downloaded) - } - out.EndOperationWithStatus(output.Success()) - } - } - for _, chartURL := range cfg.ChartURLs { - out.StartOperation(fmt.Sprintf("Fetching Helm chart from URL %s", chartURL)) - downloaded, err := helmClient.GetChartFromURL( - tempRegistryDir, - chartURL, - filepath.Dir(configFileAbs), - ) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create Helm chart bundle: %w", err) - } - - chrt, err := helm.LoadChart(downloaded) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to extract Helm chart details from local chart: %w", - err, - ) - } - - _, ok := cfg.Repositories["local"] - if !ok { - cfg.Repositories["local"] = config.HelmRepositorySyncConfig{ - Charts: make(map[string][]string, 1), - } - } - _, ok = cfg.Repositories["local"].Charts[chrt.Name()] - if !ok { - cfg.Repositories["local"].Charts[chrt.Name()] = make([]string, 0, 1) - } - cfg.Repositories["local"].Charts[chrt.Name()] = append( - cfg.Repositories["local"].Charts[chrt.Name()], - chrt.Metadata.Version, - ) - - if err := helmClient.PushHelmChartToOCIRegistry( - downloaded, ociAddress, - ); err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to push Helm chart to temporary registry: %w", err) - } - - // Best effort cleanup of downloaded chart, will be cleaned up when the cleaner deletes the temporary - // directory anyway. - _ = os.Remove(downloaded) - - out.EndOperationWithStatus(output.Success()) - } - - if err := config.WriteSanitizedHelmChartsConfig(cfg, filepath.Join(tempRegistryDir, "charts.yaml")); err != nil { - return err - } - - out.StartOperation(fmt.Sprintf("Archiving Helm charts to %s", outputFile)) - if err := archive.ArchiveDirectory(tempRegistryDir, outputFile); err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create Helm charts bundle tarball: %w", err) - } - out.EndOperationWithStatus(output.Success()) - - return nil + createBundleCmd := bundle.NewCommand(out) + createBundleCmd.SetArgs([]string{ + "--helm-charts-file", configFile, + "--output-file", outputFile, + "--overwrite", strconv.FormatBool(overwrite), + }) + return createBundleCmd.Execute() }, } @@ -266,5 +58,7 @@ func NewCommand(out output.Output) *cobra.Command { // TODO Unhide this from DKP CLI once DKP supports OCI registry for Helm charts. utils.AddCmdAnnotation(cmd, "exclude-from-dkp-cli", "true") + cmd.Deprecated = `"mindthegap create helm-bundle" is deprecated, please use "mindthegap create bundle" instead` + return cmd } diff --git a/cmd/mindthegap/create/imagebundle/image_bundle.go b/cmd/mindthegap/create/imagebundle/image_bundle.go index 73a0d684..0c9186d1 100644 --- a/cmd/mindthegap/create/imagebundle/image_bundle.go +++ b/cmd/mindthegap/create/imagebundle/image_bundle.go @@ -4,38 +4,21 @@ package imagebundle import ( - "context" - "errors" - "fmt" - "net/http" - "os" - "path/filepath" - "sync" + "strconv" + "strings" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/logs" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" "github.com/mesosphere/dkp-cli-runtime/core/output" - "github.com/mesosphere/mindthegap/archive" - "github.com/mesosphere/mindthegap/cleanup" + "github.com/mesosphere/mindthegap/cmd/mindthegap/create/bundle" "github.com/mesosphere/mindthegap/cmd/mindthegap/flags" - "github.com/mesosphere/mindthegap/cmd/mindthegap/utils" - "github.com/mesosphere/mindthegap/config" - "github.com/mesosphere/mindthegap/docker/registry" - "github.com/mesosphere/mindthegap/images" - "github.com/mesosphere/mindthegap/images/authnhelpers" - "github.com/mesosphere/mindthegap/images/httputils" ) func NewCommand(out output.Output) *cobra.Command { var ( configFile string - platforms []platform + platforms = flags.NewPlatformsValue("linux/amd64") outputFile string overwrite bool imagePullConcurrency int @@ -56,231 +39,22 @@ func NewCommand(out output.Output) *cobra.Command { return nil }, RunE: func(cmd *cobra.Command, args []string) error { - if !overwrite { - out.StartOperation("Checking if output file already exists") - _, err := os.Stat(outputFile) - switch { - case err == nil: - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "%s already exists: specify --overwrite to overwrite existing file", - outputFile, - ) - case !errors.Is(err, os.ErrNotExist): - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to check if output file %s already exists: %w", - outputFile, - err, - ) - default: - out.EndOperationWithStatus(output.Success()) - } - } - - out.StartOperation("Parsing image bundle config") - cfg, err := config.ParseImagesConfigFile(configFile) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return err - } - out.EndOperationWithStatus(output.Success()) - out.V(4).Infof("Images config: %+v", cfg) - - out.StartOperation("Creating temporary directory") - outputFileAbs, err := filepath.Abs(outputFile) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf( - "failed to determine where to create temporary directory: %w", - err, - ) - } - - cleaner := cleanup.NewCleaner() - defer cleaner.Cleanup() - - tempDir, err := os.MkdirTemp(filepath.Dir(outputFileAbs), ".image-bundle-*") - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create temporary directory: %w", err) - } - cleaner.AddCleanupFn(func() { _ = os.RemoveAll(tempDir) }) - - out.EndOperationWithStatus(output.Success()) - - out.StartOperation("Starting temporary Docker registry") - reg, err := registry.NewRegistry(registry.Config{StorageDirectory: tempDir}) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create local Docker registry: %w", err) - } - go func() { - if err := reg.ListenAndServe(); err != nil { - out.Error(err, "error serving Docker registry") - os.Exit(2) - } - }() - out.EndOperationWithStatus(output.Success()) - - logs.Debug.SetOutput(out.V(4).InfoWriter()) - logs.Warn.SetOutput(out.V(2).InfoWriter()) - - // Sort registries for deterministic ordering. - regNames := cfg.SortedRegistryNames() - - eg, egCtx := errgroup.WithContext(context.Background()) - eg.SetLimit(imagePullConcurrency) - - pullGauge := &output.ProgressGauge{} - pullGauge.SetCapacity(cfg.TotalImages()) - pullGauge.SetStatus("Pulling requested images") - - destTLSRoundTripper, err := httputils.InsecureTLSRoundTripper(remote.DefaultTransport) - if err != nil { - out.Error(err, "error configuring TLS for destination registry") - os.Exit(2) - } - defer func() { - if tr, ok := destTLSRoundTripper.(*http.Transport); ok { - tr.CloseIdleConnections() - } - }() - destRemoteOpts := []remote.Option{ - remote.WithTransport(destTLSRoundTripper), - remote.WithContext(egCtx), - remote.WithUserAgent(utils.Useragent()), - } - - out.StartOperationWithProgress(pullGauge) - - for registryIdx := range regNames { - registryName := regNames[registryIdx] - - registryConfig := cfg[registryName] - - sourceTLSRoundTripper, err := httputils.TLSConfiguredRoundTripper( - remote.DefaultTransport, - registryName, - registryConfig.TLSVerify != nil && !*registryConfig.TLSVerify, - "", - ) - if err != nil { - out.EndOperationWithStatus(output.Failure()) - out.Error(err, "error configuring TLS for source registry") - os.Exit(2) - } - - keychain := authn.NewMultiKeychain( - authn.NewKeychainFromHelper( - authnhelpers.NewStaticHelper(registryName, registryConfig.Credentials), - ), - authn.DefaultKeychain, - ) - - sourceRemoteOpts := []remote.Option{ - remote.WithTransport(sourceTLSRoundTripper), - remote.WithAuthFromKeychain(keychain), - remote.WithContext(egCtx), - remote.WithUserAgent(utils.Useragent()), - } - - platformsStrings := make([]string, 0, len(platforms)) - for _, p := range platforms { - platformsStrings = append(platformsStrings, p.String()) - } - - // Sort images for deterministic ordering. - imageNames := registryConfig.SortedImageNames() - - wg := new(sync.WaitGroup) - - for imageIdx := range imageNames { - imageName := imageNames[imageIdx] - imageTags := registryConfig.Images[imageName] - - wg.Add(len(imageTags)) - for j := range imageTags { - imageTag := imageTags[j] - - eg.Go(func() error { - defer wg.Done() - - srcImageName := fmt.Sprintf( - "%s/%s:%s", - registryName, - imageName, - imageTag, - ) - - imageIndex, err := images.ManifestListForImage( - srcImageName, - platformsStrings, - sourceRemoteOpts..., - ) - if err != nil { - return err - } - - destImageName := fmt.Sprintf( - "%s/%s:%s", - reg.Address(), - imageName, - imageTag, - ) - ref, err := name.ParseReference(destImageName, name.StrictValidation) - if err != nil { - return err - } - - if err := remote.WriteIndex(ref, imageIndex, destRemoteOpts...); err != nil { - return err - } - - pullGauge.Inc() - - return nil - }) - } - } - - go func() { - wg.Wait() - - if tr, ok := sourceTLSRoundTripper.(*http.Transport); ok { - tr.CloseIdleConnections() - } - }() - } - - if err := eg.Wait(); err != nil { - out.EndOperationWithStatus(output.Failure()) - return err - } - - out.EndOperationWithStatus(output.Success()) - - if err := config.WriteSanitizedImagesConfig(cfg, filepath.Join(tempDir, "images.yaml")); err != nil { - return err - } - - out.StartOperation(fmt.Sprintf("Archiving images to %s", outputFile)) - if err := archive.ArchiveDirectory(tempDir, outputFile); err != nil { - out.EndOperationWithStatus(output.Failure()) - return fmt.Errorf("failed to create image bundle tarball: %w", err) - } - out.EndOperationWithStatus(output.Success()) - - return nil + createBundleCmd := bundle.NewCommand(out) + createBundleCmd.SetArgs([]string{ + "--images-file", configFile, + "--platform", strings.Join(platforms.GetSlice(), ","), + "--output-file", outputFile, + "--overwrite", strconv.FormatBool(overwrite), + "--image-pull-concurrency", strconv.Itoa(imagePullConcurrency), + }) + return createBundleCmd.Execute() }, } cmd.Flags().StringVar(&configFile, "images-file", "", "File containing list of images to create bundle from, either as YAML configuration or a simple list of images") _ = cmd.MarkFlagRequired("images-file") - cmd.Flags(). - Var(newPlatformSlicesValue([]platform{{os: "linux", arch: "amd64"}}, &platforms), "platform", - "platforms to download images (required format: /[/])") + cmd.Flags().Var(&platforms, "platform", "platforms to download images for (required format: /[/])") cmd.Flags(). StringVar(&outputFile, "output-file", "images.tar", "Output file to write image bundle to") cmd.Flags(). @@ -288,5 +62,7 @@ func NewCommand(out output.Output) *cobra.Command { cmd.Flags(). IntVar(&imagePullConcurrency, "image-pull-concurrency", 1, "Image pull concurrency") + cmd.Deprecated = `"mindthegap create image-bundle" is deprecated, please use "mindthegap create bundle" instead` + return cmd } diff --git a/cmd/mindthegap/create/imagebundle/platforms_flag.go b/cmd/mindthegap/flags/platforms_flag.go similarity index 78% rename from cmd/mindthegap/create/imagebundle/platforms_flag.go rename to cmd/mindthegap/flags/platforms_flag.go index 29aa1288..5a679ad2 100644 --- a/cmd/mindthegap/create/imagebundle/platforms_flag.go +++ b/cmd/mindthegap/flags/platforms_flag.go @@ -1,7 +1,7 @@ // Copyright 2021 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imagebundle +package flags import ( "bytes" @@ -38,17 +38,22 @@ func (p platform) String() string { return s } -// -- platformSlice Value. -type platformSliceValue struct { +// Platforms Value. +type Platforms struct { value *[]platform changed bool } -func newPlatformSlicesValue(val []platform, p *[]platform) *platformSliceValue { - psv := new(platformSliceValue) - psv.value = p - *psv.value = val - return psv +func NewPlatformsValue(platforms ...string) Platforms { + ps := make([]platform, 0, len(platforms)) + for _, p := range platforms { + parsed, err := parsePlatformString(p) + if err != nil { + panic(fmt.Sprintf("invalid platform string: %s", p)) + } + ps = append(ps, parsed) + } + return Platforms{value: &ps} } func readPlatformsAsCSV(val string) ([]platform, error) { @@ -103,11 +108,11 @@ func parsePlatformString(s string) (platform, error) { } var ( - _ pflag.Value = &platformSliceValue{} - _ pflag.SliceValue = &platformSliceValue{} + _ pflag.Value = &Platforms{} + _ pflag.SliceValue = &Platforms{} ) -func (s *platformSliceValue) Set(val string) error { +func (s *Platforms) Set(val string) error { v, err := readPlatformsAsCSV(val) if err != nil { return err @@ -121,16 +126,16 @@ func (s *platformSliceValue) Set(val string) error { return nil } -func (s *platformSliceValue) Type() string { +func (s *Platforms) Type() string { return "platformSlice" } -func (s *platformSliceValue) String() string { +func (s *Platforms) String() string { str, _ := writePlatformsAsCSV(*s.value) return "[" + str + "]" } -func (s *platformSliceValue) Append(val string) error { +func (s *Platforms) Append(val string) error { p, err := parsePlatformString(val) if err != nil { return err @@ -139,7 +144,7 @@ func (s *platformSliceValue) Append(val string) error { return nil } -func (s *platformSliceValue) Replace(val []string) error { +func (s *Platforms) Replace(val []string) error { ps := make([]platform, 0, len(val)) for _, v := range val { p, err := parsePlatformString(v) @@ -152,7 +157,7 @@ func (s *platformSliceValue) Replace(val []string) error { return nil } -func (s *platformSliceValue) GetSlice() []string { +func (s *Platforms) GetSlice() []string { strs := make([]string, 0, len(*s.value)) for _, p := range *s.value { strs = append(strs, p.String()) diff --git a/cmd/mindthegap/create/imagebundle/platforms_flag_test.go b/cmd/mindthegap/flags/platforms_flag_test.go similarity index 69% rename from cmd/mindthegap/create/imagebundle/platforms_flag_test.go rename to cmd/mindthegap/flags/platforms_flag_test.go index e23ce404..31349b2e 100644 --- a/cmd/mindthegap/create/imagebundle/platforms_flag_test.go +++ b/cmd/mindthegap/flags/platforms_flag_test.go @@ -1,7 +1,7 @@ // Copyright 2021 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imagebundle +package flags import ( "fmt" @@ -13,29 +13,23 @@ import ( const argfmt = "--ps=%s" -func setUpPSFlagSet(psp *[]platform) *pflag.FlagSet { +func setUpPSFlagSet() *pflag.FlagSet { f := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.Var(newPlatformSlicesValue( - []platform{}, psp), - "ps", "Command separated list!") + pv := NewPlatformsValue() + f.Var(&pv, "ps", "Command separated list!") return f } -func setUpPSFlagSetWithDefault(psp *[]platform) *pflag.FlagSet { +func setUpPSFlagSetWithDefault() *pflag.FlagSet { f := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.Var(newPlatformSlicesValue( - []platform{ - {os: "defaultos1", arch: "defaultarch1"}, - {os: "defaultos2", arch: "defaultarch2", variant: "defaultvariant2"}, - }, psp), - "ps", "Command separated list!") + pv := NewPlatformsValue("defaultos1/defaultarch1", "defaultos2/defaultarch2/defaultvariant2") + f.Var(&pv, "ps", "Command separated list!") return f } func TestEmptyPS(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() err := f.Parse([]string{}) if err != nil { t.Fatal("expected no error; got", err) @@ -50,8 +44,7 @@ func TestEmptyPS(t *testing.T) { func TestEmptyPSValue(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() err := f.Parse([]string{"--ps="}) if err != nil { t.Fatal("expected no error; got", err) @@ -65,8 +58,7 @@ func TestEmptyPSValue(t *testing.T) { func TestPS(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() vals := []platform{ {os: "linux", arch: "amd64"}, @@ -83,24 +75,17 @@ func TestPS(t *testing.T) { if err != nil { t.Fatal("expected no error; got", err) } - for i, v := range ps { - if vals[i] != v { + getPS := f.Lookup("ps").Value + for i, v := range getPS.(pflag.SliceValue).GetSlice() { + if vals[i].String() != v { t.Fatalf("expected ps[%d] to be %s but got: %s", i, vals[i], v) } } - - getPS := f.Lookup("ps").Value.(*platformSliceValue) - for i, v := range *getPS.value { - if vals[i] != v { - t.Fatalf("expected ps[%d] to be %s from Lookup but got: %s", i, vals[i], v) - } - } } func TestPSDefault(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSetWithDefault(&ps) + f := setUpPSFlagSetWithDefault() vals := []platform{ {os: "defaultos1", arch: "defaultarch1"}, @@ -111,24 +96,17 @@ func TestPSDefault(t *testing.T) { if err != nil { t.Fatal("expected no error; got", err) } - for i, v := range ps { - if vals[i] != v { + getPS := f.Lookup("ps").Value + for i, v := range getPS.(pflag.SliceValue).GetSlice() { + if vals[i].String() != v { t.Fatalf("expected ps[%d] to be %s but got: %s", i, vals[i], v) } } - - getPS := f.Lookup("ps").Value.(*platformSliceValue) - for i, v := range *getPS.value { - if vals[i] != v { - t.Fatalf("expected ps[%d] to be %s from Lookup but got: %s", i, vals[i], v) - } - } } func TestSSWithDefault(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSetWithDefault(&ps) + f := setUpPSFlagSetWithDefault() vals := []platform{ {os: "linux", arch: "amd64"}, @@ -145,24 +123,17 @@ func TestSSWithDefault(t *testing.T) { if err != nil { t.Fatal("expected no error; got", err) } - for i, v := range ps { - if vals[i] != v { + getPS := f.Lookup("ps").Value + for i, v := range getPS.(pflag.SliceValue).GetSlice() { + if vals[i].String() != v { t.Fatalf("expected ss[%d] to be %s but got: %s", i, vals[i], v) } } - - getPS := f.Lookup("ps").Value.(*platformSliceValue) - for i, v := range *getPS.value { - if vals[i] != v { - t.Fatalf("expected ps[%d] to be %s from Lookup but got: %s", i, vals[i], v) - } - } } func TestSSCalledTwice(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{"linux/amd64", "darwin/arm64/v8,linux/arm/v7"} expected := []platform{ @@ -178,31 +149,21 @@ func TestSSCalledTwice(t *testing.T) { t.Fatal("expected no error; got", err) } + getPS := f.Lookup("ps").Value + ps := getPS.(pflag.SliceValue).GetSlice() if len(expected) != len(ps) { t.Fatalf("expected number of ss to be %d but got: %d", len(expected), len(ps)) } for i, v := range ps { - if expected[i] != v { + if expected[i].String() != v { t.Fatalf("expected ss[%d] to be %s but got: %s", i, expected[i], v) } } - - values := f.Lookup("ps").Value.(*platformSliceValue) - - if len(expected) != len(*values.value) { - t.Fatalf("expected number of values to be %d but got: %d", len(expected), len(ps)) - } - for i, v := range *values.value { - if expected[i] != v { - t.Fatalf("expected got ss[%d] to be %s but got: %s", i, expected[i], v) - } - } } func TestSSWithComma(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{`"linux/amd64"`, `"windows/amd64"`, `"darwin/arm64/v8",linux/arm/v7`} expected := []platform{ @@ -219,35 +180,21 @@ func TestSSWithComma(t *testing.T) { t.Fatal("expected no error; got", err) } + getPS := f.Lookup("ps").Value + ps := getPS.(pflag.SliceValue).GetSlice() if len(expected) != len(ps) { t.Fatalf("expected number of ps to be %d but got: %d", len(expected), len(ps)) } for i, v := range ps { - if expected[i] != v { + if expected[i].String() != v { t.Fatalf("expected ss[%d] to be %s but got: %s", i, expected[i], v) } } - - values := f.Lookup("ps").Value.(*platformSliceValue) - - if len(expected) != len(*values.value) { - t.Fatalf( - "expected number of values to be %d but got: %d", - len(expected), - len(*values.value), - ) - } - for i, v := range *values.value { - if expected[i] != v { - t.Fatalf("expected got ps[%d] to be %s but got: %s", i, expected[i], v) - } - } } func TestPSAsSliceValue(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{"linux/amd64", "darwin/arm64/v8"} arg1 := fmt.Sprintf(argfmt, in[0]) @@ -259,10 +206,12 @@ func TestPSAsSliceValue(t *testing.T) { _ = val.Replace([]string{"windows/arm/v7"}) } }) + getPS := f.Lookup("ps").Value + ps := getPS.(pflag.SliceValue).GetSlice() expectedPlatform := platform{os: "windows", arch: "arm", variant: "v7"} require.ElementsMatch( t, - []platform{expectedPlatform}, + []string{expectedPlatform.String()}, ps, "Expected ps to be overwritten with 'windows/arm/v7'", ) @@ -270,8 +219,7 @@ func TestPSAsSliceValue(t *testing.T) { func TestPSGetSlice(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{"linux/amd64", "darwin/arm64/v8"} arg1 := fmt.Sprintf(argfmt, in[0]) @@ -287,8 +235,7 @@ func TestPSGetSlice(t *testing.T) { func TestPSAppend(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{"linux/amd64", "darwin/arm64/v8"} arg1 := fmt.Sprintf(argfmt, in[0]) @@ -309,8 +256,7 @@ func TestPSAppend(t *testing.T) { func TestPSInvalidPlatform(t *testing.T) { t.Parallel() - var ps []platform - f := setUpPSFlagSet(&ps) + f := setUpPSFlagSet() in := []string{"wibble"} arg1 := fmt.Sprintf(argfmt, in[0]) diff --git a/test/e2e/helmbundle/helpers/helpers.go b/test/e2e/helmbundle/helpers/helpers.go index d8a1cf00..8a03847b 100644 --- a/test/e2e/helmbundle/helpers/helpers.go +++ b/test/e2e/helmbundle/helpers/helpers.go @@ -27,12 +27,12 @@ import ( "github.com/mesosphere/dkp-cli-runtime/core/output" - createhelmbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/helmbundle" + createbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/bundle" "github.com/mesosphere/mindthegap/helm" ) func CreateBundle(t ginkgo.GinkgoTInterface, bundleFile, cfgFile string) { - createBundleCmd := NewCommand(t, createhelmbundle.NewCommand) + createBundleCmd := NewCommand(t, createbundle.NewCommand) createBundleCmd.SetArgs([]string{ "--output-file", bundleFile, "--helm-charts-file", cfgFile, diff --git a/test/e2e/imagebundle/helpers/helpers.go b/test/e2e/imagebundle/helpers/helpers.go index ac0543ec..046b7810 100644 --- a/test/e2e/imagebundle/helpers/helpers.go +++ b/test/e2e/imagebundle/helpers/helpers.go @@ -32,7 +32,7 @@ import ( "github.com/mesosphere/dkp-cli-runtime/core/output" - createimagebundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/imagebundle" + createbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/bundle" ) func CreateBundle(t ginkgo.GinkgoTInterface, bundleFile, cfgFile string, platforms ...string) { @@ -41,7 +41,7 @@ func CreateBundle(t ginkgo.GinkgoTInterface, bundleFile, cfgFile string, platfor platformFlags = append(platformFlags, "--platform", p) } - createBundleCmd := NewCommand(t, createimagebundle.NewCommand) + createBundleCmd := NewCommand(t, createbundle.NewCommand) createBundleCmd.SetArgs(append([]string{ "--output-file", bundleFile, "--images-file", cfgFile,