From b35c5135ad9cab2846a0dada3782365b5d73681b Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Fri, 22 Dec 2023 12:48:34 -0800 Subject: [PATCH] Use rendered kots kinds (#4239) --- cmd/imagedeps/main_test.go | 14 +- cmd/kots/cli/identity.go | 6 +- cmd/kots/cli/ingress.go | 4 +- cmd/kots/cli/pull.go | 3 +- cmd/kots/cli/set-config.go | 5 +- cmd/kots/cli/velero.go | 5 +- integration/database/rqlite_migration_test.go | 3 +- .../docker/registry/temp_registry_test.go | 4 +- integration/replicated/generate.go | 4 +- integration/replicated/pull_test.go | 9 +- kurl_proxy/cmd/main.go | 2 +- pkg/airgap/airgap.go | 2 +- pkg/airgap/update.go | 5 +- pkg/apparchive/app.go | 4 +- pkg/apparchive/helm-v1beta1.go | 2 +- pkg/apparchive/helm-v1beta1_test.go | 4 +- pkg/apparchive/helm-v1beta2.go | 20 +- pkg/apparchive/installation.go | 4 +- pkg/base/helm.go | 6 +- pkg/base/images.go | 4 +- pkg/base/render.go | 44 +- pkg/base/replicated.go | 116 ++--- pkg/base/replicated_test.go | 42 +- pkg/base/templates.go | 13 +- pkg/base/write.go | 2 +- pkg/base/write_images.go | 7 +- pkg/base/write_images_test.go | 67 ++- pkg/docker/registry/images.go | 17 +- pkg/docker/registry/images_test.go | 171 +++++- pkg/gitops/gitops.go | 4 +- pkg/handlers/config.go | 61 ++- pkg/handlers/contents.go | 2 +- pkg/handlers/download.go | 2 +- pkg/handlers/identity.go | 8 +- pkg/handlers/metadata.go | 6 +- pkg/handlers/preflight.go | 3 +- pkg/handlers/registry.go | 2 +- pkg/handlers/rendered_contents.go | 3 +- pkg/handlers/upload.go | 7 +- pkg/helm/updates.go | 2 +- pkg/identity/client/http.go | 3 +- pkg/identity/deploy/deploy.go | 3 +- pkg/image/builder.go | 22 +- pkg/{image => imageutil}/image.go | 12 +- pkg/{image => imageutil}/image_test.go | 12 +- pkg/k8sutil/kustomization.go | 3 +- pkg/kotsadm/configmaps.go | 4 +- pkg/kotsadm/objects/images.go | 7 +- pkg/kotsadm/push_images.go | 19 +- pkg/kotsadmlicense/license.go | 3 +- pkg/kotsadmsnapshot/backup.go | 7 +- pkg/kotsadmupstream/upstream.go | 5 +- pkg/kotsutil/kots.go | 159 +++--- pkg/kotsutil/kots_test.go | 311 +++++++++-- pkg/midstream/registry.go | 3 +- pkg/midstream/write.go | 130 ++--- pkg/online/online.go | 3 +- pkg/operator/client/deploy.go | 2 +- pkg/operator/operator.go | 9 +- pkg/preflight/preflight.go | 14 +- pkg/print/velero.go | 6 +- pkg/pull/archive.go | 4 +- pkg/pull/pull.go | 165 +++--- pkg/registry/images.go | 4 +- pkg/registry/registry.go | 6 +- pkg/render/render.go | 3 +- pkg/reporting/app.go | 3 +- pkg/rewrite/rewrite.go | 116 +++-- pkg/secrets/sealed_secrets.go | 2 +- pkg/secrets/secrets.go | 3 +- pkg/secrets/secrets_test.go | 29 +- pkg/snapshot/filesystem_minio.go | 3 +- pkg/store/kotsstore/migrations.go | 13 +- pkg/store/kotsstore/version_store.go | 8 +- pkg/supportbundle/supportbundle.go | 6 +- pkg/template/config_context.go | 3 +- pkg/tests/base/render_test.go | 50 +- .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 pkg/tests/pull/cases/kotskinds/testcase.yaml | 13 + .../kotskinds/upstream/app/configmap.yaml | 6 + .../cases/kotskinds/upstream/kots-app.yaml | 47 ++ .../cases/kotskinds/upstream/kots-config.yaml | 406 +++++++++++++++ .../kotskinds/upstream/kots-preflight.yaml | 30 ++ .../upstream/kots-supportbundle.yaml | 55 ++ .../kotskinds/upstream/my-chart-0.1.0.tgz | Bin 0 -> 407 bytes .../upstream/my-excluded-chart-v1beta1.yaml | 24 + .../upstream/my-excluded-chart-v1beta2.yaml | 22 + .../kotskinds/upstream/userdata/config.yaml | 210 ++++++++ .../upstream/userdata/installation.yaml | 14 + .../kotskinds/upstream/userdata/license.yaml | 49 ++ .../wantResults/base/app/configmap.yaml | 6 + .../wantResults/base/kustomization.yaml | 7 + .../wantResults/kotsKinds/kots-app.yaml | 47 ++ .../wantResults/kotsKinds/kots-config.yaml | 493 ++++++++++++++++++ .../wantResults/kotsKinds/kots-preflight.yaml | 30 ++ .../kotsKinds/kots-supportbundle.yaml | 55 ++ .../kotsKinds/my-excluded-chart-v1beta1.yaml | 33 ++ .../kotsKinds/my-excluded-chart-v1beta2.yaml | 31 ++ .../kotsKinds/userdata/config.yaml | 210 ++++++++ .../kotsKinds/userdata/installation.yaml | 15 + .../kotsKinds/userdata/license.yaml | 49 ++ .../this-cluster/kustomization.yaml | 4 + .../midstream/backup-label-transformer.yaml | 39 ++ .../overlays/midstream/kustomization.yaml | 10 + .../overlays/midstream/secret.yaml | 28 + .../kotsadm-replicated-registry-secret.yaml | 17 + .../this-cluster/my-app-registry-secret.yaml | 17 + .../this-cluster/test-config-configmap.yaml | 11 + .../wantResults/upstream/app/configmap.yaml | 6 + .../wantResults/upstream/kots-app.yaml | 47 ++ .../wantResults/upstream/kots-config.yaml | 406 +++++++++++++++ .../wantResults/upstream/kots-preflight.yaml | 30 ++ .../upstream/kots-supportbundle.yaml | 55 ++ .../wantResults/upstream/my-chart-0.1.0.tgz | Bin 0 -> 407 bytes .../upstream/my-excluded-chart-v1beta1.yaml | 24 + .../upstream/my-excluded-chart-v1beta2.yaml | 22 + .../wantResults/upstream/userdata/config.yaml | 210 ++++++++ .../upstream/userdata/installation.yaml | 15 + .../upstream/userdata/license.yaml | 49 ++ .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 .../{ => userdata}/installation.yaml | 0 pkg/tests/pull/pull_test.go | 15 +- pkg/upload/archive.go | 4 +- pkg/upload/upload_license.go | 2 +- pkg/upstream/admin-console.go | 2 +- pkg/upstream/identity.go | 4 +- pkg/upstream/push_images.go | 6 +- pkg/upstream/replicated.go | 6 +- pkg/upstream/write.go | 9 +- pkg/util/tar.go | 2 +- pkg/util/util.go | 3 +- pkg/version/metrics.go | 3 +- 140 files changed, 4019 insertions(+), 723 deletions(-) rename pkg/{image => imageutil}/image.go (97%) rename pkg/{image => imageutil}/image_test.go (98%) rename pkg/tests/pull/cases/configcontext/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/customhostnames/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) create mode 100644 pkg/tests/pull/cases/kotskinds/testcase.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/app/configmap.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/kots-app.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/kots-config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/kots-preflight.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/kots-supportbundle.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/my-chart-0.1.0.tgz create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/my-excluded-chart-v1beta1.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/my-excluded-chart-v1beta2.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/userdata/config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/userdata/installation.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/upstream/userdata/license.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/base/app/configmap.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/base/kustomization.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/kots-app.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/kots-config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/kots-preflight.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/kots-supportbundle.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/my-excluded-chart-v1beta1.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/my-excluded-chart-v1beta2.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/userdata/config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/userdata/installation.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/kotsKinds/userdata/license.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/overlays/downstreams/this-cluster/kustomization.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/overlays/midstream/backup-label-transformer.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/overlays/midstream/kustomization.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/overlays/midstream/secret.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/rendered/this-cluster/kotsadm-replicated-registry-secret.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/rendered/this-cluster/my-app-registry-secret.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/rendered/this-cluster/test-config-configmap.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/app/configmap.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/kots-app.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/kots-config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/kots-preflight.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/kots-supportbundle.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/my-chart-0.1.0.tgz create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/my-excluded-chart-v1beta1.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/my-excluded-chart-v1beta2.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/userdata/config.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/userdata/installation.yaml create mode 100644 pkg/tests/pull/cases/kotskinds/wantResults/upstream/userdata/license.yaml rename pkg/tests/pull/cases/multidoc/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/samechartvariations/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/simple/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/subchart-alias/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/subchart-crds/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/subcharts/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/taganddigest-norewrite/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/taganddigest-rewrite/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) rename pkg/tests/pull/cases/v1beta2-charts/wantResults/kotsKinds/{ => userdata}/installation.yaml (100%) diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index d421083ca7..45c3aec906 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -136,10 +136,10 @@ func TestFunctional(t *testing.T) { t.Run(tc.name, func(t *testing.T) { rootDir := path.Join("testdata", tc.name) - expectedConstants, err := ioutil.ReadFile(path.Join(rootDir, "constants.go")) + expectedConstants, err := os.ReadFile(path.Join(rootDir, "constants.go")) require.Nil(t, err) - expectedEnvs, err := ioutil.ReadFile(path.Join(rootDir, ".image.env")) + expectedEnvs, err := os.ReadFile(path.Join(rootDir, ".image.env")) require.Nil(t, err) tempDir := t.TempDir() @@ -178,10 +178,10 @@ func TestFunctional(t *testing.T) { require.Nil(t, err) - actualConstants, err := ioutil.ReadFile(constantFile) + actualConstants, err := os.ReadFile(constantFile) require.Nil(t, err) - actualEnv, err := ioutil.ReadFile(envFile) + actualEnv, err := os.ReadFile(envFile) require.Nil(t, err) require.Equal(t, string(expectedConstants), string(actualConstants)) @@ -195,10 +195,10 @@ func TestFunctional(t *testing.T) { require.Nil(t, err) for _, f := range files { - expectedContent, err := ioutil.ReadFile(path.Join(expectedDir, f.Name())) + expectedContent, err := os.ReadFile(path.Join(expectedDir, f.Name())) require.Nil(t, err) - actualContent, err := ioutil.ReadFile(path.Join(actualDir, f.Name())) + actualContent, err := os.ReadFile(path.Join(actualDir, f.Name())) require.Nil(t, err) require.Equal(t, string(expectedContent), string(actualContent)) @@ -220,7 +220,7 @@ func copyDirFiles(inputDir string, outputDir string) error { } for _, f := range files { - content, err := ioutil.ReadFile(path.Join(inputDir, f.Name())) + content, err := os.ReadFile(path.Join(inputDir, f.Name())) if err != nil { return errors.Wrapf(err, "failed to read file %s", path.Join(inputDir, f.Name())) } diff --git a/cmd/kots/cli/identity.go b/cmd/kots/cli/identity.go index db515aa5e7..dadca85d2b 100644 --- a/cmd/kots/cli/identity.go +++ b/cmd/kots/cli/identity.go @@ -3,7 +3,7 @@ package cli import ( "context" "fmt" - "io/ioutil" + "os" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/identity" @@ -65,7 +65,7 @@ func IdentityServiceInstallCmd() *cobra.Command { identityConfig := kotsv1beta1.IdentityConfig{} if identityConfigPath := v.GetString("identity-config"); identityConfigPath != "" { - content, err := ioutil.ReadFile(identityConfigPath) + content, err := os.ReadFile(identityConfigPath) if err != nil { return errors.Wrap(err, "failed to read identity service config file") } @@ -132,7 +132,7 @@ func IdentityServiceConfigureCmd() *cobra.Command { identityConfig := kotsv1beta1.IdentityConfig{} if identityConfigPath := v.GetString("identity-config"); identityConfigPath != "" { - content, err := ioutil.ReadFile(identityConfigPath) + content, err := os.ReadFile(identityConfigPath) if err != nil { return errors.Wrap(err, "failed to read identity service config file") } diff --git a/cmd/kots/cli/ingress.go b/cmd/kots/cli/ingress.go index 1ce03d46b7..484bf78349 100644 --- a/cmd/kots/cli/ingress.go +++ b/cmd/kots/cli/ingress.go @@ -1,7 +1,7 @@ package cli import ( - "io/ioutil" + "os" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/identity" @@ -54,7 +54,7 @@ func IngressInstallCmd() *cobra.Command { ingressConfig := kotsv1beta1.IngressConfig{} if ingressConfigPath := v.GetString("ingress-config"); ingressConfigPath != "" { - content, err := ioutil.ReadFile(ingressConfigPath) + content, err := os.ReadFile(ingressConfigPath) if err != nil { return errors.Wrap(err, "failed to read ingress service config file") } diff --git a/cmd/kots/cli/pull.go b/cmd/kots/cli/pull.go index f882c045d2..e93de9a7dd 100644 --- a/cmd/kots/cli/pull.go +++ b/cmd/kots/cli/pull.go @@ -1,7 +1,6 @@ package cli import ( - "io/ioutil" "os" "path" "strings" @@ -151,7 +150,7 @@ func getAppSlugForPull(uri string, licenseFile string) (string, error) { return appSlug, nil } - licenseData, err := ioutil.ReadFile(licenseFile) + licenseData, err := os.ReadFile(licenseFile) if err != nil { return "", errors.Wrap(err, "failed to read license file") } diff --git a/cmd/kots/cli/set-config.go b/cmd/kots/cli/set-config.go index 800d7931bd..3897b251d0 100644 --- a/cmd/kots/cli/set-config.go +++ b/cmd/kots/cli/set-config.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -176,7 +175,7 @@ func getConfigValuesFromArgs(v *viper.Viper, args []string) ([]byte, error) { return nil, errors.New("--config-file cannot be used with other key/value arguments") } - data, err := ioutil.ReadFile(fileName) + data, err := os.ReadFile(fileName) if err != nil { return nil, errors.Wrap(err, "failed to load config from file") } @@ -206,7 +205,7 @@ func getConfigValuesFromArgs(v *viper.Viper, args []string) ([]byte, error) { Value: value, } } else if valueFile != "" { - data, err := ioutil.ReadFile(valueFile) + data, err := os.ReadFile(valueFile) if err != nil { return nil, errors.Wrap(err, "failed to load value from file") } diff --git a/cmd/kots/cli/velero.go b/cmd/kots/cli/velero.go index 8d2b6bb586..05694e278d 100644 --- a/cmd/kots/cli/velero.go +++ b/cmd/kots/cli/velero.go @@ -3,7 +3,6 @@ package cli import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -394,7 +393,7 @@ func VeleroConfigureOtherS3Cmd() *cobra.Command { if err != nil { return err } - caCertData, err = ioutil.ReadFile(realPath) + caCertData, err = os.ReadFile(realPath) if err != nil { return err } @@ -506,7 +505,7 @@ func VeleroConfigureGCPServiceAccount() *cobra.Command { jsonFile := "" if jsonFilePath := v.GetString("json-file"); jsonFilePath != "" { - content, err := ioutil.ReadFile(jsonFilePath) + content, err := os.ReadFile(jsonFilePath) if err != nil { return errors.Wrap(err, "failed to read json file") } diff --git a/integration/database/rqlite_migration_test.go b/integration/database/rqlite_migration_test.go index 7f3a986367..1183ca5240 100644 --- a/integration/database/rqlite_migration_test.go +++ b/integration/database/rqlite_migration_test.go @@ -16,6 +16,7 @@ import ( "github.com/ory/dockertest/v3/docker" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/persistence" "github.com/rqlite/gorqlite" ) @@ -72,7 +73,7 @@ func TestMigrateFromPostgresToRqlite(t *testing.T) { if err := os.WriteFile(rqliteAuthConfigPath, []byte(RQLITE_AUTH_CONFIG), 0644); err != nil { t.Fatalf("Failed to write to file %s", rqliteAuthConfigPath) } - rqliteTag, _ := image.GetTag(image.Rqlite) + rqliteTag, _ := imageutil.GetTag(image.Rqlite) rqliteRunOptions := &dockertest.RunOptions{ Name: "rqlite", Repository: "kotsadm/rqlite", diff --git a/integration/docker/registry/temp_registry_test.go b/integration/docker/registry/temp_registry_test.go index a324cecfee..1cde44b049 100644 --- a/integration/docker/registry/temp_registry_test.go +++ b/integration/docker/registry/temp_registry_test.go @@ -1,7 +1,7 @@ package replicated import ( - "io/ioutil" + "os" "path" "reflect" "testing" @@ -221,7 +221,7 @@ func TestTempRegistry_GetImageLayers(t *testing.T) { req := require.New(t) - manifestsContent, err := ioutil.ReadFile(path.Join("assets", "manifests.yaml")) + manifestsContent, err := os.ReadFile(path.Join("assets", "manifests.yaml")) req.NoError(err) var manifests map[string]string diff --git a/integration/replicated/generate.go b/integration/replicated/generate.go index 5c7ae28d62..13ee202d6e 100644 --- a/integration/replicated/generate.go +++ b/integration/replicated/generate.go @@ -93,7 +93,7 @@ func generateReplicatedAppArchive(rawArchivePath string) ([]byte, error) { if err := tarGz.Archive([]string{rawArchivePath}, archiveFile); err != nil { return nil, errors.Wrap(err, "failed to create archive") } - b, err := ioutil.ReadFile(archiveFile) + b, err := os.ReadFile(archiveFile) if err != nil { return nil, errors.Wrap(err, "failed to read archive file") } @@ -152,7 +152,7 @@ func generateExpectedFilesystem(namespace, rawArchivePath string) ([]byte, error if err := tarGz.Archive(paths, archiveFile); err != nil { return nil, errors.Wrap(err, "failed to create archive") } - b, err := ioutil.ReadFile(archiveFile) + b, err := os.ReadFile(archiveFile) if err != nil { return nil, errors.Wrap(err, "failed to read archive file") } diff --git a/integration/replicated/pull_test.go b/integration/replicated/pull_test.go index 625d562099..5fc1638b7e 100644 --- a/integration/replicated/pull_test.go +++ b/integration/replicated/pull_test.go @@ -2,6 +2,7 @@ package replicated import ( "io/ioutil" + "os" "path" "testing" @@ -42,11 +43,11 @@ func Test_PullReplicated(t *testing.T) { t.Run(testDir.Name(), func(t *testing.T) { req := require.New(t) - archiveData, err := ioutil.ReadFile(path.Join(testResourcePath, "archive.tar.gz")) + archiveData, err := os.ReadFile(path.Join(testResourcePath, "archive.tar.gz")) req.NoError(err) licenseFilepath := path.Join(testResourcePath, "license.yaml") - licenseFile, err := ioutil.ReadFile(licenseFilepath) + licenseFile, err := os.ReadFile(licenseFilepath) req.NoError(err) server, err := StartMockServer(archiveData, licenseFile) @@ -84,7 +85,7 @@ func Test_PullReplicated(t *testing.T) { err = tarGz.Archive(paths, path.Join(actualFilesystemDir, "archive.tar.gz")) req.NoError(err) - actualFilesystemBytes, err := ioutil.ReadFile(path.Join(actualFilesystemDir, "archive.tar.gz")) + actualFilesystemBytes, err := os.ReadFile(path.Join(actualFilesystemDir, "archive.tar.gz")) req.NoError(err) // create an archive of the expected @@ -98,7 +99,7 @@ func Test_PullReplicated(t *testing.T) { err = tarGz.Archive(paths, path.Join(expectedFilesystemDir, "archive.tar.gz")) req.NoError(err) - expectedFilesystemBytes, err := ioutil.ReadFile(path.Join(expectedFilesystemDir, "archive.tar.gz")) + expectedFilesystemBytes, err := os.ReadFile(path.Join(expectedFilesystemDir, "archive.tar.gz")) req.NoError(err) compareOptions := util.CompareOptions{ diff --git a/kurl_proxy/cmd/main.go b/kurl_proxy/cmd/main.go index b0a69d0f59..1d65b353fb 100644 --- a/kurl_proxy/cmd/main.go +++ b/kurl_proxy/cmd/main.go @@ -549,7 +549,7 @@ type kotsadmApp struct { func kotsadmApplication() (kotsadmApp, error) { app := kotsadmApp{} - data, err := ioutil.ReadFile("/etc/kotsadm/application.yaml") + data, err := os.ReadFile("/etc/kotsadm/application.yaml") if err != nil { return app, errors.Wrap(err, "read file /etc/kotsadm/application.yaml") } diff --git a/pkg/airgap/airgap.go b/pkg/airgap/airgap.go index 083baa5217..f8fedba44a 100644 --- a/pkg/airgap/airgap.go +++ b/pkg/airgap/airgap.go @@ -271,7 +271,7 @@ func CreateAppFromAirgap(opts CreateAirgapAppOpts) (finalError error) { return errors.Wrap(err, "failed to create support bundle dependencies") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(tmpRoot, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(tmpRoot) if err != nil { return errors.Wrap(err, "failed to load kotskinds from path") } diff --git a/pkg/airgap/update.go b/pkg/airgap/update.go index 945dd9cac7..0e0b1e1349 100644 --- a/pkg/airgap/update.go +++ b/pkg/airgap/update.go @@ -87,7 +87,7 @@ func UpdateAppFromPath(a *apptypes.App, airgapRoot string, airgapBundlePath stri } defer os.RemoveAll(archiveDir) - beforeKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + beforeKotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load current kotskinds") } @@ -177,6 +177,7 @@ func UpdateAppFromPath(a *apptypes.App, airgapRoot string, airgapBundlePath stri AppSlug: a.Slug, AppSequence: appSequence, SkipCompatibilityCheck: skipCompatibilityCheck, + KotsKinds: beforeKotsKinds, } if _, err := pull.Pull(fmt.Sprintf("replicated://%s", beforeKotsKinds.License.Spec.AppSlug), pullOptions); err != nil { @@ -185,7 +186,7 @@ func UpdateAppFromPath(a *apptypes.App, airgapRoot string, airgapBundlePath stri } } - afterKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + afterKotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to read after kotskinds") } diff --git a/pkg/apparchive/app.go b/pkg/apparchive/app.go index fdecb3f6a2..976cb78dca 100644 --- a/pkg/apparchive/app.go +++ b/pkg/apparchive/app.go @@ -131,7 +131,7 @@ func GetRenderedApp(versionArchive string, downstreamName, kustomizeBinPath stri return nil } - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return errors.Wrapf(err, "failed to read file %s", path) } @@ -216,7 +216,7 @@ func cleanBaseApp(baseDir string, filter func(path string) (bool, error)) error } } - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return errors.Wrapf(err, "failed to read file %s", path) } diff --git a/pkg/apparchive/helm-v1beta1.go b/pkg/apparchive/helm-v1beta1.go index 614599813c..a9e6ffa0b8 100644 --- a/pkg/apparchive/helm-v1beta1.go +++ b/pkg/apparchive/helm-v1beta1.go @@ -94,7 +94,7 @@ func RenderChartsArchive(baseDir string, overlaysDir string, downstreamName stri } for _, filename := range metadataFiles { - content, err := ioutil.ReadFile(filepath.Join(sourceChartsDir, relPath, filename)) + content, err := os.ReadFile(filepath.Join(sourceChartsDir, relPath, filename)) if err != nil { if os.IsNotExist(err) { continue diff --git a/pkg/apparchive/helm-v1beta1_test.go b/pkg/apparchive/helm-v1beta1_test.go index d32b4f54b7..7a258d77b0 100644 --- a/pkg/apparchive/helm-v1beta1_test.go +++ b/pkg/apparchive/helm-v1beta1_test.go @@ -481,7 +481,7 @@ kind: Kustomization req.NoError(err) for wantPath, wantContent := range tt.wantRenderedFilesMap { - gotContent, err := ioutil.ReadFile(filepath.Join(extracted, "charts", wantPath)) + gotContent, err := os.ReadFile(filepath.Join(extracted, "charts", wantPath)) require.Nil(t, err) if !reflect.DeepEqual(gotContent, wantContent) { @@ -517,7 +517,7 @@ kind: Kustomization req.NoError(err) for wantPath, wantContent := range tt.wantRenderedFilesMap { - gotContent, err := ioutil.ReadFile(filepath.Join(extracted, "charts", wantPath)) + gotContent, err := os.ReadFile(filepath.Join(extracted, "charts", wantPath)) require.Nil(t, err) if !reflect.DeepEqual(gotContent, wantContent) { diff --git a/pkg/apparchive/helm-v1beta2.go b/pkg/apparchive/helm-v1beta2.go index 69fa06fefc..fb43f1cce9 100644 --- a/pkg/apparchive/helm-v1beta2.go +++ b/pkg/apparchive/helm-v1beta2.go @@ -18,6 +18,7 @@ import ( "github.com/replicatedhq/kots/pkg/logger" upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types" "github.com/replicatedhq/kots/pkg/util" + kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" kotsv1beta2 "github.com/replicatedhq/kotskinds/apis/kots/v1beta2" "gopkg.in/yaml.v2" "helm.sh/helm/v3/pkg/action" @@ -81,6 +82,8 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error { return nil } + checkedImages := []kotsv1beta1.InstallationImage{} + for _, v1Beta2Chart := range opts.KotsKinds.V1Beta2HelmCharts.Items { helmChart := v1Beta2Chart @@ -106,7 +109,7 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error { } archivePath := path.Join(chartDir, fmt.Sprintf("%s-%s.tgz", helmChart.Spec.Chart.Name, helmChart.Spec.Chart.ChartVersion)) - if err := ioutil.WriteFile(archivePath, archive, 0644); err != nil { + if err := os.WriteFile(archivePath, archive, 0644); err != nil { return errors.Wrap(err, "failed to write helm chart archive") } @@ -142,7 +145,7 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error { } valuesPath := path.Join(chartDir, "values.yaml") - if err := ioutil.WriteFile(valuesPath, []byte(valuesContent), 0644); err != nil { + if err := os.WriteFile(valuesPath, []byte(valuesContent), 0644); err != nil { return errors.Wrap(err, "failed to write values file") } @@ -159,16 +162,13 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error { return errors.Wrap(err, "failed to process online images") } - upstreamDir := opts.Upstream.GetUpstreamDir(opts.WriteUpstreamOptions) - - installation, err := kotsutil.LoadInstallationFromPath(filepath.Join(upstreamDir, "userdata", "installation.yaml")) - if err != nil { - return errors.Wrap(err, "failed to load kotskinds from new upstream") - } + checkedImages = append(checkedImages, result.CheckedImages...) + } - installation.Spec.KnownImages = append(installation.Spec.KnownImages, result.CheckedImages...) + if len(checkedImages) > 0 { + opts.KotsKinds.Installation.Spec.KnownImages = append(opts.KotsKinds.Installation.Spec.KnownImages, checkedImages...) - if err := SaveInstallation(installation, upstreamDir); err != nil { + if err := SaveInstallation(&opts.KotsKinds.Installation, opts.Upstream.GetUpstreamDir(opts.WriteUpstreamOptions)); err != nil { return errors.Wrap(err, "failed to save installation") } } diff --git a/pkg/apparchive/installation.go b/pkg/apparchive/installation.go index 9d4ef87e7e..d7925a5d5a 100644 --- a/pkg/apparchive/installation.go +++ b/pkg/apparchive/installation.go @@ -1,7 +1,7 @@ package apparchive import ( - "io/ioutil" + "os" "path" "github.com/pkg/errors" @@ -11,7 +11,7 @@ import ( func SaveInstallation(installation *kotsv1beta1.Installation, upstreamDir string) error { filename := path.Join(upstreamDir, "userdata", "installation.yaml") - err := ioutil.WriteFile(filename, kotsutil.MustMarshalInstallation(installation), 0644) + err := os.WriteFile(filename, kotsutil.MustMarshalInstallation(installation), 0644) if err != nil { return errors.Wrap(err, "failed to write installation") } diff --git a/pkg/base/helm.go b/pkg/base/helm.go index d74a865e24..e61b2e8ea7 100644 --- a/pkg/base/helm.go +++ b/pkg/base/helm.go @@ -39,7 +39,7 @@ func RenderHelm(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (*Base, } } - if err := ioutil.WriteFile(p, file.Content, 0644); err != nil { + if err := os.WriteFile(p, file.Content, 0644); err != nil { return nil, errors.Wrap(err, "failed to write chart file") } } @@ -570,7 +570,7 @@ func FindHelmSubChartsFromBase(baseDir, parentChartName string) (*HelmSubCharts, // in the charts folder and need to be excluded when generating the pullsecrets.yaml. It feels like this // could replace the logic below that's doing the file tree walking but I'm unsure. parentChartPath := filepath.Join(searchDir, "Chart.yaml") - parentChartRaw, err := ioutil.ReadFile(parentChartPath) + parentChartRaw, err := os.ReadFile(parentChartPath) if err == nil { parentChart := new(HelmChartDependencies) err = yaml.Unmarshal(parentChartRaw, parentChart) @@ -597,7 +597,7 @@ func FindHelmSubChartsFromBase(baseDir, parentChartName string) (*HelmSubCharts, return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return errors.Wrap(err, "failed to read file") } diff --git a/pkg/base/images.go b/pkg/base/images.go index 74092bd772..946dfcedc4 100644 --- a/pkg/base/images.go +++ b/pkg/base/images.go @@ -9,8 +9,8 @@ import ( "github.com/replicatedhq/kots/pkg/docker/registry" registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types" "github.com/replicatedhq/kots/pkg/image" - kotsimage "github.com/replicatedhq/kots/pkg/image" imagetypes "github.com/replicatedhq/kots/pkg/image/types" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/k8sdoc" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" kustomizeimage "sigs.k8s.io/kustomize/api/types" @@ -77,7 +77,7 @@ func FindPrivateImages(options FindPrivateImagesOptions) (*FindPrivateImagesResu image.NewTag = "latest" } - altNames, err := kotsimage.BuildImageAltNames(image) + altNames, err := imageutil.BuildImageAltNames(image) if err != nil { return nil, errors.Wrap(err, "failed build alt names") } diff --git a/pkg/base/render.go b/pkg/base/render.go index 182cdb3d8b..b9bcdbb504 100644 --- a/pkg/base/render.go +++ b/pkg/base/render.go @@ -2,40 +2,48 @@ package base import ( "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/logger" + registrytypes "github.com/replicatedhq/kots/pkg/registry/types" upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types" ) type RenderOptions struct { - SplitMultiDocYAML bool - Namespace string - HelmVersion string - HelmValues map[string]interface{} - LocalRegistryHost string - LocalRegistryNamespace string - LocalRegistryUsername string - LocalRegistryPassword string - LocalRegistryIsReadOnly bool - ExcludeKotsKinds bool - AppSlug string - Sequence int64 - IsAirgap bool - UseHelmInstall bool - Log *logger.CLILogger + SplitMultiDocYAML bool + Namespace string + HelmVersion string + HelmValues map[string]interface{} + RegistrySettings registrytypes.RegistrySettings + ExcludeKotsKinds bool + AppSlug string + Sequence int64 + IsAirgap bool + UseHelmInstall bool + Log *logger.CLILogger +} + +// RenderKotsKinds is responsible for rendering KOTS custom resources +func RenderKotsKinds(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (map[string][]byte, error) { + renderedKotsKinds, err := renderKotsKinds(u, renderOptions) + if err != nil { + return nil, errors.Wrap(err, "failed to render kots kinds") + } + + return renderedKotsKinds, nil } // RenderUpstream is responsible for any conversions or transpilation steps are required // to take an upstream and make it a valid kubernetes base -func RenderUpstream(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (base *Base, helmBases []Base, renderedKotsKinds map[string][]byte, err error) { +func RenderUpstream(u *upstreamtypes.Upstream, renderOptions *RenderOptions, renderedKotsKinds *kotsutil.KotsKinds) (base *Base, helmBases []Base, err error) { if u.Type == "helm" { base, err = RenderHelm(u, renderOptions) return } if u.Type == "replicated" { - base, helmBases, renderedKotsKinds, err = renderReplicated(u, renderOptions) + base, helmBases, err = renderReplicated(u, renderOptions, renderedKotsKinds) return } - return nil, nil, nil, errors.New("unknown upstream type") + return nil, nil, errors.New("unknown upstream type") } diff --git a/pkg/base/replicated.go b/pkg/base/replicated.go index 3e194ccda9..54c33853b4 100644 --- a/pkg/base/replicated.go +++ b/pkg/base/replicated.go @@ -16,7 +16,6 @@ import ( kotsconfig "github.com/replicatedhq/kots/pkg/config" "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/logger" - registrytypes "github.com/replicatedhq/kots/pkg/registry/types" "github.com/replicatedhq/kots/pkg/template" upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types" "github.com/replicatedhq/kots/pkg/util" @@ -24,7 +23,6 @@ import ( kotsv1beta2 "github.com/replicatedhq/kotskinds/apis/kots/v1beta2" kotsscheme "github.com/replicatedhq/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kotskinds/pkg/helmchart" - troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "helm.sh/helm/v3/pkg/chart" @@ -45,41 +43,15 @@ type Document struct { Kind string `yaml:"kind"` } -func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (*Base, []Base, map[string][]byte, error) { +func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions, renderedKotsKinds *kotsutil.KotsKinds) (*Base, []Base, error) { commonBase := Base{ Files: []BaseFile{}, Bases: []Base{}, } - builder, itemValues, err := NewConfigContextTemplateBuilder(u, renderOptions) + builder, _, err := NewConfigContextTemplateBuilder(u, renderOptions) if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to create new config context template builder") - } - - kotsKinds, err := getKotsKinds(u) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to find config file") - } - - registry := registrytypes.RegistrySettings{ - Hostname: renderOptions.LocalRegistryHost, - Namespace: renderOptions.LocalRegistryNamespace, - Username: renderOptions.LocalRegistryUsername, - Password: renderOptions.LocalRegistryPassword, - IsReadOnly: renderOptions.LocalRegistryIsReadOnly, - } - - versionInfo := template.VersionInfoFromInstallationSpec(renderOptions.Sequence, renderOptions.IsAirgap, kotsKinds.Installation.Spec) - appInfo := template.ApplicationInfo{Slug: renderOptions.AppSlug} - - renderedConfig, err := kotsconfig.TemplateConfigObjects(kotsKinds.Config, itemValues, kotsKinds.License, &kotsKinds.KotsApplication, registry, &versionInfo, &appInfo, kotsKinds.IdentityConfig, util.PodNamespace, true) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to template config objects") - } - - renderedKotsKinds, err := renderKotsKinds(u.Files, renderedConfig, renderOptions, builder) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to render the kots kinds") + return nil, nil, errors.Wrap(err, "failed to create new config context template builder") } for _, upstreamFile := range u.Files { @@ -103,15 +75,15 @@ func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) ( upstreamFile.Content = bytes.Join(newContent, []byte("\n---\n")) } - c, err := processVariadicConfig(&upstreamFile, renderedConfig, renderOptions.Log) + c, err := processVariadicConfig(&upstreamFile, renderedKotsKinds.Config, renderOptions.Log) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to process variadic config in file %s", upstreamFile.Path) + return nil, nil, errors.Wrapf(err, "failed to process variadic config in file %s", upstreamFile.Path) } upstreamFile.Content = c baseFile, err := upstreamFileToBaseFile(upstreamFile, *builder, renderOptions.Log) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to convert upstream file %s to base", upstreamFile.Path) + return nil, nil, errors.Wrapf(err, "failed to convert upstream file %s to base", upstreamFile.Path) } baseFiles := convertToSingleDocBaseFiles([]BaseFile{baseFile}) @@ -119,7 +91,7 @@ func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) ( include, err := f.ShouldBeIncludedInBaseKustomization(renderOptions.ExcludeKotsKinds) if err != nil { if _, ok := err.(ParseError); !ok { - return nil, nil, nil, errors.Wrapf(err, "failed to determine if file %s should be included in base", f.Path) + return nil, nil, errors.Wrapf(err, "failed to determine if file %s should be included in base", f.Path) } } if include { @@ -136,21 +108,21 @@ func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) ( // NOTE: we only render v1beta1 HelmCharts to base kotsV1Beta1HelmCharts, err := findAllKotsV1Beta1HelmCharts(u.Files, *builder, renderOptions.Log) if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to find helm charts") + return nil, nil, errors.Wrap(err, "failed to find helm charts") } helmBases := []Base{} for _, kotsHelmChart := range kotsV1Beta1HelmCharts { helmBase, err := renderReplicatedHelmChart(&kotsHelmChart, u.Files, renderOptions, builder) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to render helm chart %s", kotsHelmChart.Name) + return nil, nil, errors.Wrapf(err, "failed to render helm chart %s", kotsHelmChart.Name) } else if helmBase == nil { continue } renderedHelmBase, err := renderReplicatedHelmBase(u, renderOptions, *helmBase, *builder) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to render helm chart base %s", helmBase.Path) + return nil, nil, errors.Wrapf(err, "failed to render helm chart base %s", helmBase.Path) } if kotsHelmChart.Spec.UseHelmInstall { @@ -160,13 +132,31 @@ func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) ( } } - return &commonBase, helmBases, renderedKotsKinds, nil + return &commonBase, helmBases, nil } -func renderKotsKinds(upstreamFiles []upstreamtypes.UpstreamFile, renderedConfig *kotsv1beta1.Config, renderOptions *RenderOptions, builder *template.Builder) (map[string][]byte, error) { - renderedKotsKinds := make(map[string][]byte) +func renderKotsKinds(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (map[string][]byte, error) { + renderedKotsKindsMap := make(map[string][]byte) - for _, upstreamFile := range upstreamFiles { + builder, itemValues, err := NewConfigContextTemplateBuilder(u, renderOptions) + if err != nil { + return nil, errors.Wrap(err, "failed to create new config context template builder") + } + + tkk, err := getTemplatingKotsKinds(u) + if err != nil { + return nil, errors.Wrap(err, "failed to find config file") + } + + versionInfo := template.VersionInfoFromInstallationSpec(renderOptions.Sequence, renderOptions.IsAirgap, tkk.Installation.Spec) + appInfo := template.ApplicationInfo{Slug: renderOptions.AppSlug} + + renderedConfig, err := kotsconfig.TemplateConfigObjects(tkk.Config, itemValues, tkk.License, &tkk.KotsApplication, renderOptions.RegistrySettings, &versionInfo, &appInfo, tkk.IdentityConfig, util.PodNamespace, true) + if err != nil { + return nil, errors.Wrap(err, "failed to template config objects") + } + + for _, upstreamFile := range u.Files { for _, doc := range util.ConvertToSingleDocs(upstreamFile.Content) { gvk := OverlySimpleGVK{} if err := yaml.Unmarshal(doc, &gvk); err != nil { @@ -181,8 +171,16 @@ func renderKotsKinds(upstreamFiles []upstreamtypes.UpstreamFile, renderedConfig switch gvkString { case "kots.io/v1beta1, Kind=Installation": - // Installation manifests are generated later and will have a different filename. - continue + // Installation manifest does not need rendering. + + case "kots.io/v1beta1, Kind=License": + // License manifest does not need rendering. + + case "kots.io/v1beta1, Kind=IdentityConfig": + // IdentityConfig manifest does not need rendering. + + case "kots.io/v1beta1, Kind=ConfigValues": + // ConfigValues manifest does not need rendering. case "kots.io/v1beta1, Kind=Config": // Use the rendered Config instead of the upstream. @@ -193,9 +191,6 @@ func renderKotsKinds(upstreamFiles []upstreamtypes.UpstreamFile, renderedConfig } doc = []byte(config) - case "kots.io/v1beta1, Kind=ConfigValues": - // ConfigValues do not need rendering since they should already be valid values. - case "kots.io/v1beta1, Kind=HelmChart", "kots.io/v1beta2, Kind=HelmChart": helmchart, err := builder.RenderTemplate(upstreamFile.Path, string(upstreamFile.Content)) if err != nil { @@ -216,15 +211,15 @@ func renderKotsKinds(upstreamFiles []upstreamtypes.UpstreamFile, renderedConfig doc = []byte(bytes) } - if existing, exists := renderedKotsKinds[upstreamFile.Path]; exists { + if existing, exists := renderedKotsKindsMap[upstreamFile.Path]; exists { doc = bytes.Join([][]byte{existing, doc}, []byte("\n---\n")) } - renderedKotsKinds[upstreamFile.Path] = doc + renderedKotsKindsMap[upstreamFile.Path] = doc } } - return renderedKotsKinds, nil + return renderedKotsKindsMap, nil } func extractHelmBases(b Base) []Base { @@ -444,7 +439,8 @@ func tryGetConfigFromFileContent(content []byte, log *logger.CLILogger) *kotsv1b return nil } -func getKotsKinds(u *upstreamtypes.Upstream) (*kotsutil.KotsKinds, error) { +// getTemplatingKotsKinds returns kots kinds that are necessary/used to populate the contexts of the template builder +func getTemplatingKotsKinds(u *upstreamtypes.Upstream) (*kotsutil.KotsKinds, error) { kotsKinds := &kotsutil.KotsKinds{} for _, file := range u.Files { @@ -459,7 +455,7 @@ func getKotsKinds(u *upstreamtypes.Upstream) (*kotsutil.KotsKinds, error) { decoded, gvk, err := decode(doc, nil, nil) if err != nil { if document.APIVersion == "kots.io/v1beta1" && (document.Kind == "Config" || document.Kind == "License") { - errMessage := fmt.Sprintf("Failed to decode %s", file.Path) + errMessage := fmt.Sprintf("Failed to decode %s: %v", file.Path, string(doc)) return nil, errors.Wrap(err, errMessage) } continue @@ -474,26 +470,10 @@ func getKotsKinds(u *upstreamtypes.Upstream) (*kotsutil.KotsKinds, error) { kotsKinds.KotsApplication = *decoded.(*kotsv1beta1.Application) case "kots.io/v1beta1, Kind=License": kotsKinds.License = decoded.(*kotsv1beta1.License) - case "kots.io/v1beta1, Kind=Identity": - kotsKinds.Identity = decoded.(*kotsv1beta1.Identity) case "kots.io/v1beta1, Kind=IdentityConfig": kotsKinds.IdentityConfig = decoded.(*kotsv1beta1.IdentityConfig) case "kots.io/v1beta1, Kind=Installation": kotsKinds.Installation = *decoded.(*kotsv1beta1.Installation) - case "troubleshoot.sh/v1beta2, Kind=Collector": - kotsKinds.Collector = decoded.(*troubleshootv1beta2.Collector) - case "troubleshoot.sh/v1beta2, Kind=Analyzer": - kotsKinds.Analyzer = decoded.(*troubleshootv1beta2.Analyzer) - case "troubleshoot.sh/v1beta2, Kind=SupportBundle": - kotsKinds.SupportBundle = decoded.(*troubleshootv1beta2.SupportBundle) - case "troubleshoot.sh/v1beta2, Kind=Redactor": - kotsKinds.Redactor = decoded.(*troubleshootv1beta2.Redactor) - case "troubleshoot.sh/v1beta2, Kind=Preflight": - kotsKinds.Preflight = decoded.(*troubleshootv1beta2.Preflight) - case "velero.io/v1, Kind=Backup": - kotsKinds.Backup = decoded.(*velerov1.Backup) - case "app.k8s.io/v1beta1, Kind=Application": - kotsKinds.Application = decoded.(*applicationv1beta1.Application) } } } diff --git a/pkg/base/replicated_test.go b/pkg/base/replicated_test.go index b65b42aa55..91664ae14f 100644 --- a/pkg/base/replicated_test.go +++ b/pkg/base/replicated_test.go @@ -862,7 +862,17 @@ status: {} t.Run(test.name, func(t *testing.T) { req := require.New(t) - base, _, kotsKinds, err := renderReplicated(test.upstream, test.renderOptions) + renderedKotsKindsMap, err := renderKotsKinds(test.upstream, test.renderOptions) + req.NoError(err) + + renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) + req.NoError(err) + + expectedKotsKinds, err := kotsutil.KotsKindsFromMap(test.expectedKotsKinds) + req.NoError(err) + req.Equal(expectedKotsKinds, renderedKotsKinds) + + base, _, err := renderReplicated(test.upstream, test.renderOptions, renderedKotsKinds) req.NoError(err) decode := scheme.Codecs.UniversalDeserializer().Decode @@ -876,12 +886,6 @@ status: {} expectedMultidoc := multidocobj.(*corev1.ServiceAccount) - expKindsStruct, err := kotsutil.KotsKindsFromMap(test.expectedKotsKinds) - req.NoError(err) - kindsStruct, err := kotsutil.KotsKindsFromMap(kotsKinds) - req.NoError(err) - req.Equal(expKindsStruct, kindsStruct) - var unmarshaledSecrets []*corev1.Secret for _, expectedSecret := range test.expectedSecrets { secobj, _, err := decode(expectedSecret.Content, nil, nil) @@ -1910,21 +1914,25 @@ version: 1.10.1 t.Run(test.name, func(t *testing.T) { req := require.New(t) - base, helmBase, kotsKinds, err := renderReplicated(test.upstream, test.renderOptions) + renderedKotsKindsMap, err := renderKotsKinds(test.upstream, test.renderOptions) + req.NoError(err) + + renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) req.NoError(err) - req.ElementsMatch(test.expectedHelm, helmBase) - req.ElementsMatch(test.expectedBase.Files, base.Files) - expKindsStruct, err := kotsutil.KotsKindsFromMap(test.expectedKotsKinds) + expectedKotsKinds, err := kotsutil.KotsKindsFromMap(test.expectedKotsKinds) req.NoError(err) - kindsStruct, err := kotsutil.KotsKindsFromMap(kotsKinds) + req.Equal(expectedKotsKinds, renderedKotsKinds) + + base, helmBase, err := renderReplicated(test.upstream, test.renderOptions, renderedKotsKinds) req.NoError(err) - req.Equal(expKindsStruct, kindsStruct) + req.ElementsMatch(test.expectedHelm, helmBase) + req.ElementsMatch(test.expectedBase.Files, base.Files) }) } } -func Test_getKotsKinds(t *testing.T) { +func Test_getTemplatingKotsKinds(t *testing.T) { type args struct { u *upstreamtypes.Upstream } @@ -2142,13 +2150,13 @@ spec: for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getKotsKinds(tt.args.u) + got, err := getTemplatingKotsKinds(tt.args.u) if (err != nil) != tt.wantErr { - t.Errorf("getKotsKinds() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("getTemplatingKotsKinds() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getKotsKinds() = %v, want %v", got, tt.want) + t.Errorf("getTemplatingKotsKinds() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/base/templates.go b/pkg/base/templates.go index 8d453633cf..d177a07d5c 100644 --- a/pkg/base/templates.go +++ b/pkg/base/templates.go @@ -2,14 +2,13 @@ package base import ( "github.com/pkg/errors" - registrytypes "github.com/replicatedhq/kots/pkg/registry/types" "github.com/replicatedhq/kots/pkg/template" upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" ) func NewConfigContextTemplateBuilder(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (*template.Builder, map[string]template.ItemValue, error) { - kotsKinds, err := getKotsKinds(u) + kotsKinds, err := getTemplatingKotsKinds(u) if err != nil { return nil, nil, err } @@ -35,14 +34,6 @@ func NewConfigContextTemplateBuilder(u *upstreamtypes.Upstream, renderOptions *R configGroups = kotsKinds.Config.Spec.Groups } - localRegistry := registrytypes.RegistrySettings{ - Hostname: renderOptions.LocalRegistryHost, - Namespace: renderOptions.LocalRegistryNamespace, - Username: renderOptions.LocalRegistryUsername, - Password: renderOptions.LocalRegistryPassword, - IsReadOnly: renderOptions.LocalRegistryIsReadOnly, - } - appInfo := template.ApplicationInfo{ Slug: renderOptions.AppSlug, } @@ -62,7 +53,7 @@ func NewConfigContextTemplateBuilder(u *upstreamtypes.Upstream, renderOptions *R builderOptions := template.BuilderOptions{ ConfigGroups: configGroups, ExistingValues: templateContext, - LocalRegistry: localRegistry, + LocalRegistry: renderOptions.RegistrySettings, License: kotsKinds.License, Application: &kotsKinds.KotsApplication, VersionInfo: &versionInfo, diff --git a/pkg/base/write.go b/pkg/base/write.go index 4e200426ec..c390e8f6f5 100644 --- a/pkg/base/write.go +++ b/pkg/base/write.go @@ -270,7 +270,7 @@ func (b *Base) getErrorFiles() []BaseFile { func deduplicateResources(filePaths []string, baseDir string, excludeKotsKinds bool, baseNS string) ([]string, []kustomizetypes.PatchStrategicMerge, error) { files := []BaseFile{} for _, filePath := range filePaths { - content, err := ioutil.ReadFile(filepath.Join(baseDir, filePath)) + content, err := os.ReadFile(filepath.Join(baseDir, filePath)) if err != nil { return nil, nil, errors.Wrapf(err, "failed to read base file %s", filePath) } diff --git a/pkg/base/write_images.go b/pkg/base/write_images.go index a7621a72a6..3fd24053ff 100644 --- a/pkg/base/write_images.go +++ b/pkg/base/write_images.go @@ -37,7 +37,12 @@ func RewriteImages(options RewriteImageOptions) (*RewriteImagesResult, error) { checkedImages := make(map[string]imagetypes.ImageInfo) if options.KotsKinds != nil { - additionalImages = kotsutil.GetImagesFromKotsKinds(options.KotsKinds) + kki, err := kotsutil.GetImagesFromKotsKinds(options.KotsKinds, &options.DestRegistry) + if err != nil { + return nil, errors.Wrap(err, "failed to get images from kots kinds") + } + additionalImages = kki + checkedImages = makeImageInfoMap(options.KotsKinds.Installation.Spec.KnownImages) if options.KotsKinds.KotsApplication.Spec.ProxyPublicImages { allImagesPrivate = true diff --git a/pkg/base/write_images_test.go b/pkg/base/write_images_test.go index 640568f2f3..4b1759ac62 100644 --- a/pkg/base/write_images_test.go +++ b/pkg/base/write_images_test.go @@ -2,6 +2,7 @@ package base import ( "io/ioutil" + "os" "path/filepath" "testing" @@ -55,6 +56,11 @@ func Test_RewriteImages(t *testing.T) { Image: "quay.io/replicatedcom/qa-kots-1:alpine-3.5", }, }, + { + Run: &troubleshootv1beta2.Run{ + Image: "testing.registry.com:5000/testing-ns/random-image:2", + }, + }, { RunPod: &troubleshootv1beta2.RunPod{ PodSpec: corev1.PodSpec{ @@ -77,6 +83,11 @@ func Test_RewriteImages(t *testing.T) { Image: "quay.io/replicatedcom/qa-kots-2:alpine-3.4", }, }, + { + Run: &troubleshootv1beta2.Run{ + Image: "testing.registry.com:5000/testing-ns/random-image:1", + }, + }, }, }, }, @@ -84,7 +95,7 @@ func Test_RewriteImages(t *testing.T) { CopyImages: false, AppSlug: "test-app-slug", DestRegistry: registrytypes.RegistryOptions{ - Endpoint: "ttl.sh", + Endpoint: "testing.registry.com:5000", Namespace: "testing-ns", Username: "testing-user-name", Password: "testing-password", @@ -94,87 +105,87 @@ func Test_RewriteImages(t *testing.T) { Images: []kustomizeimage.Image{ { Name: "busybox", - NewName: "ttl.sh/testing-ns/busybox", + NewName: "testing.registry.com:5000/testing-ns/busybox", NewTag: "latest", }, { Name: "docker.io/library/busybox", - NewName: "ttl.sh/testing-ns/busybox", + NewName: "testing.registry.com:5000/testing-ns/busybox", NewTag: "latest", }, { Name: "library/busybox", - NewName: "ttl.sh/testing-ns/busybox", + NewName: "testing.registry.com:5000/testing-ns/busybox", NewTag: "latest", }, { Name: "docker.io/busybox", - NewName: "ttl.sh/testing-ns/busybox", + NewName: "testing.registry.com:5000/testing-ns/busybox", NewTag: "latest", }, { Name: "registry.replicated.com/appslug/image", - NewName: "ttl.sh/testing-ns/image", + NewName: "testing.registry.com:5000/testing-ns/image", NewTag: "version", }, { Name: "quay.io/replicatedcom/qa-kots-1", - NewName: "ttl.sh/testing-ns/qa-kots-1", + NewName: "testing.registry.com:5000/testing-ns/qa-kots-1", NewTag: "alpine-3.5", }, { Name: "quay.io/replicatedcom/qa-kots-2", - NewName: "ttl.sh/testing-ns/qa-kots-2", + NewName: "testing.registry.com:5000/testing-ns/qa-kots-2", NewTag: "alpine-3.4", }, { Name: "quay.io/replicatedcom/qa-kots-3", - NewName: "ttl.sh/testing-ns/qa-kots-3", + NewName: "testing.registry.com:5000/testing-ns/qa-kots-3", NewTag: "alpine-3.6", }, { Name: "quay.io/replicatedcom/someimage", - NewName: "ttl.sh/testing-ns/someimage", + NewName: "testing.registry.com:5000/testing-ns/someimage", Digest: "sha256:25dedae0aceb6b4fe5837a0acbacc6580453717f126a095aa05a3c6fcea14dd4", }, { Name: "nginx", - NewName: "ttl.sh/testing-ns/nginx", + NewName: "testing.registry.com:5000/testing-ns/nginx", NewTag: "1", }, { Name: "docker.io/library/nginx", - NewName: "ttl.sh/testing-ns/nginx", + NewName: "testing.registry.com:5000/testing-ns/nginx", NewTag: "1", }, { Name: "library/nginx", - NewName: "ttl.sh/testing-ns/nginx", + NewName: "testing.registry.com:5000/testing-ns/nginx", NewTag: "1", }, { Name: "docker.io/nginx", - NewName: "ttl.sh/testing-ns/nginx", + NewName: "testing.registry.com:5000/testing-ns/nginx", NewTag: "1", }, { Name: "redis", - NewName: "ttl.sh/testing-ns/redis", + NewName: "testing.registry.com:5000/testing-ns/redis", Digest: "sha256:e96c03a6dda7d0f28e2de632048a3d34bb1636d0858b65ef9a554441c70f6633", }, { Name: "docker.io/library/redis", - NewName: "ttl.sh/testing-ns/redis", + NewName: "testing.registry.com:5000/testing-ns/redis", Digest: "sha256:e96c03a6dda7d0f28e2de632048a3d34bb1636d0858b65ef9a554441c70f6633", }, { Name: "library/redis", - NewName: "ttl.sh/testing-ns/redis", + NewName: "testing.registry.com:5000/testing-ns/redis", Digest: "sha256:e96c03a6dda7d0f28e2de632048a3d34bb1636d0858b65ef9a554441c70f6633", }, { Name: "docker.io/redis", - NewName: "ttl.sh/testing-ns/redis", + NewName: "testing.registry.com:5000/testing-ns/redis", Digest: "sha256:e96c03a6dda7d0f28e2de632048a3d34bb1636d0858b65ef9a554441c70f6633", }, }, @@ -260,6 +271,14 @@ func Test_RewriteImages(t *testing.T) { Image: "quay.io/replicatedcom/someimage:1@sha256:25dedae0aceb6b4fe5837a0acbacc6580453717f126a095aa05a3c6fcea14dd4", IsPrivate: true, }, + { + Image: "testing.registry.com:5000/testing-ns/random-image:2", + IsPrivate: true, + }, + { + Image: "testing.registry.com:5000/testing-ns/random-image:1", + IsPrivate: true, + }, { Image: "redis:7@sha256:e96c03a6dda7d0f28e2de632048a3d34bb1636d0858b65ef9a554441c70f6633", IsPrivate: false, @@ -490,13 +509,15 @@ func Test_RewriteImages(t *testing.T) { t.Run(test.name, func(t *testing.T) { req := require.New(t) - gotUpstreamResult, err := RewriteImages(test.processOptions) + gotResult, err := RewriteImages(test.processOptions) req.NoError(err) - assert.ElementsMatch(t, test.wantProcessResult.Images, gotUpstreamResult.Images) - assert.ElementsMatch(t, test.wantProcessResult.CheckedImages, gotUpstreamResult.CheckedImages) + assert.ElementsMatch(t, test.wantProcessResult.Images, gotResult.Images) + assert.ElementsMatch(t, test.wantProcessResult.CheckedImages, gotResult.CheckedImages) + + test.findOptions.KotsKindsImages, err = kotsutil.GetImagesFromKotsKinds(test.processOptions.KotsKinds, nil) + req.NoError(err) - test.findOptions.KotsKindsImages = kotsutil.GetImagesFromKotsKinds(test.processOptions.KotsKinds) gotFindResult, err := FindPrivateImages(test.findOptions) req.NoError(err) @@ -522,7 +543,7 @@ func loadDocs(basePath string) ([]k8sdoc.K8sDoc, error) { if file.IsDir() { continue } - content, err := ioutil.ReadFile(filepath.Join(basePath, file.Name())) + content, err := os.ReadFile(filepath.Join(basePath, file.Name())) if err != nil { return nil, errors.Wrap(err, "read file") } diff --git a/pkg/docker/registry/images.go b/pkg/docker/registry/images.go index 90f1955cdd..2a98aeb12d 100644 --- a/pkg/docker/registry/images.go +++ b/pkg/docker/registry/images.go @@ -2,17 +2,18 @@ package registry import ( "strings" + + "github.com/replicatedhq/kots/pkg/imageutil" ) func MakeProxiedImageURL(proxyHost string, appSlug string, image string) string { - parts := strings.Split(image, "@") - if len(parts) == 2 { - // we have a digest, but need to also check for a tag - parts = strings.Split(parts[0], ":") - return strings.Join([]string{proxyHost, "proxy", appSlug, parts[0]}, "/") + untagged := imageutil.StripImageTagAndDigest(image) + + registryHost := strings.Split(untagged, "/")[0] + if registryHost == proxyHost { + // already proxied + return untagged } - // TODO: host with a port breaks this - parts = strings.Split(image, ":") - return strings.Join([]string{proxyHost, "proxy", appSlug, parts[0]}, "/") + return strings.Join([]string{proxyHost, "proxy", appSlug, untagged}, "/") } diff --git a/pkg/docker/registry/images_test.go b/pkg/docker/registry/images_test.go index fa2927fcf9..b231a0bfa2 100644 --- a/pkg/docker/registry/images_test.go +++ b/pkg/docker/registry/images_test.go @@ -11,33 +11,188 @@ func Test_MakeProxiedImageURL(t *testing.T) { want string }{ { - name: "MakeProxiedImageURL with multi part image parameter with @ character returns valid proxied image URL", + name: "untagged image", proxyHost: "host", appSlug: "slug", - image: "image@image", + image: "image", want: "host/proxy/slug/image", }, { - name: "MakeProxiedImageURL multi part image parameter with : character returns valid proxied image URL", + name: "untagged image on non-ported registry", proxyHost: "host", appSlug: "slug", - image: "image:image", + image: "registry/image", + want: "host/proxy/slug/registry/image", + }, + { + name: "untagged image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry:5000/image", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tagged image", + proxyHost: "host", + appSlug: "slug", + image: "image:tag", + want: "host/proxy/slug/image", + }, + { + name: "tagged image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry/image:tag", + want: "host/proxy/slug/registry/image", + }, + { + name: "untagged image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry:5000/image:tag", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "digest image", + proxyHost: "host", + appSlug: "slug", + image: "image@digest", want: "host/proxy/slug/image", }, { - name: "MakeProxiedImageURL multi part image parameter with a namespace returns valid proxied image URL", + name: "digest image on non-ported registry", proxyHost: "host", appSlug: "slug", - image: "namespace/image:image", - want: "host/proxy/slug/namespace/image", + image: "registry/image@digest", + want: "host/proxy/slug/registry/image", }, { - name: "MakeProxiedImageURL multi part image parameter with : and @ characters returns valid proxied image URL", + name: "digest image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry:5000/image@digest", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tag and digest image", proxyHost: "host", appSlug: "slug", image: "image:tag@digest", want: "host/proxy/slug/image", }, + { + name: "tag and digest image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry/image:tag@digest", + want: "host/proxy/slug/registry/image", + }, + { + name: "tag and digest image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "registry:5000/image:tag@digest", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tag and digest image on ported registry with namespace", + proxyHost: "host", + appSlug: "slug", + image: "registry:5000/namespace/image:tag@digest", + want: "host/proxy/slug/registry:5000/namespace/image", + }, + // ---- test cases for images that are already proxied ---- // + { + name: "untagged proxied image", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/image", + want: "host/proxy/slug/image", + }, + { + name: "untagged proxied image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry/image", + want: "host/proxy/slug/registry/image", + }, + { + name: "untagged proxied image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry:5000/image", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tagged proxied image", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/image:tag", + want: "host/proxy/slug/image", + }, + { + name: "tagged proxied image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry/image:tag", + want: "host/proxy/slug/registry/image", + }, + { + name: "untagged proxied image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry:5000/image:tag", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "digest proxied image", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/image@digest", + want: "host/proxy/slug/image", + }, + { + name: "digest proxied image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry/image@digest", + want: "host/proxy/slug/registry/image", + }, + { + name: "digest proxied image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry:5000/image@digest", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tag and digest proxied image", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/image:tag@digest", + want: "host/proxy/slug/image", + }, + { + name: "tag and digest proxied image on non-ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry/image:tag@digest", + want: "host/proxy/slug/registry/image", + }, + { + name: "tag and digest proxied image on ported registry", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry:5000/image:tag@digest", + want: "host/proxy/slug/registry:5000/image", + }, + { + name: "tag and digest proxied image on ported registry with namespace", + proxyHost: "host", + appSlug: "slug", + image: "host/proxy/slug/registry:5000/namespace/image:tag@digest", + want: "host/proxy/slug/registry:5000/namespace/image", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/gitops/gitops.go b/pkg/gitops/gitops.go index 4157d6c5dd..7dee4915d8 100644 --- a/pkg/gitops/gitops.go +++ b/pkg/gitops/gitops.go @@ -713,7 +713,7 @@ func getAuth(privateKey string) (transport.AuthMethod, error) { } func CreateGitOpsCommit(gitOpsConfig *GitOpsConfig, appSlug string, appName string, newSequence int, archiveDir string, downstreamName string) (string, error) { - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return "", errors.Wrap(err, "failed to load kots kinds") } @@ -764,7 +764,7 @@ func CreateGitOpsCommit(gitOpsConfig *GitOpsConfig, appSlug string, appName stri filePath := filepath.Join(dirPath, fmt.Sprintf("%s.yaml", appSlug)) _, err = os.Stat(filePath) if err == nil { // if the file has not changed, end now - currentRevision, err := ioutil.ReadFile(filePath) + currentRevision, err := os.ReadFile(filePath) if err != nil { return "", errors.Wrap(err, "failed to read current app yaml") } diff --git a/pkg/handlers/config.go b/pkg/handlers/config.go index 814a7ef151..9aee7e35c6 100644 --- a/pkg/handlers/config.go +++ b/pkg/handlers/config.go @@ -271,6 +271,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { } var kotsKinds *kotsutil.KotsKinds + var nonRenderedConfig *kotsv1beta1.Config var appLicense *kotsv1beta1.License var app apptypes.AppType var localRegistry registrytypes.RegistrySettings @@ -295,6 +296,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { return } kotsKinds = &k + nonRenderedConfig = kotsKinds.Config appLicense = kotsKinds.License createNewVersion = true } else { @@ -321,6 +323,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { k.License = licenseData.License kotsKinds = &k + nonRenderedConfig = kotsKinds.Config appLicense = kotsKinds.License } } else { @@ -341,7 +344,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { return } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { liveAppConfigResponse.Error = "failed to create temp dir" logger.Error(errors.Wrap(err, liveAppConfigResponse.Error)) @@ -358,7 +361,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { return } - kotsKinds, err = kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err = kotsutil.LoadKotsKinds(archiveDir) if err != nil { liveAppConfigResponse.Error = "failed to load kots kinds from path" logger.Error(errors.Wrap(err, liveAppConfigResponse.Error)) @@ -366,6 +369,15 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { return } + // get the non-rendered config from the upstream directory because we have to re-render it with the new values + nonRenderedConfig, err = kotsutil.FindConfigInPath(filepath.Join(archiveDir, "upstream")) + if err != nil { + liveAppConfigResponse.Error = "failed to find non-rendered config" + logger.Error(errors.Wrap(err, liveAppConfigResponse.Error)) + JSON(w, http.StatusInternalServerError, liveAppConfigResponse) + return + } + registryInfo, err := store.GetStore().GetRegistryDetailsForApp(foundApp.ID) if err != nil { liveAppConfigResponse.Error = "failed to get app registry info" @@ -392,7 +404,7 @@ func (h *Handler) LiveAppConfig(w http.ResponseWriter, r *http.Request) { versionInfo := template.VersionInfoFromInstallationSpec(sequence, app.GetIsAirgap(), kotsKinds.Installation.Spec) // sequence +1 because the sequence will be incremented on save (and we want the preview to be accurate) appInfo := template.ApplicationInfo{Slug: app.GetSlug()} - renderedConfig, err := kotsconfig.TemplateConfigObjects(kotsKinds.Config, configValues, appLicense, &kotsKinds.KotsApplication, localRegistry, &versionInfo, &appInfo, kotsKinds.IdentityConfig, app.GetNamespace(), false) + renderedConfig, err := kotsconfig.TemplateConfigObjects(nonRenderedConfig, configValues, appLicense, &kotsKinds.KotsApplication, localRegistry, &versionInfo, &appInfo, kotsKinds.IdentityConfig, app.GetNamespace(), false) if err != nil { liveAppConfigResponse.Error = "failed to render templates" logger.Error(errors.Wrap(err, liveAppConfigResponse.Error)) @@ -483,6 +495,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { } var kotsKinds *kotsutil.KotsKinds + var nonRenderedConfig *kotsv1beta1.Config var license *kotsv1beta1.License var localRegistry registrytypes.RegistrySettings var app apptypes.AppType @@ -520,6 +533,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { return } kotsKinds = &k + nonRenderedConfig = kotsKinds.Config license = kotsKinds.License createNewVersion = true } else { @@ -555,6 +569,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { k.License = licenseData.License kotsKinds = &k + nonRenderedConfig = kotsKinds.Config license = kotsKinds.License } } else { @@ -590,7 +605,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { return } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { currentAppConfigResponse.Error = "failed to create temp dir" logger.Error(errors.Wrap(err, currentAppConfigResponse.Error)) @@ -607,7 +622,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { return } - kotsKinds, err = kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err = kotsutil.LoadKotsKinds(archiveDir) if err != nil { currentAppConfigResponse.Error = "failed to load kots kinds from path" logger.Error(errors.Wrap(err, currentAppConfigResponse.Error)) @@ -615,6 +630,15 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { return } + // get the non-rendered config from the upstream directory because we have to re-render it with the new values + nonRenderedConfig, err = kotsutil.FindConfigInPath(filepath.Join(archiveDir, "upstream")) + if err != nil { + currentAppConfigResponse.Error = "failed to find non-rendered config" + logger.Error(errors.Wrap(err, currentAppConfigResponse.Error)) + JSON(w, http.StatusInternalServerError, currentAppConfigResponse) + return + } + registryInfo, err := store.GetStore().GetRegistryDetailsForApp(foundApp.ID) if err != nil { currentAppConfigResponse.Error = "failed to get app registry info" @@ -657,7 +681,7 @@ func (h *Handler) CurrentAppConfig(w http.ResponseWriter, r *http.Request) { versionInfo := template.VersionInfoFromInstallationSpec(sequence, app.GetIsAirgap(), kotsKinds.Installation.Spec) // sequence +1 because the sequence will be incremented on save (and we want the preview to be accurate) appInfo := template.ApplicationInfo{Slug: app.GetSlug()} - renderedConfig, err := kotsconfig.TemplateConfigObjects(kotsKinds.Config, configValues, license, &kotsKinds.KotsApplication, localRegistry, &versionInfo, &appInfo, kotsKinds.IdentityConfig, app.GetNamespace(), false) + renderedConfig, err := kotsconfig.TemplateConfigObjects(nonRenderedConfig, configValues, license, &kotsKinds.KotsApplication, localRegistry, &versionInfo, &appInfo, kotsKinds.IdentityConfig, app.GetNamespace(), false) if err != nil { logger.Error(err) currentAppConfigResponse.Error = "failed to render templates" @@ -749,7 +773,7 @@ func getAppConfigValueForFile(downloadApp *apptypes.App, sequence int64, filenam return "", errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return "", errors.Wrap(err, "failed to load kots kinds from archive") } @@ -770,7 +794,7 @@ func updateAppConfig(updateApp *apptypes.App, sequence int64, configGroups []kot Success: false, } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { updateAppConfigResponse.Error = "failed to create temp dir" return updateAppConfigResponse, err @@ -783,7 +807,7 @@ func updateAppConfig(updateApp *apptypes.App, sequence int64, configGroups []kot return updateAppConfigResponse, err } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { updateAppConfigResponse.Error = "failed to load kots kinds from path" return updateAppConfigResponse, err @@ -810,7 +834,7 @@ func updateAppConfig(updateApp *apptypes.App, sequence int64, configGroups []kot return updateAppConfigResponse, err } - if err := ioutil.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "config.yaml"), []byte(configValuesSpec), 0644); err != nil { + if err := os.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "config.yaml"), []byte(configValuesSpec), 0644); err != nil { updateAppConfigResponse.Error = "failed to write config.yaml to upstream/userdata" return updateAppConfigResponse, err } @@ -1079,7 +1103,7 @@ func (h *Handler) SetAppConfigValues(w http.ResponseWriter, r *http.Request) { return } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { setAppConfigValuesResponse.Error = "failed to load kots kinds from path" logger.Error(errors.Wrap(err, setAppConfigValuesResponse.Error)) @@ -1087,7 +1111,16 @@ func (h *Handler) SetAppConfigValues(w http.ResponseWriter, r *http.Request) { return } - if kotsKinds.Config == nil { + // get the non-rendered config from the upstream directory because we have to re-render it with the new values + nonRenderedConfig, err := kotsutil.FindConfigInPath(filepath.Join(archiveDir, "upstream")) + if err != nil { + setAppConfigValuesResponse.Error = "failed to find non-rendered config" + logger.Error(errors.Wrap(err, setAppConfigValuesResponse.Error)) + JSON(w, http.StatusInternalServerError, setAppConfigValuesResponse) + return + } + + if nonRenderedConfig == nil { setAppConfigValuesResponse.Error = fmt.Sprintf("app %s does not have a config", foundApp.Slug) logger.Errorf(setAppConfigValuesResponse.Error) JSON(w, http.StatusInternalServerError, setAppConfigValuesResponse) @@ -1102,7 +1135,7 @@ func (h *Handler) SetAppConfigValues(w http.ResponseWriter, r *http.Request) { return } - newConfigValues, err = mergeConfigValues(kotsKinds.Config, kotsKinds.ConfigValues, newConfigValues) + newConfigValues, err = mergeConfigValues(nonRenderedConfig, kotsKinds.ConfigValues, newConfigValues) if err != nil { setAppConfigValuesResponse.Error = "failed to create new config" logger.Error(errors.Wrap(err, setAppConfigValuesResponse.Error)) @@ -1111,7 +1144,7 @@ func (h *Handler) SetAppConfigValues(w http.ResponseWriter, r *http.Request) { } } - newConfig, err := updateConfigObject(kotsKinds.Config, newConfigValues, setAppConfigValuesRequest.Merge) + newConfig, err := updateConfigObject(nonRenderedConfig, newConfigValues, setAppConfigValuesRequest.Merge) if err != nil { setAppConfigValuesResponse.Error = "failed to create new config object" logger.Error(errors.Wrap(err, setAppConfigValuesResponse.Error)) diff --git a/pkg/handlers/contents.go b/pkg/handlers/contents.go index db0ee8dcab..de1c4abb1a 100644 --- a/pkg/handlers/contents.go +++ b/pkg/handlers/contents.go @@ -75,7 +75,7 @@ func (h *Handler) GetAppContents(w http.ResponseWriter, r *http.Request) { return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } diff --git a/pkg/handlers/download.go b/pkg/handlers/download.go index 4f6e01f56a..2ba9b09679 100644 --- a/pkg/handlers/download.go +++ b/pkg/handlers/download.go @@ -73,7 +73,7 @@ func (h *Handler) DownloadApp(w http.ResponseWriter, r *http.Request) { } if decryptPasswordValues { - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archivePath, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archivePath) if err != nil { logger.Error(err) w.WriteHeader(500) diff --git a/pkg/handlers/identity.go b/pkg/handlers/identity.go index 598fa71248..f0e9b4459b 100644 --- a/pkg/handlers/identity.go +++ b/pkg/handlers/identity.go @@ -279,7 +279,7 @@ func (h *Handler) ConfigureAppIdentityService(w http.ResponseWriter, r *http.Req return } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { err = errors.Wrap(err, "failed to load kots kinds from path") logger.Error(err) @@ -312,7 +312,7 @@ func (h *Handler) ConfigureAppIdentityService(w http.ResponseWriter, r *http.Req return } - b, err := ioutil.ReadFile(identityConfigFile) + b, err := os.ReadFile(identityConfigFile) if err != nil { err = errors.Wrap(err, "failed to read identityconfig file") logger.Error(err) @@ -424,7 +424,7 @@ func (h *Handler) ConfigureAppIdentityService(w http.ResponseWriter, r *http.Req return } - if err := ioutil.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "identityconfig.yaml"), []byte(identityConfigSpec), 0644); err != nil { + if err := os.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "identityconfig.yaml"), []byte(identityConfigSpec), 0644); err != nil { err = errors.Wrap(err, "failed to write identityconfig.yaml to upstream/userdata") logger.Error(err) w.WriteHeader(http.StatusInternalServerError) @@ -680,7 +680,7 @@ func (h *Handler) GetAppIdentityServiceConfig(w http.ResponseWriter, r *http.Req return } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { err = errors.Wrap(err, "failed to load kotskinds from path") logger.Error(err) diff --git a/pkg/handlers/metadata.go b/pkg/handlers/metadata.go index cc7253551e..013e5beb1e 100644 --- a/pkg/handlers/metadata.go +++ b/pkg/handlers/metadata.go @@ -168,7 +168,7 @@ func getBrandingResponse(kotsStore store.Store, appID string) MetadataResponseBr return response } - applicationYaml, err := ioutil.ReadFile(filepath.Join(tmpDir, "application.yaml")) + applicationYaml, err := os.ReadFile(filepath.Join(tmpDir, "application.yaml")) if err != nil { logger.Error(errors.Wrap(err, "failed to read application.yaml")) return response @@ -188,7 +188,7 @@ func getBrandingResponse(kotsStore store.Store, appID string) MetadataResponseBr continue } - contents, err := ioutil.ReadFile(filepath.Join(tmpDir, source)) + contents, err := os.ReadFile(filepath.Join(tmpDir, source)) if err != nil { logger.Error(errors.Wrapf(err, "failed to read font file %s", source)) continue @@ -212,7 +212,7 @@ func getBrandingResponse(kotsStore store.Store, appID string) MetadataResponseBr continue } - contents, err := ioutil.ReadFile(filepath.Join(tmpDir, source)) + contents, err := os.ReadFile(filepath.Join(tmpDir, source)) if err != nil { logger.Error(errors.Wrapf(err, "failed to read font file %s", source)) continue diff --git a/pkg/handlers/preflight.go b/pkg/handlers/preflight.go index 550545d4a3..ccbe724113 100644 --- a/pkg/handlers/preflight.go +++ b/pkg/handlers/preflight.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "net/http" "os" - "path/filepath" "strconv" "github.com/gorilla/mux" @@ -274,7 +273,7 @@ func (h *Handler) GetPreflightCommand(w http.ResponseWriter, r *http.Request) { return } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archivePath, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archivePath) if err != nil { logger.Error(errors.Wrap(err, "failed to load kots kinds")) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/handlers/registry.go b/pkg/handlers/registry.go index 75ce23dd61..ebabf8a05b 100644 --- a/pkg/handlers/registry.go +++ b/pkg/handlers/registry.go @@ -314,7 +314,7 @@ func registrySettingsChanged(app *apptypes.App, new UpdateAppRegistryRequest, cu return false, errors.Wrap(err, "failed to get version archive") } - secretData, err := ioutil.ReadFile(filepath.Join(archiveDir, "overlays", "midstream", "secret.yaml")) + secretData, err := os.ReadFile(filepath.Join(archiveDir, "overlays", "midstream", "secret.yaml")) if err != nil { if os.IsNotExist(err) { if new.Hostname != "" { diff --git a/pkg/handlers/rendered_contents.go b/pkg/handlers/rendered_contents.go index 1b99ad3460..f595e95958 100644 --- a/pkg/handlers/rendered_contents.go +++ b/pkg/handlers/rendered_contents.go @@ -4,7 +4,6 @@ import ( "io/ioutil" "net/http" "os" - "path/filepath" "strconv" "github.com/gorilla/mux" @@ -67,7 +66,7 @@ func (h *Handler) GetAppRenderedContents(w http.ResponseWriter, r *http.Request) return } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archivePath, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archivePath) if err != nil { logger.Error(errors.Wrap(err, "failed to load kots kinds from path")) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/handlers/upload.go b/pkg/handlers/upload.go index c481683f36..f87b18bec7 100644 --- a/pkg/handlers/upload.go +++ b/pkg/handlers/upload.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "os" "path/filepath" @@ -63,7 +62,7 @@ func (h *Handler) UploadExistingApp(w http.ResponseWriter, r *http.Request) { return } - tmpFile, err := ioutil.TempFile("", "kotsadm") + tmpFile, err := os.CreateTemp("", "kotsadm") if err != nil { uploadResponse.Error = util.StrPointer("failed to create temp file") logger.Error(errors.Wrap(err, *uploadResponse.Error)) @@ -89,7 +88,7 @@ func (h *Handler) UploadExistingApp(w http.ResponseWriter, r *http.Request) { defer os.RemoveAll(archiveDir) // encrypt any plain text values - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { uploadResponse.Error = util.StrPointer("failed to load kotskinds") logger.Error(errors.Wrap(err, *uploadResponse.Error)) @@ -112,7 +111,7 @@ func (h *Handler) UploadExistingApp(w http.ResponseWriter, r *http.Request) { return } - if err := ioutil.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "config.yaml"), []byte(updated), 0644); err != nil { + if err := os.WriteFile(filepath.Join(archiveDir, "upstream", "userdata", "config.yaml"), []byte(updated), 0644); err != nil { uploadResponse.Error = util.StrPointer("failed to write config values") logger.Error(errors.Wrap(err, *uploadResponse.Error)) JSON(w, http.StatusInternalServerError, uploadResponse) diff --git a/pkg/helm/updates.go b/pkg/helm/updates.go index f167b50299..6fdb57b682 100644 --- a/pkg/helm/updates.go +++ b/pkg/helm/updates.go @@ -335,7 +335,7 @@ var ( func getUpdateChartFromCache(helmApp *apptypes.HelmApp, version string) (*bytes.Buffer, error) { fileName := getUpdateChacheFileName(helmApp, version) - b, err := ioutil.ReadFile(fileName) + b, err := os.ReadFile(fileName) if err != nil { if os.IsNotExist(err) { return nil, nil diff --git a/pkg/identity/client/http.go b/pkg/identity/client/http.go index 20d60db9b1..72dc33c349 100644 --- a/pkg/identity/client/http.go +++ b/pkg/identity/client/http.go @@ -6,7 +6,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" - "io/ioutil" "log" "net/http" "os" @@ -95,5 +94,5 @@ func getKurlProxyTLSCert() ([]byte, error) { if certPath == "" { return nil, nil } - return ioutil.ReadFile(certPath) + return os.ReadFile(certPath) } diff --git a/pkg/identity/deploy/deploy.go b/pkg/identity/deploy/deploy.go index 90d09a795d..3f007351ad 100644 --- a/pkg/identity/deploy/deploy.go +++ b/pkg/identity/deploy/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/identity/types" "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/ingress" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" @@ -372,7 +373,7 @@ var ( func deploymentResource(issuerURL, configChecksum string, options Options) (*appsv1.Deployment, error) { // TODO: use GetAdminConsoleImage function - dexVersion, err := image.GetTag(image.Dex) + dexVersion, err := imageutil.GetTag(image.Dex) if err != nil { return nil, err } diff --git a/pkg/image/builder.go b/pkg/image/builder.go index ebb2846773..43fb105061 100644 --- a/pkg/image/builder.go +++ b/pkg/image/builder.go @@ -26,6 +26,7 @@ import ( dockerregistrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types" dockertypes "github.com/replicatedhq/kots/pkg/docker/types" "github.com/replicatedhq/kots/pkg/image/types" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/k8sdoc" "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/logger" @@ -40,11 +41,11 @@ var imagePolicy = []byte(`{ "default": [{"type": "insecureAcceptAnything"}] }`) -func RewriteImages(srcRegistry, destRegistry dockerregistrytypes.RegistryOptions, appSlug string, log *logger.CLILogger, reportWriter io.Writer, upstreamDir string, additionalImages []string, copyImages, allImagesPrivate bool, checkedImages map[string]types.ImageInfo, dockerHubRegistry dockerregistrytypes.RegistryOptions) ([]kustomizeimage.Image, error) { +func RewriteImages(srcRegistry, destRegistry dockerregistrytypes.RegistryOptions, appSlug string, log *logger.CLILogger, reportWriter io.Writer, baseDir string, additionalImages []string, copyImages, allImagesPrivate bool, checkedImages map[string]types.ImageInfo, dockerHubRegistry dockerregistrytypes.RegistryOptions) ([]kustomizeimage.Image, error) { newImages := []kustomizeimage.Image{} savedImages := map[string]bool{} - err := filepath.Walk(upstreamDir, + err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -54,7 +55,7 @@ func RewriteImages(srcRegistry, destRegistry dockerregistrytypes.RegistryOptions return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } @@ -145,7 +146,7 @@ func GetPrivateImages(baseDir string, kotsKindsImages []string, checkedImages ma return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { logger.Debugf("Failed to read file %s: %v", path, err) return err @@ -300,7 +301,7 @@ func rewriteOneImage(srcRegistry, destRegistry dockerregistrytypes.RegistryOptio return nil, errors.Wrapf(err, "failed to parse source image name %s", sourceImage) } - destImage, err := DestImage(destRegistry, image) + destImage, err := imageutil.DestImage(destRegistry, image) if err != nil { return nil, errors.Wrap(err, "failed to get destination image") } @@ -335,7 +336,7 @@ func rewriteOneImage(srcRegistry, destRegistry dockerregistrytypes.RegistryOptio } if !copyImages { - return kustomizeImage(destRegistry, image) + return imageutil.KustomizeImage(destRegistry, image) } imageListSelection := copy.CopySystemImage @@ -400,7 +401,7 @@ func rewriteOneImage(srcRegistry, destRegistry dockerregistrytypes.RegistryOptio } } - return kustomizeImage(destRegistry, image) + return imageutil.KustomizeImage(destRegistry, image) } func CopyImage(opts types.CopyImageOptions) error { @@ -657,7 +658,12 @@ func RewriteImagesBetweenRegistries(options RewriteImagesBetweenRegistriesOption checkedImages := make(map[string]types.ImageInfo) if options.KotsKinds != nil { - additionalImages = kotsutil.GetImagesFromKotsKinds(options.KotsKinds) + kki, err := kotsutil.GetImagesFromKotsKinds(options.KotsKinds, &options.DestRegistry) + if err != nil { + return nil, errors.Wrap(err, "failed to get images from kots kinds") + } + additionalImages = kki + checkedImages = makeImageInfoMap(options.KotsKinds.Installation.Spec.KnownImages) if options.KotsKinds.KotsApplication.Spec.ProxyPublicImages { allImagesPrivate = true diff --git a/pkg/image/image.go b/pkg/imageutil/image.go similarity index 97% rename from pkg/image/image.go rename to pkg/imageutil/image.go index 52316114e6..2316b47555 100644 --- a/pkg/image/image.go +++ b/pkg/imageutil/image.go @@ -1,4 +1,4 @@ -package image +package imageutil import ( "fmt" @@ -71,8 +71,8 @@ func RewriteDockerRegistryImage(destRegistry registrytypes.RegistryOptions, srcI } rewrittenImage := kustomizetypes.Image{} - rewrittenImage.Name = stripImageTagAndDigest(srcImage) - rewrittenImage.NewName = stripImageTagAndDigest(destImage) + rewrittenImage.Name = StripImageTagAndDigest(srcImage) + rewrittenImage.NewName = StripImageTagAndDigest(destImage) if can, ok := parsedSrc.(reference.Canonical); ok { rewrittenImage.Digest = can.Digest().String() @@ -269,9 +269,9 @@ func BuildImageAltNames(rewrittenImage kustomizetypes.Image) ([]kustomizetypes.I return images, nil } -// stripImageTagAndDigest removes the tag and digest from an image while preserving the original name. +// StripImageTagAndDigest removes the tag and digest from an image while preserving the original name. // This can be helpful because parsing the image as a docker reference can modify the hostname (e.g. adds docker.io/library) -func stripImageTagAndDigest(image string) string { +func StripImageTagAndDigest(image string) string { // grab last section of image name imageParts := strings.Split(image, "/") lastPart := imageParts[len(imageParts)-1] @@ -286,7 +286,7 @@ func stripImageTagAndDigest(image string) string { return image } -func kustomizeImage(destRegistry registrytypes.RegistryOptions, image string) ([]kustomizetypes.Image, error) { +func KustomizeImage(destRegistry registrytypes.RegistryOptions, image string) ([]kustomizetypes.Image, error) { rewrittenImage, err := RewriteDockerRegistryImage(destRegistry, image) if err != nil { return nil, errors.Wrap(err, "failed to rewrite image") diff --git a/pkg/image/image_test.go b/pkg/imageutil/image_test.go similarity index 98% rename from pkg/image/image_test.go rename to pkg/imageutil/image_test.go index bbd8d7c745..510ae0c378 100644 --- a/pkg/image/image_test.go +++ b/pkg/imageutil/image_test.go @@ -1,4 +1,4 @@ -package image +package imageutil import ( "fmt" @@ -718,7 +718,7 @@ func Test_BuildImageAltNames(t *testing.T) { } } -func Test_kustomizeImage(t *testing.T) { +func Test_KustomizeImage(t *testing.T) { tests := []struct { name string destRegistry registrytypes.RegistryOptions @@ -1000,14 +1000,14 @@ func Test_kustomizeImage(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := require.New(t) - got, err := kustomizeImage(tt.destRegistry, tt.image) + got, err := KustomizeImage(tt.destRegistry, tt.image) req.NoError(err) req.Equal(tt.want, got) }) } } -func Test_stripImageTagAndDigest(t *testing.T) { +func Test_StripImageTagAndDigest(t *testing.T) { tests := []struct { name string image string @@ -1056,8 +1056,8 @@ func Test_stripImageTagAndDigest(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := stripImageTagAndDigest(tt.image); got != tt.want { - t.Errorf("stripImageTagAndDigest() = %v, want %v", got, tt.want) + if got := StripImageTagAndDigest(tt.image); got != tt.want { + t.Errorf("StripImageTagAndDigest() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/k8sutil/kustomization.go b/pkg/k8sutil/kustomization.go index ef6ef1eb0c..4253bfe151 100644 --- a/pkg/k8sutil/kustomization.go +++ b/pkg/k8sutil/kustomization.go @@ -2,6 +2,7 @@ package k8sutil import ( "io/ioutil" + "os" "sort" "strings" @@ -11,7 +12,7 @@ import ( ) func ReadKustomizationFromFile(file string) (*kustomizetypes.Kustomization, error) { - b, err := ioutil.ReadFile(file) + b, err := os.ReadFile(file) if err != nil { return nil, errors.Wrap(err, "failed to read kustomization file") } diff --git a/pkg/kotsadm/configmaps.go b/pkg/kotsadm/configmaps.go index 016f21df60..309f69d2b7 100644 --- a/pkg/kotsadm/configmaps.go +++ b/pkg/kotsadm/configmaps.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "encoding/base64" - "io/ioutil" + "os" "path/filepath" "github.com/pkg/errors" @@ -144,7 +144,7 @@ func ensureConfigFromFile(deployOptions types.DeployOptions, clientset *kubernet } func configMapFromFile(deployOptions types.DeployOptions, configMapName string, filename string) (*corev1.ConfigMap, error) { - fileData, err := ioutil.ReadFile(filename) + fileData, err := os.ReadFile(filename) if err != nil { return nil, errors.Wrap(err, "failed to load file") } diff --git a/pkg/kotsadm/objects/images.go b/pkg/kotsadm/objects/images.go index bfc799b3ce..5ffd8b1af9 100644 --- a/pkg/kotsadm/objects/images.go +++ b/pkg/kotsadm/objects/images.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" ) @@ -14,9 +15,9 @@ func GetAdminConsoleImage(deployOptions types.DeployOptions, imageKey string) st func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string { // TODO: Add error handling to this function - minioTag, _ := image.GetTag(image.Minio) - rqliteTag, _ := image.GetTag(image.Rqlite) - dexTag, _ := image.GetTag(image.Dex) + minioTag, _ := imageutil.GetTag(image.Minio) + rqliteTag, _ := imageutil.GetTag(image.Rqlite) + dexTag, _ := imageutil.GetTag(image.Dex) minioImage := image.Minio rqliteImage := image.Rqlite diff --git a/pkg/kotsadm/push_images.go b/pkg/kotsadm/push_images.go index bedb06a886..dd2ecfd7e1 100644 --- a/pkg/kotsadm/push_images.go +++ b/pkg/kotsadm/push_images.go @@ -25,6 +25,7 @@ import ( dockertypes "github.com/replicatedhq/kots/pkg/docker/types" "github.com/replicatedhq/kots/pkg/image" imagetypes "github.com/replicatedhq/kots/pkg/image/types" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/replicatedhq/kots/pkg/kotsutil" "k8s.io/client-go/kubernetes/scheme" @@ -327,7 +328,7 @@ func PushAppImagesFromTempRegistry(airgapRootDir string, imageList []string, opt return nil, errors.Wrapf(err, "failed to parse source image %s", imageID) } - destImage, err := image.DestImage(options.Registry, imageID) + destImage, err := imageutil.DestImage(options.Registry, imageID) if err != nil { return nil, errors.Wrapf(err, "failed to get destination image for %s", imageID) } @@ -337,7 +338,7 @@ func PushAppImagesFromTempRegistry(airgapRootDir string, imageList []string, opt return nil, errors.Wrapf(err, "failed to parse dest image %s", destStr) } - rewrittenImage, err := image.RewriteDockerRegistryImage(options.Registry, imageID) + rewrittenImage, err := imageutil.RewriteDockerRegistryImage(options.Registry, imageID) if err != nil { return nil, errors.Wrapf(err, "failed to rewrite image %s", imageID) } @@ -419,7 +420,7 @@ func PushAppImagesFromDockerArchivePath(airgapRootDir string, options types.Push for imagePath, imageInfo := range imageInfos { formatRoot := path.Join(imagesDir, imageInfo.Format) pathWithoutRoot := imagePath[len(formatRoot)+1:] - rewrittenImage, err := image.RewriteDockerArchiveImage(options.Registry, strings.Split(pathWithoutRoot, string(os.PathSeparator))) + rewrittenImage, err := imageutil.RewriteDockerArchiveImage(options.Registry, strings.Split(pathWithoutRoot, string(os.PathSeparator))) if err != nil { return nil, errors.Wrap(err, "failed to rewrite docker archive image") } @@ -430,7 +431,7 @@ func PushAppImagesFromDockerArchivePath(airgapRootDir string, options types.Push return nil, errors.Wrap(err, "failed to parse src image name") } - destStr := fmt.Sprintf("docker://%s", image.DestImageFromKustomizeImage(rewrittenImage)) + destStr := fmt.Sprintf("docker://%s", imageutil.DestImageFromKustomizeImage(rewrittenImage)) destRef, err := alltransports.ParseImageName(destStr) if err != nil { return nil, errors.Wrapf(err, "failed to parse dest image name %s", destStr) @@ -540,7 +541,7 @@ func PushAppImagesFromDockerArchiveBundle(airgapBundle string, options types.Pus return nil, errors.Errorf("not enough path parts in %q", imagePath) } - rewrittenImage, err := image.RewriteDockerArchiveImage(options.Registry, pathParts[2:]) + rewrittenImage, err := imageutil.RewriteDockerArchiveImage(options.Registry, pathParts[2:]) if err != nil { return nil, errors.Wrap(err, "failed to rewrite docker archive image") } @@ -551,7 +552,7 @@ func PushAppImagesFromDockerArchiveBundle(airgapBundle string, options types.Pus return nil, errors.Wrap(err, "failed to parse src image name") } - destStr := fmt.Sprintf("docker://%s", image.DestImageFromKustomizeImage(rewrittenImage)) + destStr := fmt.Sprintf("docker://%s", imageutil.DestImageFromKustomizeImage(rewrittenImage)) destRef, err := alltransports.ParseImageName(destStr) if err != nil { return nil, errors.Wrapf(err, "failed to parse dest image name %s", destStr) @@ -637,7 +638,7 @@ func GetImagesFromBundle(airgapBundle string, options types.PushImagesOptions) ( switch airgap.Spec.Format { case dockertypes.FormatDockerRegistry: for _, savedImage := range airgap.Spec.SavedImages { - rewrittenImage, err := image.RewriteDockerRegistryImage(options.Registry, savedImage) + rewrittenImage, err := imageutil.RewriteDockerRegistryImage(options.Registry, savedImage) if err != nil { return nil, errors.Wrapf(err, "failed to rewrite image %s", savedImage) } @@ -653,7 +654,7 @@ func GetImagesFromBundle(airgapBundle string, options types.PushImagesOptions) ( if len(pathParts) < 3 { return nil, errors.Errorf("not enough path parts in %q", imagePath) } - rewrittenImage, err := image.RewriteDockerArchiveImage(options.Registry, pathParts[2:]) + rewrittenImage, err := imageutil.RewriteDockerArchiveImage(options.Registry, pathParts[2:]) if err != nil { return nil, errors.Wrap(err, "failed to rewrite docker archive image") } @@ -953,7 +954,7 @@ func isAppArchive(rootDir string) bool { continue } - contents, err := ioutil.ReadFile(filepath.Join(rootDir, info.Name())) + contents, err := os.ReadFile(filepath.Join(rootDir, info.Name())) if err != nil { continue } diff --git a/pkg/kotsadmlicense/license.go b/pkg/kotsadmlicense/license.go index 79be707e32..f8312706f3 100644 --- a/pkg/kotsadmlicense/license.go +++ b/pkg/kotsadmlicense/license.go @@ -3,7 +3,6 @@ package license import ( "io/ioutil" "os" - "path/filepath" "github.com/pkg/errors" apptypes "github.com/replicatedhq/kots/pkg/app/types" @@ -72,7 +71,7 @@ func Sync(a *apptypes.App, licenseString string, failOnVersionCreate bool) (*kot return nil, false, errors.Wrap(err, "failed to get latest app sequence") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return nil, false, errors.Wrap(err, "failed to load kotskinds from path") } diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 779661c7c9..7e1a435a37 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "math" "os" - "path/filepath" "strconv" "time" @@ -58,7 +57,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b zap.String("appID", a.ID), zap.Int64("sequence", parentSequence)) - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { return nil, errors.Wrap(err, "failed to create temp dir") } @@ -94,7 +93,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return nil, errors.New("no backup store location found") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return nil, errors.Wrap(err, "failed to load kots kinds from path") } @@ -268,7 +267,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return nil, errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return nil, errors.Wrap(err, "failed to load kots kinds from path") } diff --git a/pkg/kotsadmupstream/upstream.go b/pkg/kotsadmupstream/upstream.go index 1e2e77e5fd..3db3095c3c 100644 --- a/pkg/kotsadmupstream/upstream.go +++ b/pkg/kotsadmupstream/upstream.go @@ -128,7 +128,7 @@ func DownloadUpdate(appID string, update types.Update, skipPreflights bool, skip } defer os.RemoveAll(archiveDir) - beforeKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + beforeKotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { finalError = errors.Wrap(err, "failed to read kots kinds before update") return @@ -230,6 +230,7 @@ func DownloadUpdate(appID string, update types.Update, skipPreflights bool, skip RewriteImages: registrySettings.IsValid(), RewriteImageOptions: registrySettings, SkipCompatibilityCheck: skipCompatibilityCheck, + KotsKinds: beforeKotsKinds, } _, err = pull.Pull(fmt.Sprintf("replicated://%s", beforeKotsKinds.License.Spec.AppSlug), pullOptions) @@ -241,7 +242,7 @@ func DownloadUpdate(appID string, update types.Update, skipPreflights bool, skip } if update.AppSequence == nil { - afterKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + afterKotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { finalError = errors.Wrap(err, "failed to read kots kinds after update") return diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 3a25a7ff7f..f4a5afbaa2 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -16,12 +15,14 @@ import ( "time" "github.com/blang/semver" + dockerref "github.com/containers/image/v5/docker/reference" "github.com/pkg/errors" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" "github.com/replicatedhq/kots/pkg/archives" "github.com/replicatedhq/kots/pkg/binaries" "github.com/replicatedhq/kots/pkg/buildversion" "github.com/replicatedhq/kots/pkg/crypto" + registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types" "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/kurl" "github.com/replicatedhq/kots/pkg/logger" @@ -478,7 +479,7 @@ func (k *KotsKinds) addKotsKinds(content []byte) error { for _, doc := range docs { decoded, gvk, err := decode(doc, nil, nil) if err != nil { - return errors.Wrap(err, "failed to decode yaml content") + return errors.Wrapf(err, "failed to decode: %v", string(content)) } if strings.HasPrefix(gvk.String(), "troubleshoot.replicated.com/v1beta1,") { @@ -488,7 +489,7 @@ func (k *KotsKinds) addKotsKinds(content []byte) error { } decoded, gvk, err = decode(doc, nil, nil) if err != nil { - return err + return errors.Wrapf(err, "failed to decode troubleshoot doc: %v", string(content)) } } @@ -555,25 +556,9 @@ func (k *KotsKinds) addKotsKinds(content []byte) error { return nil } -func GenUniqueKotsKindFilename(renderedKotsKinds map[string][]byte, prefix string) string { - filename := fmt.Sprintf("%s.yaml", prefix) - if _, exists := renderedKotsKinds[filename]; exists { - index := 1 - for { - filename = fmt.Sprintf("%s-%d.yaml", prefix, index) - if _, exists := renderedKotsKinds[filename]; !exists { - break - } - index += 1 - } - } - - return filename -} - -func GetImagesFromKotsKinds(kotsKinds *KotsKinds) []string { +func GetImagesFromKotsKinds(kotsKinds *KotsKinds, destRegistry *registrytypes.RegistryOptions) ([]string, error) { if kotsKinds == nil { - return nil + return nil, nil } allImages := []string{} @@ -614,16 +599,26 @@ func GetImagesFromKotsKinds(kotsKinds *KotsKinds) []string { if image == "" { continue } + // Images that use templates like LocalImageName should be included in application's additionalImages list. + // We want the original image names here only, not the templated ones. if strings.Contains(image, "repl{{") || strings.Contains(image, "{{repl") { - // Images that use templates like LocalImageName should be included in application's additionalImages list. - // We want the original image names here only, not the templated ones. continue } + if destRegistry != nil { + dockerRef, err := dockerref.ParseDockerRef(image) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse docker ref %q", image) + } + if strings.HasPrefix(destRegistry.Endpoint, dockerref.Domain(dockerRef)) { + // image points to the destination registry + continue + } + } allImages = append(allImages, image) } } - return allImages + return allImages, nil } // create a new kots kinds, ensuring that the require objets exist as empty defaults @@ -646,9 +641,31 @@ func EmptyKotsKinds() KotsKinds { return kotsKinds } -func LoadKotsKindsFromPath(fromDir string) (*KotsKinds, error) { +// GetKotsKindsPath returns the path to load kots kinds from in an app version archive created by kots +func GetKotsKindsPath(archive string) string { + if archive == "" { + return "" + } + + kotsKindsPath := archive + if _, err := os.Stat(filepath.Join(archive, "kotsKinds")); err == nil { + // contains the rendered kots kinds if exists, prioritize it over upstream. only newer versions of kots create this directory. + kotsKindsPath = filepath.Join(archive, "kotsKinds") + } else if _, err := os.Stat(filepath.Join(archive, "upstream")); err == nil { + // contains the non-rendered kots kinds, fallback to it if kotsKinds directory doesn't exist. this directory should always exist. + kotsKindsPath = filepath.Join(archive, "upstream") + } + + return kotsKindsPath +} + +// LoadKotsKinds loads kots kinds from an app version archive created by kots. +// it loads the rendered kots kinds if they exist (should always be the case for app version archives created by newer kots versions). +// otherwise it loads the non-rendered kots kinds (app version archives created by older kots versions). +func LoadKotsKinds(archive string) (*KotsKinds, error) { kotsKinds := EmptyKotsKinds() + fromDir := GetKotsKindsPath(archive) if fromDir == "" { return &kotsKinds, nil } @@ -750,7 +767,7 @@ func loadRuntimeObjectsFromPath(apiVersion, kind, fromDir string) ([]runtime.Obj return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return errors.Wrap(err, "failed to read file") } @@ -761,7 +778,7 @@ func loadRuntimeObjectsFromPath(apiVersion, kind, fromDir string) ([]runtime.Obj decoded, gvk, err := decode(contents, nil, nil) if err != nil { - return errors.Wrap(err, "failed to decode") + return errors.Wrapf(err, "failed to decode: %v", string(contents)) } if gvk.String() == fmt.Sprintf("%s, Kind=%s", apiVersion, kind) { @@ -790,36 +807,8 @@ func IsApiVersionKind(content []byte, apiVersion, kind string) bool { return false } -func LoadV1Beta1HelmChartsFromPath(fromDir string) ([]*kotsv1beta1.HelmChart, error) { - objects, err := loadRuntimeObjectsFromPath("kots.io/v1beta1", "HelmChart", fromDir) - if err != nil { - return nil, errors.Wrapf(err, "failed to load v1beta1 HelmCharts from %s", fromDir) - } - - charts := []*kotsv1beta1.HelmChart{} - for _, o := range objects { - charts = append(charts, o.(*kotsv1beta1.HelmChart)) - } - - return charts, nil -} - -func LoadV1Beta2HelmChartsFromPath(fromDir string) ([]*kotsv1beta2.HelmChart, error) { - objects, err := loadRuntimeObjectsFromPath("kots.io/v1beta2", "HelmChart", fromDir) - if err != nil { - return nil, errors.Wrapf(err, "failed to load v1beta2 HelmCharts from %s", fromDir) - } - - charts := []*kotsv1beta2.HelmChart{} - for _, o := range objects { - charts = append(charts, o.(*kotsv1beta2.HelmChart)) - } - - return charts, nil -} - func LoadInstallationFromPath(installationFilePath string) (*kotsv1beta1.Installation, error) { - installationData, err := ioutil.ReadFile(installationFilePath) + installationData, err := os.ReadFile(installationFilePath) if err != nil { return nil, errors.Wrap(err, "failed to read installation file") } @@ -831,7 +820,7 @@ func LoadSupportBundleFromContents(data []byte) (*troubleshootv1beta2.SupportBun decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode([]byte(data), nil, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to decode support bundle data of length %d", len(data)) + return nil, errors.Wrapf(err, "failed to decode: %v", string(data)) } if gvk.Group != "troubleshoot.sh" || gvk.Version != "v1beta2" || gvk.Kind != "SupportBundle" { @@ -841,11 +830,25 @@ func LoadSupportBundleFromContents(data []byte) (*troubleshootv1beta2.SupportBun return obj.(*troubleshootv1beta2.SupportBundle), nil } +func FindKotsAppInPath(fromDir string) (*kotsv1beta1.Application, error) { + objects, err := loadRuntimeObjectsFromPath("kots.io/v1beta1", "Application", fromDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to load v1beta2 HelmCharts from %s", fromDir) + } + + if len(objects) == 0 { + return nil, nil + } + + // we only support having one kots app spec + return objects[0].(*kotsv1beta1.Application), nil +} + func LoadKotsAppFromContents(data []byte) (*kotsv1beta1.Application, error) { decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode([]byte(data), nil, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to decode kots app data of length %d", len(data)) + return nil, errors.Wrapf(err, "failed to decode: %v", string(data)) } if gvk.Group != "kots.io" || gvk.Version != "v1beta1" || gvk.Kind != "Application" { @@ -859,7 +862,7 @@ func LoadV1Beta1HelmChartListFromContents(data []byte) (*kotsv1beta1.HelmChartLi decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(data, nil, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to decode helm chart data of length %d", len(data)) + return nil, errors.Wrapf(err, "failed to decode: %v", string(data)) } if gvk.Group != "kots.io" || gvk.Version != "v1beta1" || gvk.Kind != "HelmChartList" { @@ -873,7 +876,7 @@ func LoadV1Beta1HelmChartFromContents(content []byte) (*kotsv1beta1.HelmChart, e decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode chart") + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) } if gvk.Group != "kots.io" || gvk.Version != "v1beta1" || gvk.Kind != "HelmChart" { @@ -887,7 +890,7 @@ func LoadV1Beta2HelmChartFromContents(content []byte) (*kotsv1beta2.HelmChart, e decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode chart") + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) } if gvk.Group != "kots.io" || gvk.Version != "v1beta2" || gvk.Kind != "HelmChart" { @@ -901,7 +904,7 @@ func LoadInstallationFromContents(installationData []byte) (*kotsv1beta1.Install decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode([]byte(installationData), nil, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to decode installation data of length %d", len(installationData)) + return nil, errors.Wrapf(err, "failed to decode: %v", string(installationData)) } if gvk.Group != "kots.io" || gvk.Version != "v1beta1" || gvk.Kind != "Installation" { @@ -912,7 +915,7 @@ func LoadInstallationFromContents(installationData []byte) (*kotsv1beta1.Install } func LoadLicenseFromPath(licenseFilePath string) (*kotsv1beta1.License, error) { - licenseData, err := ioutil.ReadFile(licenseFilePath) + licenseData, err := os.ReadFile(licenseFilePath) if err != nil { return nil, errors.Wrap(err, "failed to read license file") } @@ -938,7 +941,7 @@ func LoadEmbeddedClusterConfigFromBytes(data []byte) (*embeddedclusterv1beta1.Co decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode([]byte(data), nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode embedded cluster config data") + return nil, errors.Wrapf(err, "failed to decode: %v", string(data)) } if gvk.Group != "embeddedcluster.replicated.com" || gvk.Version != "v1beta1" || gvk.Kind != "Config" { return nil, errors.Errorf("unexpected GVK: %s", gvk.String()) @@ -947,7 +950,7 @@ func LoadEmbeddedClusterConfigFromBytes(data []byte) (*embeddedclusterv1beta1.Co } func LoadConfigValuesFromFile(configValuesFilePath string) (*kotsv1beta1.ConfigValues, error) { - configValuesData, err := ioutil.ReadFile(configValuesFilePath) + configValuesData, err := os.ReadFile(configValuesFilePath) if err != nil { return nil, errors.Wrap(err, "failed to read configvalues file") } @@ -965,6 +968,20 @@ func LoadConfigValuesFromFile(configValuesFilePath string) (*kotsv1beta1.ConfigV return obj.(*kotsv1beta1.ConfigValues), nil } +func FindConfigInPath(fromDir string) (*kotsv1beta1.Config, error) { + objects, err := loadRuntimeObjectsFromPath("kots.io/v1beta1", "Config", fromDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to load Config from %s", fromDir) + } + + if len(objects) == 0 { + return nil, nil + } + + // we only support having one config spec + return objects[0].(*kotsv1beta1.Config), nil +} + func LoadConfigFromBytes(data []byte) (*kotsv1beta1.Config, error) { decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(data, nil, nil) @@ -1023,7 +1040,7 @@ func LoadBackupFromContents(content []byte) (*velerov1.Backup, error) { obj, gvk, err := decode(content, nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode content") + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) } if gvk.String() != "velero.io/v1, Kind=Backup" { @@ -1038,7 +1055,7 @@ func LoadApplicationFromContents(content []byte) (*applicationv1beta1.Applicatio obj, gvk, err := decode(content, nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode content") + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) } if gvk.String() != "app.k8s.io/v1beta1, Kind=Application" { @@ -1053,7 +1070,7 @@ func LoadApplicationFromBytes(content []byte) (*kotsv1beta1.Application, error) obj, gvk, err := decode(content, nil, nil) if err != nil { - return nil, errors.Wrap(err, "failed to decode content") + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) } if gvk.String() != "kots.io/v1beta1, Kind=Application" { @@ -1307,7 +1324,7 @@ func RemoveAppVersionLabelFromInstallationParams(configMapName string) error { } func FindAirgapMetaInDir(root string) (*kotsv1beta1.Airgap, error) { - files, err := ioutil.ReadDir(root) + files, err := os.ReadDir(root) if err != nil { return nil, errors.Wrap(err, "failed to read airgap directory content") } @@ -1317,7 +1334,7 @@ func FindAirgapMetaInDir(root string) (*kotsv1beta1.Airgap, error) { continue } - contents, err := ioutil.ReadFile(filepath.Join(root, file.Name())) + contents, err := os.ReadFile(filepath.Join(root, file.Name())) if err != nil { // TODO: log? continue @@ -1400,7 +1417,7 @@ func LoadBrandingArchiveFromPath(archivePath string) (*bytes.Buffer, error) { return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return errors.Wrap(err, "failed to read file") } diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index 5747cc643e..53b8de27fb 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -17,6 +17,273 @@ import ( ) var _ = Describe("Kots", func() { + Describe("GetKotsKindsPath()", func() { + It("returns the path to the kotsKinds directory if it exists", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + kotsKindsDir := filepath.Join(dir, "kotsKinds") + err = os.MkdirAll(kotsKindsDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + path := kotsutil.GetKotsKindsPath(dir) + Expect(path).To(Equal(kotsKindsDir)) + }) + + It("returns the path to the upstream directory if kotsKinds does not exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + upstreamDir := filepath.Join(dir, "upstream") + err = os.MkdirAll(upstreamDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + path := kotsutil.GetKotsKindsPath(dir) + Expect(path).To(Equal(upstreamDir)) + }) + + It("returns the path to the kotsKinds directory if both kotsKinds and upstream exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + kotsKindsDir := filepath.Join(dir, "kotsKinds") + err = os.MkdirAll(kotsKindsDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + upstreamDir := filepath.Join(dir, "upstream") + err = os.MkdirAll(upstreamDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + path := kotsutil.GetKotsKindsPath(dir) + Expect(path).To(Equal(kotsKindsDir)) + }) + + It("returns the path to root directory if neither kotsKinds nor upstream exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + path := kotsutil.GetKotsKindsPath(dir) + Expect(path).To(Equal(dir)) + }) + }) + + Describe("LoadKotsKinds()", func() { + It("loads kots kinds from 'kotsKinds' directory if exists", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + kotsKindsDir := filepath.Join(dir, "kotsKinds") + err = os.MkdirAll(kotsKindsDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(kotsKindsDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsKinds, err := kotsutil.LoadKotsKinds(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsKinds).ToNot(BeNil()) + Expect(kotsKinds.KotsApplication.Spec.Title).To(Equal("foo")) + }) + + It("loads kots kinds from 'upstream' directory if 'kotsKinds' does not exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + upstreamDir := filepath.Join(dir, "upstream") + err = os.MkdirAll(upstreamDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(upstreamDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsKinds, err := kotsutil.LoadKotsKinds(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsKinds).ToNot(BeNil()) + Expect(kotsKinds.KotsApplication.Spec.Title).To(Equal("foo")) + }) + + It("loads kots kinds from 'kotsKinds' directory if both 'kotsKinds' and 'upstream' exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + kotsKindsDir := filepath.Join(dir, "kotsKinds") + err = os.MkdirAll(kotsKindsDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + upstreamDir := filepath.Join(dir, "upstream") + err = os.MkdirAll(upstreamDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(kotsKindsDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(upstreamDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: bar\nspec:\n title: bar"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsKinds, err := kotsutil.LoadKotsKinds(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsKinds).ToNot(BeNil()) + Expect(kotsKinds.KotsApplication.Spec.Title).To(Equal("foo")) + }) + + It("loads kots kinds from root directory if neither 'kotsKinds' nor 'upstream' exist", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + err = os.WriteFile(filepath.Join(dir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsKinds, err := kotsutil.LoadKotsKinds(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsKinds).ToNot(BeNil()) + Expect(kotsKinds.KotsApplication.Spec.Title).To(Equal("foo")) + }) + }) + + Describe("FindKotsAppInPath()", func() { + It("returns nil if no kots app is found", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + err = os.WriteFile(filepath.Join(dir, "foo.yaml"), []byte("apiVersion: custom.io/v1beta1\nkind: Foo\nmetadata:\n name: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsApp, err := kotsutil.FindKotsAppInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsApp).To(BeNil()) + }) + + It("returns the kots app if found in the root directory", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + err = os.WriteFile(filepath.Join(dir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsApp, err := kotsutil.FindKotsAppInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsApp).ToNot(BeNil()) + Expect(kotsApp.Spec.Title).To(Equal("foo")) + }) + + It("returns the kots app if found in a subdirectory", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + subDir := filepath.Join(dir, "subdir") + err = os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(subDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsApp, err := kotsutil.FindKotsAppInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsApp).ToNot(BeNil()) + Expect(kotsApp.Spec.Title).To(Equal("foo")) + }) + + It("returns only one kots app if multiple are found", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + subDir := filepath.Join(dir, "subdir") + err = os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(dir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: foo\nspec:\n title: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(subDir, "kots-app.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Application\nmetadata:\n name: bar\nspec:\n title: bar"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsApp, err := kotsutil.FindKotsAppInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsApp).ToNot(BeNil()) + Expect(kotsApp.Spec.Title).To(Equal("foo")) + }) + }) + + Describe("FindConfigInPath()", func() { + It("returns nil if no config is found", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + err = os.WriteFile(filepath.Join(dir, "foo.yaml"), []byte("apiVersion: custom.io/v1beta1\nkind: Foo\nmetadata:\n name: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsConfig, err := kotsutil.FindConfigInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsConfig).To(BeNil()) + }) + + It("returns the config if found in the root directory", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + err = os.WriteFile(filepath.Join(dir, "kots-config.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Config\nmetadata:\n name: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsConfig, err := kotsutil.FindConfigInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsConfig).ToNot(BeNil()) + Expect(kotsConfig.ObjectMeta.Name).To(Equal("foo")) + }) + + It("returns the config if found in a subdirectory", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + subDir := filepath.Join(dir, "subdir") + err = os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(subDir, "kots-config.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Config\nmetadata:\n name: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsConfig, err := kotsutil.FindConfigInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsConfig).ToNot(BeNil()) + Expect(kotsConfig.ObjectMeta.Name).To(Equal("foo")) + }) + + It("returns only one config if multiple are found", func() { + dir, err := os.MkdirTemp("", "kotsutil-test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dir) + + subDir := filepath.Join(dir, "subdir") + err = os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(dir, "kots-config.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Config\nmetadata:\n name: foo"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(subDir, "kots-config.yaml"), []byte("apiVersion: kots.io/v1beta1\nkind: Config\nmetadata:\n name: bar"), 0644) + Expect(err).ToNot(HaveOccurred()) + + kotsConfig, err := kotsutil.FindConfigInPath(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(kotsConfig).ToNot(BeNil()) + Expect(kotsConfig.ObjectMeta.Name).To(Equal("foo")) + }) + }) + Describe("EncryptConfigValues()", func() { It("does not error when the config field is missing", func() { kotsKind := &kotsutil.KotsKinds{ @@ -381,50 +648,6 @@ var _ = Describe("Kots", func() { Expect(err).To(HaveOccurred()) }) }) - - Describe("GenUniqueKotsKindsFilename()", func() { - It("returns the same name when there are no file entries", func() { - filename := kotsutil.GenUniqueKotsKindFilename(nil, "unique") - Expect(filename).To(Equal("unique.yaml")) - - tmpRendered := map[string][]byte{} - filename = kotsutil.GenUniqueKotsKindFilename(tmpRendered, "unique") - Expect(filename).To(Equal("unique.yaml")) - }) - - It("returns the same name when there is no conflict", func() { - tmpRendered := map[string][]byte{ - "random.yaml": nil, - "example.yaml": nil, - } - - filename := kotsutil.GenUniqueKotsKindFilename(tmpRendered, "unique") - Expect(filename).To(Equal("unique.yaml")) - }) - - It("returns a unique name when there is a conflict", func() { - tmpRendered := map[string][]byte{ - "unique.yaml": nil, - "example.yaml": nil, - } - - filename := kotsutil.GenUniqueKotsKindFilename(tmpRendered, "unique") - Expect(filename).To(Equal("unique-1.yaml")) - }) - - It("returns a unique name when there is a conflict and the generated name creates a new conflict", func() { - tmpRendered := map[string][]byte{ - "unique.yaml": nil, - "unique-1.yaml": nil, - "unique-2.yaml": nil, - "unique-4.yaml": nil, - "example.yaml": nil, - } - - filename := kotsutil.GenUniqueKotsKindFilename(tmpRendered, "unique") - Expect(filename).To(Equal("unique-3.yaml")) - }) - }) }) func TestIsKotsKind(t *testing.T) { diff --git a/pkg/midstream/registry.go b/pkg/midstream/registry.go index 9223bfcd4a..8cb4ef2ea2 100644 --- a/pkg/midstream/registry.go +++ b/pkg/midstream/registry.go @@ -1,7 +1,6 @@ package midstream import ( - "io/ioutil" "os" "path/filepath" @@ -14,7 +13,7 @@ import ( func LoadPrivateRegistryInfo(archivePath string) (*registrytypes.RegistrySettings, error) { filename := filepath.Join(archivePath, "overlays", "midstream", secretFilename) - secretData, err := ioutil.ReadFile(filename) + secretData, err := os.ReadFile(filename) if err != nil { if os.IsNotExist(err) { return nil, nil diff --git a/pkg/midstream/write.go b/pkg/midstream/write.go index cdee5a5c32..9a58634b0a 100644 --- a/pkg/midstream/write.go +++ b/pkg/midstream/write.go @@ -37,20 +37,27 @@ const ( ) type WriteOptions struct { - MidstreamDir string - BaseDir string - AppSlug string - IsGitOps bool - IsOpenShift bool - Builder template.Builder - HTTPProxyEnvValue string - HTTPSProxyEnvValue string - NoProxyEnvValue string - UseHelmInstall map[string]bool - NewHelmCharts []*kotsv1beta1.HelmChart + MidstreamDir string + Base *base.Base + BaseDir string + AppSlug string + IsGitOps bool + IsOpenShift bool + Builder template.Builder + HTTPProxyEnvValue string + HTTPSProxyEnvValue string + NoProxyEnvValue string + UseHelmInstall map[string]bool + NewHelmCharts []*kotsv1beta1.HelmChart + ProcessImageOptions image.ProcessImageOptions + License *kotsv1beta1.License + RenderedKotsKinds *kotsutil.KotsKinds + IdentityConfig *kotsv1beta1.IdentityConfig + UpstreamDir string + Log *logger.CLILogger } -func WriteMidstream(writeMidstreamOptions WriteOptions, processImageOptions image.ProcessImageOptions, b *base.Base, license *kotsv1beta1.License, identityConfig *kotsv1beta1.IdentityConfig, upstreamDir string, log *logger.CLILogger) (*Midstream, error) { +func WriteMidstream(opts WriteOptions) (*Midstream, error) { var images []kustomizetypes.Image var objects []k8sdoc.K8sDoc var pullSecretRegistries []string @@ -62,85 +69,75 @@ func WriteMidstream(writeMidstreamOptions WriteOptions, processImageOptions imag return nil, errors.Wrap(err, "failed to get k8s clientset") } - newKotsKinds, err := kotsutil.LoadKotsKindsFromPath(upstreamDir) - if err != nil { - return nil, errors.Wrap(err, "failed to load kotskinds from new upstream") - } - - identitySpec, err := upstream.LoadIdentity(upstreamDir) - if err != nil { - return nil, errors.Wrap(err, "failed to load identity") - } - // do not fail on being unable to get dockerhub credentials, since they're just used to increase the rate limit var dockerHubRegistryCreds registry.Credentials - dockerhubSecret, _ := registry.GetDockerHubPullSecret(clientset, util.PodNamespace, processImageOptions.Namespace, processImageOptions.AppSlug) + dockerhubSecret, _ := registry.GetDockerHubPullSecret(clientset, util.PodNamespace, opts.ProcessImageOptions.Namespace, opts.ProcessImageOptions.AppSlug) if dockerhubSecret != nil { dockerHubRegistryCreds, _ = registry.GetCredentialsForRegistryFromConfigJSON(dockerhubSecret.Data[".dockerconfigjson"], registry.DockerHubRegistryName) } - if processImageOptions.RewriteImages { + if opts.ProcessImageOptions.RewriteImages { // A target registry is configured. Rewrite all images and copy them (if necessary) to the configured registry. - if processImageOptions.RegistrySettings.IsReadOnly { - log.ActionWithSpinner("Rewriting images") - io.WriteString(processImageOptions.ReportWriter, "Rewriting images\n") + if opts.ProcessImageOptions.RegistrySettings.IsReadOnly { + opts.Log.ActionWithSpinner("Rewriting images") + io.WriteString(opts.ProcessImageOptions.ReportWriter, "Rewriting images\n") } else { - log.ActionWithSpinner("Copying images") - io.WriteString(processImageOptions.ReportWriter, "Copying images\n") + opts.Log.ActionWithSpinner("Copying images") + io.WriteString(opts.ProcessImageOptions.ReportWriter, "Copying images\n") } - if processImageOptions.AirgapRoot == "" { + if opts.ProcessImageOptions.AirgapRoot == "" { // This is an online installation. Pull and rewrite images from online and copy them (if necessary) to the configured registry. - rewriteResult, err := RewriteBaseImages(processImageOptions, writeMidstreamOptions.BaseDir, newKotsKinds, license, dockerHubRegistryCreds, log) + rewriteResult, err := RewriteBaseImages(opts.ProcessImageOptions, opts.BaseDir, opts.RenderedKotsKinds, opts.License, dockerHubRegistryCreds, opts.Log) if err != nil { return nil, errors.Wrap(err, "failed to rewrite base images") } images = rewriteResult.Images - newKotsKinds.Installation.Spec.KnownImages = rewriteResult.CheckedImages + opts.RenderedKotsKinds.Installation.Spec.KnownImages = rewriteResult.CheckedImages } else { // This is an airgapped installation. Copy and rewrite images from the airgap bundle to the configured registry. - result, err := ProcessAirgapImages(processImageOptions, newKotsKinds, license, log) + result, err := ProcessAirgapImages(opts.ProcessImageOptions, opts.RenderedKotsKinds, opts.License, opts.Log) if err != nil { return nil, errors.Wrap(err, "failed to process airgap images") } images = result.KustomizeImages - newKotsKinds.Installation.Spec.KnownImages = result.KnownImages + opts.RenderedKotsKinds.Installation.Spec.KnownImages = result.KnownImages } - objects = base.FindObjectsWithImages(b) + objects = base.FindObjectsWithImages(opts.Base) // Use target registry credentials to create image pull secrets for all objects that have images. - pullSecretRegistries = []string{processImageOptions.RegistrySettings.Hostname} - pullSecretUsername = processImageOptions.RegistrySettings.Username - pullSecretPassword = processImageOptions.RegistrySettings.Password + pullSecretRegistries = []string{opts.ProcessImageOptions.RegistrySettings.Hostname} + pullSecretUsername = opts.ProcessImageOptions.RegistrySettings.Username + pullSecretPassword = opts.ProcessImageOptions.RegistrySettings.Password if pullSecretUsername == "" { - pullSecretUsername, pullSecretPassword, err = registry.LoadAuthForRegistry(processImageOptions.RegistrySettings.Hostname) + pullSecretUsername, pullSecretPassword, err = registry.LoadAuthForRegistry(opts.ProcessImageOptions.RegistrySettings.Hostname) if err != nil { - return nil, errors.Wrapf(err, "failed to load registry auth for %q", processImageOptions.RegistrySettings.Hostname) + return nil, errors.Wrapf(err, "failed to load registry auth for %q", opts.ProcessImageOptions.RegistrySettings.Hostname) } } - } else if license != nil { + } else if opts.License != nil { // A target registry is NOT configured. Find and rewrite private images to be proxied through proxy.replicated.com - findResult, err := findPrivateImages(writeMidstreamOptions, b, newKotsKinds, license, dockerHubRegistryCreds) + findResult, err := findPrivateImages(opts, dockerHubRegistryCreds) if err != nil { return nil, errors.Wrap(err, "failed to find private images") } images = findResult.Images - newKotsKinds.Installation.Spec.KnownImages = findResult.CheckedImages + opts.RenderedKotsKinds.Installation.Spec.KnownImages = findResult.CheckedImages objects = findResult.Docs // Use license to create image pull secrets for all objects that have private images. - pullSecretRegistries = registry.GetRegistryProxyInfo(license, &newKotsKinds.Installation, &newKotsKinds.KotsApplication).ToSlice() - pullSecretUsername = license.Spec.LicenseID - pullSecretPassword = license.Spec.LicenseID + pullSecretRegistries = registry.GetRegistryProxyInfo(opts.License, &opts.RenderedKotsKinds.Installation, &opts.RenderedKotsKinds.KotsApplication).ToSlice() + pullSecretUsername = opts.License.Spec.LicenseID + pullSecretPassword = opts.License.Spec.LicenseID } // For the newer style charts, create a new secret per chart as helm adds chart specific // details to annotations and labels to it. - namePrefix := processImageOptions.AppSlug - for _, v := range writeMidstreamOptions.NewHelmCharts { - if v.Spec.UseHelmInstall && filepath.Base(b.Path) != "." { - namePrefix = fmt.Sprintf("%s-%s", processImageOptions.AppSlug, filepath.Base(b.Path)) + namePrefix := opts.ProcessImageOptions.AppSlug + for _, v := range opts.NewHelmCharts { + if v.Spec.UseHelmInstall && filepath.Base(opts.Base.Path) != "." { + namePrefix = fmt.Sprintf("%s-%s", opts.ProcessImageOptions.AppSlug, filepath.Base(opts.Base.Path)) break } } @@ -148,7 +145,7 @@ func WriteMidstream(writeMidstreamOptions WriteOptions, processImageOptions imag pullSecretRegistries, pullSecretUsername, pullSecretPassword, - processImageOptions.Namespace, + opts.ProcessImageOptions.Namespace, namePrefix, ) if err != nil { @@ -156,16 +153,16 @@ func WriteMidstream(writeMidstreamOptions WriteOptions, processImageOptions imag } pullSecrets.DockerHubSecret = dockerhubSecret - if err := apparchive.SaveInstallation(&newKotsKinds.Installation, upstreamDir); err != nil { + if err := apparchive.SaveInstallation(&opts.RenderedKotsKinds.Installation, opts.UpstreamDir); err != nil { return nil, errors.Wrap(err, "failed to save installation") } - m, err := CreateMidstream(b, images, objects, &pullSecrets, identitySpec, identityConfig) + m, err := CreateMidstream(opts.Base, images, objects, &pullSecrets, opts.RenderedKotsKinds.Identity, opts.IdentityConfig) if err != nil { return nil, errors.Wrap(err, "failed to create midstream") } - if err := m.Write(writeMidstreamOptions); err != nil { + if err := m.Write(opts); err != nil { return nil, errors.Wrap(err, "failed to write common midstream") } @@ -242,7 +239,7 @@ func ProcessAirgapImages(options image.ProcessImageOptions, kotsKinds *kotsutil. processAirgapImageOptions.ReplicatedRegistry.Password = license.Spec.LicenseID } - imagesData, err := ioutil.ReadFile(filepath.Join(options.AirgapRoot, "images.json")) + imagesData, err := os.ReadFile(filepath.Join(options.AirgapRoot, "images.json")) if err != nil && !os.IsNotExist(err) { return nil, errors.Wrap(err, "failed to load images file") } @@ -265,13 +262,18 @@ func ProcessAirgapImages(options image.ProcessImageOptions, kotsKinds *kotsutil. } // findPrivateImages Finds and rewrites private images to be proxied through proxy.replicated.com -func findPrivateImages(writeMidstreamOptions WriteOptions, b *base.Base, kotsKinds *kotsutil.KotsKinds, license *kotsv1beta1.License, dockerHubRegistryCreds registry.Credentials) (*base.FindPrivateImagesResult, error) { - replicatedRegistryInfo := registry.GetRegistryProxyInfo(license, &kotsKinds.Installation, &kotsKinds.KotsApplication) - allPrivate := kotsKinds.KotsApplication.Spec.ProxyPublicImages +func findPrivateImages(opts WriteOptions, dockerHubRegistryCreds registry.Credentials) (*base.FindPrivateImagesResult, error) { + replicatedRegistryInfo := registry.GetRegistryProxyInfo(opts.License, &opts.RenderedKotsKinds.Installation, &opts.RenderedKotsKinds.KotsApplication) + allPrivate := opts.RenderedKotsKinds.KotsApplication.Spec.ProxyPublicImages + + kotsKindsImages, err := kotsutil.GetImagesFromKotsKinds(opts.RenderedKotsKinds, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to get images from kots kinds") + } findPrivateImagesOptions := base.FindPrivateImagesOptions{ - BaseDir: writeMidstreamOptions.BaseDir, - AppSlug: license.Spec.AppSlug, + BaseDir: opts.BaseDir, + AppSlug: opts.License.Spec.AppSlug, ReplicatedRegistry: dockerregistrytypes.RegistryOptions{ Endpoint: replicatedRegistryInfo.Registry, ProxyEndpoint: replicatedRegistryInfo.Proxy, @@ -281,11 +283,11 @@ func findPrivateImages(writeMidstreamOptions WriteOptions, b *base.Base, kotsKin Username: dockerHubRegistryCreds.Username, Password: dockerHubRegistryCreds.Password, }, - Installation: &kotsKinds.Installation, + Installation: &opts.RenderedKotsKinds.Installation, AllImagesPrivate: allPrivate, - HelmChartPath: b.Path, - UseHelmInstall: writeMidstreamOptions.UseHelmInstall, - KotsKindsImages: kotsutil.GetImagesFromKotsKinds(kotsKinds), + HelmChartPath: opts.Base.Path, + UseHelmInstall: opts.UseHelmInstall, + KotsKindsImages: kotsKindsImages, } findResult, err := base.FindPrivateImages(findPrivateImagesOptions) if err != nil { diff --git a/pkg/online/online.go b/pkg/online/online.go index 2d91e585e4..6075d02ae8 100644 --- a/pkg/online/online.go +++ b/pkg/online/online.go @@ -5,7 +5,6 @@ import ( "io" "io/ioutil" "os" - "path/filepath" "time" "github.com/pkg/errors" @@ -194,7 +193,7 @@ func CreateAppFromOnline(opts CreateOnlineAppOpts) (_ *kotsutil.KotsKinds, final return nil, errors.Wrap(err, "failed to create rendered support bundle spec") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(tmpRoot, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(tmpRoot) if err != nil { return nil, errors.Wrap(err, "failed to load kotskinds from path") } diff --git a/pkg/operator/client/deploy.go b/pkg/operator/client/deploy.go index 4d2f881e3a..a266650349 100644 --- a/pkg/operator/client/deploy.go +++ b/pkg/operator/client/deploy.go @@ -448,7 +448,7 @@ func getSortedCharts(v1Beta1ChartsDir string, v1Beta2ChartsDir string, kotsChart func findChartNameAndVersion(chartDir string) (string, string, error) { chartfilePath := filepath.Join(chartDir, "Chart.yaml") - chartFile, err := ioutil.ReadFile(chartfilePath) + chartFile, err := os.ReadFile(chartfilePath) if err != nil { return "", "", errors.Wrapf(err, "failed to parse %s", chartfilePath) } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 9409ca25d2..962c9e80df 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "strconv" "sync" "time" @@ -235,7 +234,7 @@ func (o *Operator) DeployApp(appID string, sequence int64) (deployed bool, deplo return false, errors.Wrap(err, "failed to ensure disaster recovery label transformer") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(deployedVersionArchive, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(deployedVersionArchive) if err != nil { return false, errors.Wrap(err, "failed to load kotskinds") } @@ -356,7 +355,7 @@ func (o *Operator) DeployApp(appID string, sequence int64) (deployed bool, deplo return false, errors.Wrap(err, "failed to get previously deployed app version archive") } - previousKotsKinds, err = kotsutil.LoadKotsKindsFromPath(filepath.Join(previouslyDeployedVersionArchive, "upstream")) + previousKotsKinds, err = kotsutil.LoadKotsKinds(previouslyDeployedVersionArchive) if err != nil { return false, errors.Wrap(err, "failed to load kotskinds for previously deployed app version") } @@ -533,7 +532,7 @@ func (o *Operator) resumeInformersForApp(app *apptypes.App) error { return errors.Wrap(err, "failed to get image pull secrets") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(deployedVersionArchive, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(deployedVersionArchive) if err != nil { return errors.Wrap(err, "failed to load kotskinds") } @@ -750,7 +749,7 @@ func (o *Operator) UndeployApp(a *apptypes.App, d *downstreamtypes.Downstream, i return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(deployedVersionArchive, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(deployedVersionArchive) if err != nil { return errors.Wrap(err, "failed to load kotskinds") } diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go index 88aea32c97..d1d46c2cc9 100644 --- a/pkg/preflight/preflight.go +++ b/pkg/preflight/preflight.go @@ -45,7 +45,7 @@ const ( ) func Run(appID string, appSlug string, sequence int64, isAirgap bool, archiveDir string) error { - upstreamKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load rendered kots kinds") } @@ -89,7 +89,7 @@ func Run(appID string, appSlug string, sequence int64, isAirgap bool, archiveDir preflight = troubleshootpreflight.ConcatPreflightSpec(preflight, &v) } - injectDefaultPreflights(preflight, upstreamKotsKinds, registrySettings) + injectDefaultPreflights(preflight, kotsKinds, registrySettings) numAnalyzers := 0 for _, analyzer := range preflight.Spec.Analyzers { @@ -99,15 +99,15 @@ func Run(appID string, appSlug string, sequence int64, isAirgap bool, archiveDir } } runPreflights = numAnalyzers > 0 - } else if upstreamKotsKinds.Preflight != nil { + } else if kotsKinds.Preflight != nil { // render the preflight file // we need to convert to bytes first, so that we can reuse the renderfile function - renderedMarshalledPreflights, err := upstreamKotsKinds.Marshal("troubleshoot.replicated.com", "v1beta1", "Preflight") + renderedMarshalledPreflights, err := kotsKinds.Marshal("troubleshoot.replicated.com", "v1beta1", "Preflight") if err != nil { return errors.Wrap(err, "failed to marshal rendered preflight") } - renderedPreflight, err := render.RenderFile(upstreamKotsKinds, registrySettings, appSlug, sequence, isAirgap, util.PodNamespace, []byte(renderedMarshalledPreflights)) + renderedPreflight, err := render.RenderFile(kotsKinds, registrySettings, appSlug, sequence, isAirgap, util.PodNamespace, []byte(renderedMarshalledPreflights)) if err != nil { return errors.Wrap(err, "failed to render preflights") } @@ -116,7 +116,7 @@ func Run(appID string, appSlug string, sequence int64, isAirgap bool, archiveDir return errors.Wrap(err, "failed to load rendered preflight") } - injectDefaultPreflights(preflight, upstreamKotsKinds, registrySettings) + injectDefaultPreflights(preflight, kotsKinds, registrySettings) numAnalyzers := 0 for _, analyzer := range preflight.Spec.Analyzers { @@ -153,7 +153,7 @@ func Run(appID string, appSlug string, sequence int64, isAirgap bool, archiveDir } } - collectors, err := registry.UpdateCollectorSpecsWithRegistryData(preflight.Spec.Collectors, registrySettings, upstreamKotsKinds.Installation, upstreamKotsKinds.License, &upstreamKotsKinds.KotsApplication) + collectors, err := registry.UpdateCollectorSpecsWithRegistryData(preflight.Spec.Collectors, registrySettings, kotsKinds.Installation, kotsKinds.License, &kotsKinds.KotsApplication) if err != nil { preflightErr = errors.Wrap(err, "failed to rewrite images in preflight") return preflightErr diff --git a/pkg/print/velero.go b/pkg/print/velero.go index eccf6517ef..b23eae95c8 100644 --- a/pkg/print/velero.go +++ b/pkg/print/velero.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/fatih/color" - "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" "github.com/replicatedhq/kots/pkg/logger" @@ -39,7 +39,7 @@ func VeleroInstallationInstructionsForCLI(log *logger.CLILogger, plugin snapshot // this is an airgapped installation registry := kotsadmversion.KotsadmRegistry(*registryConfig) pluginName := strings.Split(strings.Split(string(plugin), "/")[1], ":")[0] - pluginTag, _ := image.GetTag(string(plugin)) + pluginTag, _ := imageutil.GetTag(string(plugin)) veleroAirgapCommand := fmt.Sprintf(`velero install \ --no-default-backup-location \ @@ -102,7 +102,7 @@ func VeleroInstallationInstructionsForUI(plugin snapshottypes.VeleroPlugin, regi // this is an airgapped installation registry := kotsadmversion.KotsadmRegistry(*registryConfig) pluginName := strings.Split(strings.Split(string(plugin), "/")[1], ":")[0] - pluginTag, _ := image.GetTag(string(plugin)) + pluginTag, _ := imageutil.GetTag(string(plugin)) veleroAirgapCommand := fmt.Sprintf(`velero install --no-default-backup-location --no-secret --use-node-agent --uploader-type=restic --use-volume-snapshots=false --image %s/velero:%s --plugins %s/%s:%s`, registry, "", registry, pluginName, pluginTag) diff --git a/pkg/pull/archive.go b/pkg/pull/archive.go index 09767a9648..bafbd32aea 100644 --- a/pkg/pull/archive.go +++ b/pkg/pull/archive.go @@ -55,7 +55,7 @@ func writeArchiveAsConfigMap(pullOptions PullOptions, u *upstreamtypes.Upstream, return errors.Wrap(err, "failed to create tar gz") } - archive, err := ioutil.ReadFile(path.Join(tempDir, "kots-uploadable-archive.tar.gz")) + archive, err := os.ReadFile(path.Join(tempDir, "kots-uploadable-archive.tar.gz")) if err != nil { return errors.Wrap(err, "failed to read temp file") } @@ -110,7 +110,7 @@ func writeArchiveAsConfigMap(pullOptions PullOptions, u *upstreamtypes.Upstream, } func CleanBaseArchive(path string) error { - files, err := ioutil.ReadDir(path) + files, err := os.ReadDir(path) if err != nil { return errors.Wrap(err, "failed to read dir") } diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go index 24ef5f4436..862531c54c 100644 --- a/pkg/pull/pull.go +++ b/pkg/pull/pull.go @@ -30,6 +30,7 @@ import ( upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types" "github.com/replicatedhq/kots/pkg/util" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" + kotsv1beta2 "github.com/replicatedhq/kotskinds/apis/kots/v1beta2" k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/kubernetes/scheme" ) @@ -68,6 +69,7 @@ type PullOptions struct { NoProxyEnvValue string ReportingInfo *reportingtypes.ReportingInfo SkipCompatibilityCheck bool + KotsKinds *kotsutil.KotsKinds } var ( @@ -238,7 +240,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { return "", errors.Wrap(err, "failed to validate app key") } - airgapAppFiles, err := ioutil.TempDir("", "airgap-kots") + airgapAppFiles, err := os.MkdirTemp("", "airgap-kots") if err != nil { return "", errors.Wrap(err, "failed to create temp airgap dir") } @@ -253,10 +255,12 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { fetchOptions.LocalPath = airgapAppFiles } - // the root directory contains the manifests of the previous version and gets rewritten when fetching the upstream, so we load previous charts before fetching the upstream. - prevHelmCharts, err := kotsutil.LoadV1Beta1HelmChartsFromPath(pullOptions.RootDir) - if err != nil { - return "", errors.Wrap(err, "failed to load previous helm charts") + prevV1Beta1HelmCharts := []*kotsv1beta1.HelmChart{} + if pullOptions.KotsKinds != nil && pullOptions.KotsKinds.V1Beta1HelmCharts != nil { + for _, v1Beta1Chart := range pullOptions.KotsKinds.V1Beta1HelmCharts.Items { + kc := v1Beta1Chart + prevV1Beta1HelmCharts = append(prevV1Beta1HelmCharts, &kc) + } } log.ActionWithSpinner("Pulling upstream") @@ -297,11 +301,6 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { } log.FinishSpinner() - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { - return "", errors.Wrap(err, "failed to load kotskinds") - } - registrySettings := registrytypes.RegistrySettings{ Hostname: pullOptions.RewriteImageOptions.Hostname, Namespace: pullOptions.RewriteImageOptions.Namespace, @@ -310,7 +309,33 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { IsReadOnly: pullOptions.RewriteImageOptions.IsReadOnly, } - needsConfig, err := kotsadmconfig.NeedsConfiguration(pullOptions.AppSlug, pullOptions.AppSequence, pullOptions.AirgapRoot != "", kotsKinds, registrySettings) + renderOptions := base.RenderOptions{ + SplitMultiDocYAML: true, + Namespace: pullOptions.Namespace, + RegistrySettings: registrySettings, + ExcludeKotsKinds: pullOptions.ExcludeKotsKinds, + Log: log, + AppSlug: pullOptions.AppSlug, + Sequence: pullOptions.AppSequence, + IsAirgap: pullOptions.AirgapRoot != "", + } + log.ActionWithSpinner("Rendering KOTS custom resources") + io.WriteString(pullOptions.ReportWriter, "Rendering KOTS custom resources\n") + + renderedKotsKindsMap, err := base.RenderKotsKinds(u, &renderOptions) + if err != nil { + log.FinishSpinnerWithError() + return "", errors.Wrap(err, "failed to render upstream") + } + + renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) + if err != nil { + log.FinishSpinnerWithError() + return "", errors.Wrap(err, "failed to load rendered kotskinds from map") + } + log.FinishSpinner() + + needsConfig, err := kotsadmconfig.NeedsConfiguration(pullOptions.AppSlug, pullOptions.AppSequence, pullOptions.AirgapRoot != "", renderedKotsKinds, registrySettings) if err != nil { return "", errors.Wrap(err, "failed to check if version needs configuration") } @@ -333,31 +358,31 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { if needsConfig { if processImageOptions.RewriteImages && processImageOptions.AirgapRoot != "" { // if this is an airgap install, we still need to process the images - if _, err = midstream.ProcessAirgapImages(processImageOptions, kotsKinds, fetchOptions.License, log); err != nil { + if _, err = midstream.ProcessAirgapImages(processImageOptions, renderedKotsKinds, fetchOptions.License, log); err != nil { return "", errors.Wrap(err, "failed to process airgap images") } } - return "", ErrConfigNeeded } - renderDir := pullOptions.RootDir - if pullOptions.CreateAppDir { - renderDir = filepath.Join(pullOptions.RootDir, u.Name) - } - - v1Beta1HelmCharts, err := kotsutil.LoadV1Beta1HelmChartsFromPath(renderDir) - if err != nil { - return "", errors.Wrap(err, "failed to load new v1beta1 helm charts") + var v1Beta1HelmCharts []*kotsv1beta1.HelmChart + if renderedKotsKinds.V1Beta1HelmCharts != nil { + for _, v1Beta1Chart := range renderedKotsKinds.V1Beta1HelmCharts.Items { + kc := v1Beta1Chart + v1Beta1HelmCharts = append(v1Beta1HelmCharts, &kc) + } } - v1Beta2HelmCharts, err := kotsutil.LoadV1Beta2HelmChartsFromPath(renderDir) - if err != nil { - return "", errors.Wrap(err, "failed to load new v1beta2 helm charts") + var v1Beta2HelmCharts []*kotsv1beta2.HelmChart + if renderedKotsKinds.V1Beta2HelmCharts != nil { + for _, v1Beta2Chart := range renderedKotsKinds.V1Beta2HelmCharts.Items { + kc := v1Beta2Chart + v1Beta2HelmCharts = append(v1Beta2HelmCharts, &kc) + } } if !pullOptions.SkipHelmChartCheck { - for _, prevChart := range prevHelmCharts { + for _, prevChart := range prevV1Beta1HelmCharts { if !prevChart.Spec.Exclude.IsEmpty() { isExcluded, err := prevChart.Spec.Exclude.Boolean() if err == nil && isExcluded { @@ -378,7 +403,6 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { if prevChart.GetReleaseName() != newChart.GetReleaseName() { continue } - if !prevChart.Spec.UseHelmInstall { return "", errors.Errorf("cannot upgrade chart release %s to v1beta2 because useHelmInstall is false", newChart.GetReleaseName()) } @@ -386,24 +410,10 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { } } - renderOptions := base.RenderOptions{ - SplitMultiDocYAML: true, - Namespace: pullOptions.Namespace, - LocalRegistryHost: pullOptions.RewriteImageOptions.Hostname, - LocalRegistryNamespace: pullOptions.RewriteImageOptions.Namespace, - LocalRegistryUsername: pullOptions.RewriteImageOptions.Username, - LocalRegistryPassword: pullOptions.RewriteImageOptions.Password, - LocalRegistryIsReadOnly: pullOptions.RewriteImageOptions.IsReadOnly, - ExcludeKotsKinds: pullOptions.ExcludeKotsKinds, - Log: log, - AppSlug: pullOptions.AppSlug, - Sequence: pullOptions.AppSequence, - IsAirgap: pullOptions.AirgapRoot != "", - } log.ActionWithSpinner("Creating base") io.WriteString(pullOptions.ReportWriter, "Creating base\n") - commonBase, helmBases, renderedKotsKindsMap, err := base.RenderUpstream(u, &renderOptions) + commonBase, helmBases, err := base.RenderUpstream(u, &renderOptions, renderedKotsKinds) if err != nil { log.FinishSpinnerWithError() return "", errors.Wrap(err, "failed to render upstream") @@ -427,14 +437,9 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { files = append(files, file) } - newKotsKinds, err := kotsutil.LoadKotsKindsFromPath(u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { - return "", errors.Wrap(err, "failed to load kotskinds") - } - newKotsKinds.Installation.Spec.YAMLErrors = files + renderedKotsKinds.Installation.Spec.YAMLErrors = files - err = apparchive.SaveInstallation(&newKotsKinds.Installation, u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { + if err := apparchive.SaveInstallation(&renderedKotsKinds.Installation, u.GetUpstreamDir(writeUpstreamOptions)); err != nil { log.FinishSpinnerWithError() return "", errors.Wrap(err, "failed to save installation") } @@ -480,14 +485,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { return "", errors.Wrap(err, "failed to create new config context template builder") } - newKotsKinds, err := kotsutil.LoadKotsKindsFromPath(u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { - log.FinishSpinnerWithError() - return "", errors.Wrap(err, "failed to load kotskinds") - } - - err = crypto.InitFromString(newKotsKinds.Installation.Spec.EncryptionKey) - if err != nil { + if err := crypto.InitFromString(renderedKotsKinds.Installation.Spec.EncryptionKey); err != nil { log.FinishSpinnerWithError() return "", errors.Wrap(err, "failed to load encryption cipher") } @@ -501,6 +499,11 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { HTTPSProxyEnvValue: pullOptions.HTTPSProxyEnvValue, NoProxyEnvValue: pullOptions.NoProxyEnvValue, NewHelmCharts: v1Beta1HelmCharts, + License: fetchOptions.License, + RenderedKotsKinds: renderedKotsKinds, + IdentityConfig: identityConfig, + UpstreamDir: u.GetUpstreamDir(writeUpstreamOptions), + Log: log, } // the UseHelmInstall map blocks visibility into charts and subcharts when searching for private images @@ -526,8 +529,10 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { writeMidstreamOptions := commonWriteMidstreamOptions writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream") writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), commonBase.Path) + writeMidstreamOptions.ProcessImageOptions = processImageOptions + writeMidstreamOptions.Base = commonBase - m, err := midstream.WriteMidstream(writeMidstreamOptions, processImageOptions, commonBase, fetchOptions.License, identityConfig, u.GetUpstreamDir(writeUpstreamOptions), log) + m, err := midstream.WriteMidstream(writeMidstreamOptions) if err != nil { log.FinishSpinnerWithError() return "", errors.Wrap(err, "failed to write common midstream") @@ -542,16 +547,18 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { previousUseHelmInstall := writeMidstreamOptions.UseHelmInstall[helmBase.Path] writeMidstreamOptions.UseHelmInstall[helmBase.Path] = false - writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream", helmBase.Path) - writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBase.Path) - helmBaseCopy := helmBase.DeepCopy() processImageOptionsCopy := processImageOptions processImageOptionsCopy.Namespace = helmBaseCopy.Namespace processImageOptionsCopy.PushImages = false // never push images more than once - helmMidstream, err := midstream.WriteMidstream(writeMidstreamOptions, processImageOptionsCopy, helmBaseCopy, fetchOptions.License, identityConfig, u.GetUpstreamDir(writeUpstreamOptions), log) + writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream", helmBaseCopy.Path) + writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBaseCopy.Path) + writeMidstreamOptions.ProcessImageOptions = processImageOptionsCopy + writeMidstreamOptions.Base = helmBaseCopy + + helmMidstream, err := midstream.WriteMidstream(writeMidstreamOptions) if err != nil { log.FinishSpinnerWithError() return "", errors.Wrapf(err, "failed to write helm midstream %s", helmBase.Path) @@ -569,12 +576,6 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { return "", errors.Wrapf(err, "failed to remove unused helm midstreams") } - renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) - if err != nil { - log.FinishSpinnerWithError() - return "", errors.Wrap(err, "failed to load rendered kotskinds from map") - } - writeV1Beta2HelmChartsOpts := apparchive.WriteV1Beta2HelmChartsOptions{ Upstream: u, WriteUpstreamOptions: writeUpstreamOptions, @@ -600,16 +601,14 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { } } - installationBytes, err := ioutil.ReadFile(filepath.Join(u.GetUpstreamDir(writeUpstreamOptions), "userdata", "installation.yaml")) + // installation spec gets updated during the process, ensure the map has the latest version + installationBytes, err := os.ReadFile(filepath.Join(u.GetUpstreamDir(writeUpstreamOptions), "userdata", "installation.yaml")) if err != nil { return "", errors.Wrap(err, "failed to read installation file") } + renderedKotsKindsMap["userdata/installation.yaml"] = []byte(installationBytes) - installationFilename := kotsutil.GenUniqueKotsKindFilename(renderedKotsKindsMap, "installation") - renderedKotsKindsMap[installationFilename] = []byte(installationBytes) - - err = kotsutil.WriteKotsKinds(renderedKotsKindsMap, u.GetKotsKindsDir(writeUpstreamOptions)) - if err != nil { + if err := kotsutil.WriteKotsKinds(renderedKotsKindsMap, u.GetKotsKindsDir(writeUpstreamOptions)); err != nil { return "", errors.Wrap(err, "failed to write the rendered kots kinds") } @@ -618,7 +617,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { OverlaysDir: u.GetOverlaysDir(writeUpstreamOptions), RenderedDir: u.GetRenderedDir(writeUpstreamOptions), Downstreams: pullOptions.Downstreams, - KustomizeBinPath: kotsKinds.GetKustomizeBinaryPath(), + KustomizeBinPath: renderedKotsKinds.GetKustomizeBinaryPath(), HelmDir: u.GetHelmDir(writeUpstreamOptions), Log: log, KotsKinds: renderedKotsKinds, @@ -726,7 +725,7 @@ func writeDownstreams(options PullOptions, overlaysDir string, m *midstream.Mids } func ParseConfigValuesFromFile(filename string) (*kotsv1beta1.ConfigValues, error) { - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -750,7 +749,7 @@ func ParseConfigValuesFromFile(filename string) (*kotsv1beta1.ConfigValues, erro } func ParseIdentityConfigFromFile(filename string) (*kotsv1beta1.IdentityConfig, error) { - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -779,7 +778,7 @@ func GetAppMetadataFromAirgap(airgapArchive string) (*replicatedapp.ApplicationM return nil, errors.Wrap(err, "failed to extract app archive") } - tempDir, err := ioutil.TempDir("", "kotsadm") + tempDir, err := os.MkdirTemp("", "kotsadm") if err != nil { return nil, errors.Wrap(err, "failed to create temp dir") } @@ -790,15 +789,19 @@ func GetAppMetadataFromAirgap(airgapArchive string) (*replicatedapp.ApplicationM return nil, errors.Wrap(err, "failed to extract app archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(tempDir) + kotsApp, err := kotsutil.FindKotsAppInPath(tempDir) if err != nil { return nil, errors.Wrap(err, "failed to read kots kinds") } + if kotsApp == nil { + ka := kotsutil.EmptyKotsKinds().KotsApplication + kotsApp = &ka + } s := k8sjson.NewYAMLSerializer(k8sjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) var b bytes.Buffer - if err := s.Encode(&kotsKinds.KotsApplication, &b); err != nil { + if err := s.Encode(kotsApp, &b); err != nil { return nil, errors.Wrap(err, "failed to encode metadata") } @@ -814,7 +817,7 @@ func GetAppMetadataFromAirgap(airgapArchive string) (*replicatedapp.ApplicationM } func parseInstallationFromFile(filename string) (*kotsv1beta1.Installation, error) { - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -887,7 +890,7 @@ func findConfig(localPath string) (*kotsv1beta1.Config, *kotsv1beta1.ConfigValue return nil } - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return err } diff --git a/pkg/registry/images.go b/pkg/registry/images.go index a185dcc91e..97b1fe1e01 100644 --- a/pkg/registry/images.go +++ b/pkg/registry/images.go @@ -18,7 +18,7 @@ import ( "github.com/pkg/errors" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types" - "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/k8sutil" kotsadmobjects "github.com/replicatedhq/kots/pkg/kotsadm/objects" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" @@ -298,7 +298,7 @@ func deleteUnusedImages(ctx context.Context, registry types.RegistrySettings, us Password: registry.Password, } - appImage, err := image.DestImage(registryOptions, usedImage) + appImage, err := imageutil.DestImage(registryOptions, usedImage) if err != nil { return errors.Wrapf(err, "failed to get destination image for %s", appImage) } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index a9d0cf0fa8..cca11302b5 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -6,7 +6,6 @@ import ( _ "embed" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -66,7 +65,7 @@ func RewriteImages(appID string, sequence int64, hostname string, username strin }() // get the archive and store it in a temporary location - appDir, err := ioutil.TempDir("", "kotsadm") + appDir, err := os.MkdirTemp("", "kotsadm") if err != nil { return "", errors.Wrap(err, "failed to create temp dir") } @@ -77,7 +76,7 @@ func RewriteImages(appID string, sequence int64, hostname string, username strin return "", errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(appDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(appDir) if err != nil { return "", errors.Wrap(err, "failed to load kotskinds from path") } @@ -134,7 +133,6 @@ func RewriteImages(appID string, sequence int64, hostname string, username strin ExcludeKotsKinds: true, License: kotsKinds.License, ConfigValues: configValues, - KotsApplication: &kotsKinds.KotsApplication, K8sNamespace: appNamespace, ReportWriter: pipeWriter, IsAirgap: a.IsAirgap, diff --git a/pkg/render/render.go b/pkg/render/render.go index 8b09f05c17..16ecee507d 100644 --- a/pkg/render/render.go +++ b/pkg/render/render.go @@ -102,7 +102,7 @@ func (r Renderer) RenderDir(archiveDir string, a *apptypes.App, downstreams []do } func RenderDir(archiveDir string, a *apptypes.App, downstreams []downstreamtypes.Downstream, registrySettings registrytypes.RegistrySettings, sequence int64) error { - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kotskinds from path") } @@ -128,7 +128,6 @@ func RenderDir(archiveDir string, a *apptypes.App, downstreams []downstreamtypes ExcludeKotsKinds: true, License: kotsKinds.License, ConfigValues: kotsKinds.ConfigValues, - KotsApplication: &kotsKinds.KotsApplication, K8sNamespace: appNamespace, CopyImages: false, IsAirgap: a.IsAirgap, diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index 96d4b3cacc..8b375242d0 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -9,7 +9,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "github.com/pkg/errors" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" @@ -322,7 +321,7 @@ func getDownstreamInfo(appID string) (*types.DownstreamInfo, error) { return nil, errors.Wrap(err, "failed to get app version archive") } - deployedKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(deployedArchiveDir, "upstream")) + deployedKotsKinds, err := kotsutil.LoadKotsKinds(deployedArchiveDir) if err != nil { return nil, errors.Wrap(err, "failed to load kotskinds from path") } diff --git a/pkg/rewrite/rewrite.go b/pkg/rewrite/rewrite.go index 3b1ba5e4ec..e552f0f846 100644 --- a/pkg/rewrite/rewrite.go +++ b/pkg/rewrite/rewrite.go @@ -38,7 +38,6 @@ type RewriteOptions struct { Installation *kotsv1beta1.Installation License *kotsv1beta1.License ConfigValues *kotsv1beta1.ConfigValues - KotsApplication *kotsv1beta1.Application ReportWriter io.Writer CopyImages bool // can be false even if registry is not read-only IsAirgap bool @@ -115,23 +114,35 @@ func Rewrite(rewriteOptions RewriteOptions) error { log.FinishSpinner() renderOptions := base.RenderOptions{ - SplitMultiDocYAML: true, - Namespace: rewriteOptions.K8sNamespace, - LocalRegistryHost: rewriteOptions.RegistrySettings.Hostname, - LocalRegistryNamespace: rewriteOptions.RegistrySettings.Namespace, - LocalRegistryUsername: rewriteOptions.RegistrySettings.Username, - LocalRegistryPassword: rewriteOptions.RegistrySettings.Password, - LocalRegistryIsReadOnly: rewriteOptions.RegistrySettings.IsReadOnly, - ExcludeKotsKinds: rewriteOptions.ExcludeKotsKinds, - Log: log, - AppSlug: rewriteOptions.AppSlug, - Sequence: rewriteOptions.AppSequence, - IsAirgap: rewriteOptions.IsAirgap, + SplitMultiDocYAML: true, + Namespace: rewriteOptions.K8sNamespace, + RegistrySettings: rewriteOptions.RegistrySettings, + ExcludeKotsKinds: rewriteOptions.ExcludeKotsKinds, + Log: log, + AppSlug: rewriteOptions.AppSlug, + Sequence: rewriteOptions.AppSequence, + IsAirgap: rewriteOptions.IsAirgap, + } + log.ActionWithSpinner("Rendering KOTS custom resources") + io.WriteString(rewriteOptions.ReportWriter, "Rendering KOTS custom resources\n") + + renderedKotsKindsMap, err := base.RenderKotsKinds(u, &renderOptions) + if err != nil { + log.FinishSpinnerWithError() + return errors.Wrap(err, "failed to render upstream") } + + renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) + if err != nil { + log.FinishSpinnerWithError() + return errors.Wrap(err, "failed to load rendered kotskinds from map") + } + log.FinishSpinner() + log.ActionWithSpinner("Creating base") io.WriteString(rewriteOptions.ReportWriter, "Creating base\n") - commonBase, helmBases, renderedKotsKindsMap, err := base.RenderUpstream(u, &renderOptions) + commonBase, helmBases, err := base.RenderUpstream(u, &renderOptions, renderedKotsKinds) if err != nil { return errors.Wrap(err, "failed to render upstream") } @@ -154,14 +165,9 @@ func Rewrite(rewriteOptions RewriteOptions) error { files = append(files, file) } - newKotsKinds, err := kotsutil.LoadKotsKindsFromPath(u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { - return errors.Wrap(err, "failed to load installation") - } - newKotsKinds.Installation.Spec.YAMLErrors = files + renderedKotsKinds.Installation.Spec.YAMLErrors = files - err = apparchive.SaveInstallation(&newKotsKinds.Installation, u.GetUpstreamDir(writeUpstreamOptions)) - if err != nil { + if err := apparchive.SaveInstallation(&renderedKotsKinds.Installation, u.GetUpstreamDir(writeUpstreamOptions)); err != nil { return errors.Wrap(err, "failed to save installation") } } @@ -203,14 +209,21 @@ func Rewrite(rewriteOptions RewriteOptions) error { return errors.Wrap(err, "failed to create new config context template builder") } - err = crypto.InitFromString(rewriteOptions.Installation.Spec.EncryptionKey) - if err != nil { + if err := crypto.InitFromString(rewriteOptions.Installation.Spec.EncryptionKey); err != nil { return errors.Wrap(err, "failed to create cipher from installation spec") } - v1Beta1HelmCharts, err := kotsutil.LoadV1Beta1HelmChartsFromPath(rewriteOptions.UpstreamPath) + identityConfig, err := upstream.LoadIdentityConfig(u.GetUpstreamDir(writeUpstreamOptions)) if err != nil { - return errors.Wrap(err, "failed to load v1beta1 helm charts") + return errors.Wrap(err, "failed to load identity config") + } + + var v1Beta1HelmCharts []*kotsv1beta1.HelmChart + if renderedKotsKinds.V1Beta1HelmCharts != nil { + for _, v1Beta1Chart := range renderedKotsKinds.V1Beta1HelmCharts.Items { + kc := v1Beta1Chart + v1Beta1HelmCharts = append(v1Beta1HelmCharts, &kc) + } } commonWriteMidstreamOptions := midstream.WriteOptions{ @@ -222,6 +235,11 @@ func Rewrite(rewriteOptions RewriteOptions) error { HTTPSProxyEnvValue: rewriteOptions.HTTPSProxyEnvValue, NoProxyEnvValue: rewriteOptions.NoProxyEnvValue, NewHelmCharts: v1Beta1HelmCharts, + License: rewriteOptions.License, + RenderedKotsKinds: renderedKotsKinds, + IdentityConfig: identityConfig, + UpstreamDir: u.GetUpstreamDir(writeUpstreamOptions), + Log: log, } // the UseHelmInstall map blocks visibility into charts and subcharts when searching for private images @@ -243,10 +261,6 @@ func Rewrite(rewriteOptions RewriteOptions) error { } } - writeMidstreamOptions := commonWriteMidstreamOptions - writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream") - writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), commonBase.Path) - processImageOptions := image.ProcessImageOptions{ AppSlug: rewriteOptions.AppSlug, Namespace: rewriteOptions.K8sNamespace, @@ -262,13 +276,13 @@ func Rewrite(rewriteOptions RewriteOptions) error { ReportWriter: rewriteOptions.ReportWriter, } - upstreamDir := u.GetUpstreamDir(writeUpstreamOptions) - identityConfig, err := upstream.LoadIdentityConfig(upstreamDir) - if err != nil { - return errors.Wrap(err, "failed to load identity config") - } + writeMidstreamOptions := commonWriteMidstreamOptions + writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream") + writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), commonBase.Path) + writeMidstreamOptions.ProcessImageOptions = processImageOptions + writeMidstreamOptions.Base = commonBase - m, err := midstream.WriteMidstream(writeMidstreamOptions, processImageOptions, commonBase, rewriteOptions.License, identityConfig, upstreamDir, log) + m, err := midstream.WriteMidstream(writeMidstreamOptions) if err != nil { return errors.Wrap(err, "failed to write common midstream") } @@ -283,16 +297,18 @@ func Rewrite(rewriteOptions RewriteOptions) error { previousUseHelmInstall := writeMidstreamOptions.UseHelmInstall[helmBase.Path] writeMidstreamOptions.UseHelmInstall[helmBase.Path] = false - writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream", helmBase.Path) - writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBase.Path) - helmBaseCopy := helmBase.DeepCopy() processImageOptionsCopy := processImageOptions processImageOptionsCopy.Namespace = helmBaseCopy.Namespace processImageOptionsCopy.CopyImages = false // don't copy images more than once - helmMidstream, err := midstream.WriteMidstream(writeMidstreamOptions, processImageOptionsCopy, helmBaseCopy, rewriteOptions.License, identityConfig, upstreamDir, log) + writeMidstreamOptions.MidstreamDir = filepath.Join(u.GetOverlaysDir(writeUpstreamOptions), "midstream", helmBaseCopy.Path) + writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBaseCopy.Path) + writeMidstreamOptions.ProcessImageOptions = processImageOptionsCopy + writeMidstreamOptions.Base = helmBaseCopy + + helmMidstream, err := midstream.WriteMidstream(writeMidstreamOptions) if err != nil { return errors.Wrapf(err, "failed to write helm midstream %s", helmBase.Path) } @@ -303,11 +319,6 @@ func Rewrite(rewriteOptions RewriteOptions) error { helmMidstreams = append(helmMidstreams, *helmMidstream) } - renderedKotsKinds, err := kotsutil.KotsKindsFromMap(renderedKotsKindsMap) - if err != nil { - return errors.Wrap(err, "failed to load rendered kotskinds from map") - } - writeV1Beta2HelmChartsOpts := apparchive.WriteV1Beta2HelmChartsOptions{ Upstream: u, WriteUpstreamOptions: writeUpstreamOptions, @@ -325,23 +336,16 @@ func Rewrite(rewriteOptions RewriteOptions) error { return errors.Wrap(err, "failed to write downstreams") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(rewriteOptions.UpstreamPath) - if err != nil { - return errors.Wrap(err, "failed to load kotskinds") - } - - err = store.GetStore().UpdateAppVersionInstallationSpec(rewriteOptions.AppID, rewriteOptions.AppSequence, kotsKinds.Installation) - if err != nil { + if err := store.GetStore().UpdateAppVersionInstallationSpec(rewriteOptions.AppID, rewriteOptions.AppSequence, renderedKotsKinds.Installation); err != nil { return errors.Wrap(err, "failed to update installation spec") } - installationBytes, err := ioutil.ReadFile(filepath.Join(u.GetUpstreamDir(writeUpstreamOptions), "userdata", "installation.yaml")) + // installation spec gets updated during the process, ensure the map has the latest version + installationBytes, err := os.ReadFile(filepath.Join(u.GetUpstreamDir(writeUpstreamOptions), "userdata", "installation.yaml")) if err != nil { return errors.Wrap(err, "failed to read installation file") } - - installationFilename := kotsutil.GenUniqueKotsKindFilename(renderedKotsKindsMap, "installation") - renderedKotsKindsMap[installationFilename] = []byte(installationBytes) + renderedKotsKindsMap["userdata/installation.yaml"] = installationBytes if err := kotsutil.WriteKotsKinds(renderedKotsKindsMap, u.GetKotsKindsDir(writeUpstreamOptions)); err != nil { return errors.Wrap(err, "failed to write kots base") @@ -352,7 +356,7 @@ func Rewrite(rewriteOptions RewriteOptions) error { OverlaysDir: u.GetOverlaysDir(writeUpstreamOptions), RenderedDir: u.GetRenderedDir(writeUpstreamOptions), Downstreams: rewriteOptions.Downstreams, - KustomizeBinPath: kotsKinds.GetKustomizeBinaryPath(), + KustomizeBinPath: renderedKotsKinds.GetKustomizeBinaryPath(), HelmDir: u.GetHelmDir(writeUpstreamOptions), Log: log, KotsKinds: renderedKotsKinds, diff --git a/pkg/secrets/sealed_secrets.go b/pkg/secrets/sealed_secrets.go index a04b94a88a..18ba22b882 100644 --- a/pkg/secrets/sealed_secrets.go +++ b/pkg/secrets/sealed_secrets.go @@ -42,7 +42,7 @@ func replaceSecretsWithSealedSecrets(archivePath string, config map[string][]byt decode := scheme.Codecs.UniversalDeserializer().Decode for _, secretPath := range secretPaths { - contents, err := ioutil.ReadFile(secretPath) + contents, err := os.ReadFile(secretPath) if err != nil { return errors.Wrap(err, "failed to read file") } diff --git a/pkg/secrets/secrets.go b/pkg/secrets/secrets.go index 4df089ed3e..35de9a65ca 100644 --- a/pkg/secrets/secrets.go +++ b/pkg/secrets/secrets.go @@ -2,7 +2,6 @@ package secrets import ( "context" - "io/ioutil" "os" "path/filepath" @@ -57,7 +56,7 @@ func findPathsWithSecrets(archiveDir string) ([]string, error) { return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } diff --git a/pkg/secrets/secrets_test.go b/pkg/secrets/secrets_test.go index 842a14ac83..4b50658a85 100644 --- a/pkg/secrets/secrets_test.go +++ b/pkg/secrets/secrets_test.go @@ -6,15 +6,16 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "io/ioutil" + "os" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/replicatedhq/kots/pkg/secrets" "github.com/replicatedhq/kots/pkg/util" - "io/ioutil" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" - "os" ) var _ = Describe("Secrets", func() { @@ -174,7 +175,7 @@ var _ = Describe("Secrets", func() { err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring("invalid yaml")) }) @@ -202,7 +203,7 @@ var _ = Describe("Secrets", func() { err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(Equal(secret)) }) @@ -230,7 +231,7 @@ var _ = Describe("Secrets", func() { err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(Equal(secret)) }) @@ -265,7 +266,7 @@ metadata: err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring(wronglyLabeledSecret)) }) @@ -382,7 +383,7 @@ metadata: err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring("kind: SealedSecret")) Expect(string(secretContents)).To(ContainSubstring(fmt.Sprintf("namespace: %s", devNamespace))) @@ -424,7 +425,7 @@ metadata: err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring("kind: SealedSecret")) Expect(string(secretContents)).To(ContainSubstring(fmt.Sprintf("namespace: %s", podNamespace))) @@ -455,7 +456,7 @@ metadata: err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring("kind: SealedSecret")) Expect(string(secretContents)).To(ContainSubstring("namespace: test-namespace")) @@ -509,7 +510,7 @@ metadata: Expect(err).ToNot(HaveOccurred()) for i := 0; i < len(secretsFiles); i++ { - secretContents, err := ioutil.ReadFile(secretsFiles[0].Name()) + secretContents, err := os.ReadFile(secretsFiles[0].Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring("kind: SealedSecret")) Expect(string(secretContents)).To(ContainSubstring("namespace: test-namespace")) @@ -556,7 +557,7 @@ metadata: err = secrets.ReplaceSecretsInPath(tmpArchiveDir, clientset) Expect(err).ToNot(HaveOccurred()) - secretContents, err := ioutil.ReadFile(tmpFile.Name()) + secretContents, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(secretContents)).To(ContainSubstring(transformedSecret1)) Expect(string(secretContents)).To(ContainSubstring(transformedSecret2)) @@ -612,7 +613,7 @@ metadata: creationTimestamp: null name: test-secret-1 namespace: test-namespace` - updatedFile, err := ioutil.ReadFile(tmpFile.Name()) + updatedFile, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(updatedFile)).To(ContainSubstring(transformedSecret)) Expect(string(updatedFile)).ToNot(ContainSubstring(originalSecretContents)) @@ -669,7 +670,7 @@ metadata: creationTimestamp: null name: test-secret-1 namespace: test-namespace` - updatedFile, err := ioutil.ReadFile(tmpFile.Name()) + updatedFile, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(updatedFile)).To(ContainSubstring(transformedSecret)) Expect(string(updatedFile)).ToNot(ContainSubstring(originalSecretContents)) @@ -730,7 +731,7 @@ metadata: undesiredExtraWhitespace := `--- ` - updatedFile, err := ioutil.ReadFile(tmpFile.Name()) + updatedFile, err := os.ReadFile(tmpFile.Name()) Expect(err).ToNot(HaveOccurred()) Expect(string(updatedFile)).To(ContainSubstring(transformedSecret)) Expect(string(updatedFile)).ToNot(ContainSubstring(originalSecretContents)) diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index c188d94bf9..afe5970f86 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" "github.com/replicatedhq/kots/pkg/k8sutil" kotsadmresources "github.com/replicatedhq/kots/pkg/kotsadm/resources" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" @@ -265,7 +266,7 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe return nil, errors.Wrap(err, "failed to find minio image") } - minioTag, err := image.GetTag(existingImage) + minioTag, err := imageutil.GetTag(existingImage) if err != nil { return nil, errors.Wrap(err, "failed to get minio image tag") } diff --git a/pkg/store/kotsstore/migrations.go b/pkg/store/kotsstore/migrations.go index 73d482a40c..1b414e3953 100644 --- a/pkg/store/kotsstore/migrations.go +++ b/pkg/store/kotsstore/migrations.go @@ -4,7 +4,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/kotsutil" @@ -97,7 +96,7 @@ func (s *KOTSStore) migrateKotsAppSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } @@ -169,7 +168,7 @@ func (s *KOTSStore) migrateKotsInstallationSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } @@ -241,7 +240,7 @@ func (s *KOTSStore) migrateSupportBundleSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } @@ -313,7 +312,7 @@ func (s *KOTSStore) migratePreflightSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } @@ -385,7 +384,7 @@ func (s *KOTSStore) migrateAnalyzerSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } @@ -457,7 +456,7 @@ func (s *KOTSStore) migrateAppSpec() error { return errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return errors.Wrap(err, "failed to load kots kinds from path") } diff --git a/pkg/store/kotsstore/version_store.go b/pkg/store/kotsstore/version_store.go index 27ed91aa4d..3350fab3bb 100644 --- a/pkg/store/kotsstore/version_store.go +++ b/pkg/store/kotsstore/version_store.go @@ -128,7 +128,7 @@ func (s *KOTSStore) IsSnapshotsSupportedForVersion(a *apptypes.App, sequence int return false, errors.Wrap(err, "failed to get app version archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { return false, errors.Wrap(err, "failed to load kots kinds from path") } @@ -353,7 +353,7 @@ func (s *KOTSStore) GetAppVersionBaseArchive(appID string, versionLabel string) return "", -1, errors.Wrapf(err, "failed to get base sequence for version %s", versionLabel) } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { return "", -1, errors.Wrap(err, "failed to create temp dir") } @@ -498,7 +498,7 @@ func (s *KOTSStore) createAppVersionStatements(appID string, baseSequence *int64 func (s *KOTSStore) upsertAppVersionStatements(appID string, sequence int64, baseSequence *int64, filesInDir string, source string, skipPreflights bool, gitops gitopstypes.DownstreamGitOps, renderer rendertypes.Renderer) ([]gorqlite.ParameterizedStatement, error) { statements := []gorqlite.ParameterizedStatement{} - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(filesInDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(filesInDir) if err != nil { return nil, errors.Wrap(err, "failed to read kots kinds") } @@ -904,7 +904,7 @@ func (s *KOTSStore) UpdateNextAppVersionDiffSummary(appID string, baseSequence i return errors.Wrap(err, "failed to get next archive dir") } - nextKotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(nextArchiveDir, "upstream")) + nextKotsKinds, err := kotsutil.LoadKotsKinds(nextArchiveDir) if err != nil { return errors.Wrap(err, "failed to read kots kinds") } diff --git a/pkg/supportbundle/supportbundle.go b/pkg/supportbundle/supportbundle.go index ba68edf0b5..e6bf5410fa 100644 --- a/pkg/supportbundle/supportbundle.go +++ b/pkg/supportbundle/supportbundle.go @@ -211,7 +211,7 @@ func getKotsKindsForApp(app *apptypes.App, sequence int64) (*kotsutil.KotsKinds, return nil, errors.Wrap(err, "failed to get current archive") } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archivePath, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archivePath) if err != nil { return nil, errors.Wrap(err, "failed to load current kotskinds") } @@ -281,7 +281,7 @@ func getAnalysisFromBundle(archivePath string) ([]byte, error) { trimmedRelPath = strings.TrimPrefix(trimmedRelPath, string(os.PathSeparator)) // extra measure to ensure no leading slashes. for example: "/analysis.json" if trimmedRelPath == "analysis.json" { - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { return errors.Wrap(err, "failed to read analysis file") } @@ -341,7 +341,7 @@ func CreateSupportBundleAnalysis(appID string, archivePath string, bundle *types return err } - kotsKinds, err := kotsutil.LoadKotsKindsFromPath(filepath.Join(archiveDir, "upstream")) + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { err = errors.Wrap(err, "failed to load kots kinds from archive") logger.Error(err) diff --git a/pkg/template/config_context.go b/pkg/template/config_context.go index 17349fab88..17fdabd9d7 100644 --- a/pkg/template/config_context.go +++ b/pkg/template/config_context.go @@ -11,6 +11,7 @@ import ( "github.com/replicatedhq/kots/pkg/docker/registry" dockerregistrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types" "github.com/replicatedhq/kots/pkg/image" + "github.com/replicatedhq/kots/pkg/imageutil" registrytypes "github.com/replicatedhq/kots/pkg/registry/types" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" corev1 "k8s.io/api/core/v1" @@ -292,7 +293,7 @@ func (ctx ConfigCtx) localImageName(imageRef string) string { Endpoint: ctx.localRegistryHost(), Namespace: ctx.localRegistryNamespace(), } - destImage, err := image.DestImage(destRegistry, imageRef) + destImage, err := imageutil.DestImage(destRegistry, imageRef) if err != nil { // TODO: log return "" diff --git a/pkg/tests/base/render_test.go b/pkg/tests/base/render_test.go index 9772a48bc2..a4534b7852 100644 --- a/pkg/tests/base/render_test.go +++ b/pkg/tests/base/render_test.go @@ -3,7 +3,6 @@ package base import ( "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -61,7 +60,7 @@ func TestRenderUpstream(t *testing.T) { require.NoError(t, err, path) } - b, err := ioutil.ReadFile(testcaseFilepath) + b, err := os.ReadFile(testcaseFilepath) require.NoError(t, err, path) var spec TestCaseSpec @@ -78,7 +77,7 @@ func TestRenderUpstream(t *testing.T) { test.WantBase = baseFromDir(t, filepath.Join(path, "base"), false) - test.WantKotsKinds, err = kotsutil.LoadKotsKindsFromPath(filepath.Join(path, "kotsKinds")) + test.WantKotsKinds, err = kotsutil.LoadKotsKinds(path) require.NoError(t, err, "kotsKinds") chartsPath := filepath.Join(path, "base", "charts") @@ -112,26 +111,9 @@ func TestRenderUpstream(t *testing.T) { template.TestingDisableKurlValues = true defer func() { template.TestingDisableKurlValues = false }() - gotBase, gotHelmBases, gotKotsKindsFiles, err := base.RenderUpstream(&tt.Upstream, &tt.RenderOptions) + gotKotsKindsFiles, err := base.RenderKotsKinds(&tt.Upstream, &tt.RenderOptions) require.NoError(t, err) - if len(tt.WantBase.Files) > 0 { - if !assert.IsEqual(tt.WantBase, gotBase) { - t.Log(diffJSON(gotBase, tt.WantBase)) - t.FailNow() - } - - if !assert.IsEqual(tt.WantBase.Files, gotBase.Files) { - for idx := range tt.WantBase.Files { - if len(gotBase.Files) > idx && gotBase.Files[idx].Path == tt.WantBase.Files[idx].Path { - t.Log("FILE", tt.WantBase.Files[idx].Path) - t.Log(diffString(string(gotBase.Files[idx].Content), string(tt.WantBase.Files[idx].Content))) - } - } - t.FailNow() - } - } - gotKotsKinds, err := kotsutil.KotsKindsFromMap(gotKotsKindsFiles) require.NoError(t, err, "kots kinds from map") @@ -149,6 +131,26 @@ func TestRenderUpstream(t *testing.T) { require.Equal(t, tt.WantKotsKinds, gotKotsKinds) + gotBase, gotHelmBases, err := base.RenderUpstream(&tt.Upstream, &tt.RenderOptions, gotKotsKinds) + require.NoError(t, err) + + if len(tt.WantBase.Files) > 0 { + if !assert.IsEqual(tt.WantBase, gotBase) { + t.Log(diffJSON(gotBase, tt.WantBase)) + t.FailNow() + } + + if !assert.IsEqual(tt.WantBase.Files, gotBase.Files) { + for idx := range tt.WantBase.Files { + if len(gotBase.Files) > idx && gotBase.Files[idx].Path == tt.WantBase.Files[idx].Path { + t.Log("FILE", tt.WantBase.Files[idx].Path) + t.Log(diffString(string(gotBase.Files[idx].Content), string(tt.WantBase.Files[idx].Content))) + } + } + t.FailNow() + } + } + // TODO: Need to test upstream with multiple Helm charts. // HACK: Also right now "no files" in WantHelmBase implies test does not include any charts. if len(tt.WantHelmBase.Files) == 0 { @@ -251,7 +253,7 @@ func upstreamFilesFromDir(t *testing.T, root string) []upstreamtypes.UpstreamFil return nil } - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) require.NoError(t, err, path) relPath, err := filepath.Rel(root, path) @@ -283,7 +285,7 @@ func baseFromDir(t *testing.T, root string, isHelm bool) base.Base { for _, path := range additionalFiles { additionalFile := filepath.Join(root, path) if _, err := os.Stat(additionalFile); err == nil { - data, err := ioutil.ReadFile(additionalFile) + data, err := os.ReadFile(additionalFile) require.NoError(t, err, additionalFile) b.AdditionalFiles = append(b.AdditionalFiles, base.BaseFile{ @@ -316,7 +318,7 @@ func baseFilesFromDir(t *testing.T, root string, isHelm bool) []base.BaseFile { return nil } - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) require.NoError(t, err, path) var fPath string fPath, err = filepath.Rel(root, path) diff --git a/pkg/tests/pull/cases/configcontext/wantResults/kotsKinds/installation.yaml b/pkg/tests/pull/cases/configcontext/wantResults/kotsKinds/userdata/installation.yaml similarity index 100% rename from pkg/tests/pull/cases/configcontext/wantResults/kotsKinds/installation.yaml rename to pkg/tests/pull/cases/configcontext/wantResults/kotsKinds/userdata/installation.yaml diff --git a/pkg/tests/pull/cases/customhostnames/wantResults/kotsKinds/installation.yaml b/pkg/tests/pull/cases/customhostnames/wantResults/kotsKinds/userdata/installation.yaml similarity index 100% rename from pkg/tests/pull/cases/customhostnames/wantResults/kotsKinds/installation.yaml rename to pkg/tests/pull/cases/customhostnames/wantResults/kotsKinds/userdata/installation.yaml diff --git a/pkg/tests/pull/cases/kotskinds/testcase.yaml b/pkg/tests/pull/cases/kotskinds/testcase.yaml new file mode 100644 index 0000000000..5b0fcd75a1 --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/testcase.yaml @@ -0,0 +1,13 @@ +Name: test templating kots kinds +PullOptions: + Namespace: app-namespace + ExcludeAdminConsole: true + IsAirgap: true + Silent: true + LocalPath: cases/kotskinds/upstream + RootDir: cases/kotskinds/results + SharedPassword: dummy-pass + RewriteImages: false + Downstreams: + - this-cluster + SkipHelmChartCheck: true \ No newline at end of file diff --git a/pkg/tests/pull/cases/kotskinds/upstream/app/configmap.yaml b/pkg/tests/pull/cases/kotskinds/upstream/app/configmap.yaml new file mode 100644 index 0000000000..51ade02102 --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/upstream/app/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +data: + key: value diff --git a/pkg/tests/pull/cases/kotskinds/upstream/kots-app.yaml b/pkg/tests/pull/cases/kotskinds/upstream/kots-app.yaml new file mode 100644 index 0000000000..45e6bb34d5 --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/upstream/kots-app.yaml @@ -0,0 +1,47 @@ +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: my-app + annotations: + kots.io/exclude: "true" +spec: + title: repl{{ ConfigOption "kots_app_app_title" }} + icon: repl{{ ConfigOption "kots_app_app_icon" }} + branding: + css: + - repl{{ ConfigOption "kots_app_branding_css_1" }} + fonts: + - fontFamily: repl{{ ConfigOption "kots_app_branding_fonts_1_font_family" }} + sources: + - repl{{ ConfigOption "kots_app_branding_fonts_1_font_sources_1" }} + applicationPorts: + - serviceName: repl{{ ConfigOption "kots_app_application_ports_1_service_name" }} + servicePort: 80 # not templatable yet + localPort: 80 # not templatable yet + applicationUrl: repl{{ ConfigOption "kots_app_application_ports_1_application_url" }} + releaseNotes: "my release notes" # not templatable yet + allowRollback: false # not templatable yet + statusInformers: + - repl{{ ConfigOption "kots_app_status_informers_1" }} + graphs: + - title: repl{{ ConfigOption "kots_app_graphs_1_title" }} + query: repl{{ ConfigOption "kots_app_graphs_1_query" }} + legend: repl{{ ConfigOption "kots_app_graphs_1_legend" }} + queries: + - query: repl{{ ConfigOption "kots_app_graphs_1_queries_1_query" }} + legend: repl{{ ConfigOption "kots_app_graphs_1_queries_1_legend" }} + - query: repl{{ ConfigOption "kots_app_graphs_1_queries_2_query" }} + legend: repl{{ ConfigOption "kots_app_graphs_1_queries_2_legend" }} + durationSeconds: 3600 # not templatable yet + yAxisFormat: repl{{ ConfigOption "kots_app_graphs_1_y_axis_format" }} + yAxisTemplate: repl{{ ConfigOption "kots_app_graphs_1_y_axis_template" }} + minKotsVersion: repl{{ ConfigOption "kots_app_min_kots_version" }} + targetKotsVersion: repl{{ ConfigOption "kots_app_target_kots_version" }} + kubectlVersion: repl{{ ConfigOption "kots_app_kubectl_version" }} + kustomizeVersion: repl{{ ConfigOption "kots_app_kustomize_version" }} + additionalImages: + - repl{{ ConfigOption "kots_app_additional_images_1" }} + additionalNamespaces: + - repl{{ ConfigOption "kots_app_additional_namespacse_1" }} + requireMinimalRBACPrivileges: false # not templatable yet + supportMinimalRBACPrivileges: false # not templatable yet diff --git a/pkg/tests/pull/cases/kotskinds/upstream/kots-config.yaml b/pkg/tests/pull/cases/kotskinds/upstream/kots-config.yaml new file mode 100644 index 0000000000..f400bb4761 --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/upstream/kots-config.yaml @@ -0,0 +1,406 @@ +apiVersion: kots.io/v1beta1 +kind: Config +metadata: + name: my-app +spec: + groups: + - name: setup + title: Setup + items: + # ---- KOTS APP ---- # + - name: kots_app_app_title + title: My Application + type: text + default: "My Application" + - name: kots_app_app_icon + title: Application Icon + type: text + default: "https://cdn.example.com/my-app-icon.png" + - name: kots_app_branding_css_1 + title: Branding CSS 1 + type: text + default: "https://cdn.example.com/my-app.css" + - name: kots_app_branding_fonts_1_font_family + title: Branding Fonts 1 Font Family + type: text + default: "Arial" + - name: kots_app_branding_fonts_1_font_sources_1 + title: Branding Fonts 1 Font Sources 1 + type: text + default: "https://cdn.example.com/my-app-font.woff" + - name: kots_app_application_ports_1_service_name + title: Application Ports 1 Service Name + type: text + default: "my-app" + - name: kots_app_application_ports_1_service_port + title: Application Ports 1 Service Port + type: text + default: "80" + - name: kots_app_application_ports_1_local_port + title: Application Ports 1 Local Port + type: text + default: "80" + - name: kots_app_application_ports_1_application_url + title: Application Ports 1 Application URL + type: text + default: "http://localhost:80" + - name: kots_app_release_notes + title: Release Notes + type: text + default: "my release notes" + - name: kots_app_allow_rollback + title: Allow Rollback + type: bool + default: "0" + - name: kots_app_status_informers_1 + title: Status Informers 1 + type: text + default: "deployment/my-app" + - name: kots_app_graphs_1_title + title: Graphs 1 Title + type: text + default: "My Graph" + - name: kots_app_graphs_1_query + title: Graphs 1 Query + type: text + default: "sum(kube_pod_container_resource_requests_cpu_cores)" + - name: kots_app_graphs_1_legend + title: Graphs 1 Legend + type: text + default: "CPU" + - name: kots_app_graphs_1_queries_1_query + title: Graphs 1 Queries 1 Query + type: text + default: "sum(kube_pod_container_resource_requests_memory_bytes)" + - name: kots_app_graphs_1_queries_1_legend + title: Graphs 1 Queries 1 Legend + type: text + default: "Memory" + - name: kots_app_graphs_1_queries_2_query + title: Graphs 1 Queries 2 Query + type: text + default: "sum(kube_pod_container_resource_requests_storage_bytes)" + - name: kots_app_graphs_1_queries_2_legend + title: Graphs 1 Queries 2 Legend + type: text + default: "Storage" + - name: kots_app_graphs_1_duration_seconds + title: Graphs 1 Duration Seconds + type: text + default: "3600" + - name: kots_app_graphs_1_y_axis_format + title: Graphs 1 Y Axis Format + type: text + default: "bytes" + - name: kots_app_graphs_1_y_axis_template + title: Graphs 1 Y Axis Template + type: text + default: "y-axis-template" + - name: kots_app_min_kots_version + title: Min Kots Version + type: text + default: "1.0.0" + - name: kots_app_target_kots_version + title: Target Kots Version + type: text + default: "2.0.0" + - name: kots_app_kubectl_version + title: Kubectl Version + type: text + default: "1.25.0" + - name: kots_app_kustomize_version + title: Kustomize Version + type: text + default: "4.7.0" + - name: kots_app_additional_images_1 + title: Additional Images 1 + type: text + default: "my-app-image:1.0.0" + - name: kots_app_additional_namespacse_1 + title: Additional Namespaces 1 + type: text + default: "my-app-namespace" + - name: kots_app_require_minimal_rbac_privileges + title: Require Minimal RBAC Privileges + type: bool + default: "0" + - name: kots_app_support_minimal_rbac_privileges + title: Support Minimal RBAC Privileges + type: bool + default: "0" + - name: kots_app_console_feature_flags_1 + title: Console Feature Flags 1 + type: text + default: "my-app-feature" + - name: kots_app_replicated_registry_domain + title: Replicated Registry Domain + type: text + default: "registry.example.com" + - name: kots_app_proxy_registry_domain + title: Proxy Registry Domain + type: text + default: "proxy.example.com" + + # ---- HelmChart v1beta1 ---- # + - name: v1beta1_helmchart_chart_release_name + title: v1beta1 Chart Release Name + type: text + default: "my-chart-release-name" + - name: v1beta1_helmchart_use_helm_install + title: v1beta1 Use Helm Install + type: bool + default: "1" + - name: v1beta1_helmchart_values + title: v1beta1 Values + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta1_helmchart_exclude + title: v1beta1 Exclude + type: bool + default: "1" + - name: v1beta1_helmchart_helm_version + title: v1beta1 Helm Version + type: text + default: "v3" + - name: v1beta1_helmchart_namespace + title: v1beta1 Namespace + type: text + default: "my-namespace" + - name: v1beta1_helmchart_optional_values_1_when + title: v1beta1 Optional Values 1 When + type: bool + default: "1" + - name: v1beta1_helmchart_optional_values_1_recursive_merge + title: v1beta1 Optional Values 1 Recursive Merge + type: bool + default: "1" + - name: v1beta1_helmchart_optional_values_1_values + title: v1beta1 Optional Values 1 Values + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta1_helmchart_builder + title: v1beta1 Builder + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta1_helmchart_weight + title: v1beta1 Weight + type: text + default: "1" + - name: v1beta1_helmchart_helm_upgrade_flags_1 + title: v1beta1 Helm Upgrade Flags 1 + type: text + default: "--timeout 60s" + + # ---- HelmChart v1beta2 ---- # + - name: v1beta2_helmchart_release_name + title: v1beta2 Release Name + type: text + default: "my-chart-release-name" + - name: v1beta2_helmchart_values + title: v1beta2 Values + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta2_helmchart_exclude + title: v1beta2 Exclude + type: bool + default: "1" + - name: v1beta2_helmchart_namespace + title: v1beta2 Namespace + type: text + default: "my-namespace" + - name: v1beta2_helmchart_optional_values_1_when + title: v1beta2 Optional Values 1 When + type: bool + default: "1" + - name: v1beta2_helmchart_optional_values_1_recursive_merge + title: v1beta2 Optional Values 1 Recursive Merge + type: bool + default: "1" + - name: v1beta2_helmchart_optional_values_1_values + title: v1beta2 Optional Values 1 Values + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta2_helmchart_builder + title: v1beta2 Builder + type: textarea + default: | + foo: bar + bar: baz + - name: v1beta2_helmchart_weight + title: v1beta2 Weight + type: text + default: "1" + - name: v1beta2_helmchart_helm_upgrade_flags_1 + title: v1beta2 Helm Upgrade Flags 1 + type: text + default: "--timeout 70s" + + # ---- SupportBundle v1beta2 ---- # + - name: v1beta2_supportbundle_collectors_exec_collector_name + title: v1beta2 Exec Collector Name + type: text + default: "my-exec-collector-name" + - name: v1beta2_supportbundle_collectors_exec_name + title: v1beta2 Exec Name + type: text + default: "my-exec-name" + - name: v1beta2_supportbundle_collectors_exec_selector_0 + title: v1beta2 Exec Selector 0 + type: text + default: "my-exec-selector-0" + - name: v1beta2_supportbundle_collectors_exec_command_0 + title: v1beta2 Exec Command 0 + type: text + default: "my-exec-command-0" + - name: v1beta2_supportbundle_collectors_exec_timeout + title: v1beta2 Exec Timeout + type: text + default: "70s" + - name: v1beta2_supportbundle_collectors_logs_collector_name + title: v1beta2 Logs Collector Name + type: text + default: "my-logs-collector-name" + - name: v1beta2_supportbundle_collectors_logs_name + title: v1beta2 Logs Name + type: text + default: "my-logs-name" + - name: v1beta2_supportbundle_collectors_logs_selector_0 + title: v1beta2 Logs Selector 0 + type: text + default: "my-logs-selector-0" + - name: v1beta2_supportbundle_analyzers_cluster_version_outcomes_fail_when + title: v1beta2 Cluster Version Outcomes Fail When + type: text + default: "< 1.16.0" + - name: v1beta2_supportbundle_analyzers_cluster_version_outcomes_fail_message + title: v1beta2 Cluster Version Outcomes Fail Message + type: text + default: "Cluster version is too old" + - name: v1beta2_supportbundle_analyzers_cluster_version_outcomes_pass_message + title: v1beta2 Cluster Version Outcomes Pass Message + type: text + default: "Cluster version is new enough" + - name: v1beta2_supportbundle_analyzers_container_runtime_outcomes_fail_when + title: v1beta2 Container Runtime Outcomes Fail When + type: text + default: "== gvisor" + - name: v1beta2_supportbundle_analyzers_container_runtime_outcomes_fail_message + title: v1beta2 Container Runtime Outcomes Fail Message + type: text + default: "Container runtime is gvisor" + - name: v1beta2_supportbundle_analyzers_container_runtime_outcomes_pass_message + title: v1beta2 Container Runtime Outcomes Pass Message + type: text + default: "Container runtime is not gvisor" + - name: v1beta2_supportbundle_analyzers_text_analyze_check_name + title: v1beta2 Text Analyze Check Name + type: text + default: "my-text-analyze-check-name" + - name: v1beta2_supportbundle_analyzers_text_analyze_exclude + title: v1beta2 Text Analyze Exclude + type: bool + default: "0" + - name: v1beta2_supportbundle_analyzers_text_analyze_ignore_if_no_files + title: v1beta2 Text Analyze Ignore If No Files + type: bool + default: "0" + - name: v1beta2_supportbundle_analyzers_text_analyze_file_name + title: v1beta2 Text Analyze File Name + type: text + default: "my-text-analyze-file-name" + - name: v1beta2_supportbundle_analyzers_text_analyze_outcomes_fail_when + title: v1beta2 Text Analyze Outcomes Fail When + type: text + default: "contains" + - name: v1beta2_supportbundle_analyzers_text_analyze_outcomes_fail_message + title: v1beta2 Text Analyze Outcomes Fail Message + type: text + default: "Text analyze failed" + - name: v1beta2_supportbundle_analyzers_text_analyze_outcomes_pass_message + title: v1beta2 Text Analyze Outcomes Pass Message + type: text + default: "Text analyze passed" + - name: v1beta2_supportbundle_analyzers_text_analyze_regex_groups + title: v1beta2 Text Analyze Regex Groups + type: text + default: "my-text-analyze-regex-groups" + - name: v1beta2_supportbundle_analyzers_node_resources_check_name + title: v1beta2 Node Resources Check Name + type: text + default: "my-node-resources-check-name" + - name: v1beta2_supportbundle_analyzers_node_resources_outcomes_fail_when + title: v1beta2 Node Resources Outcomes Fail When + type: text + default: "< 1" + - name: v1beta2_supportbundle_analyzers_node_resources_outcomes_fail_message + title: v1beta2 Node Resources Outcomes Fail Message + type: text + default: "Node resources are too low" + - name: v1beta2_supportbundle_analyzers_node_resources_outcomes_pass_message + title: v1beta2 Node Resources Outcomes Pass Message + type: text + default: "Node resources are high enough" + + # ---- Preflight v1beta2 ---- # + - name: v1beta2_preflight_collectors_run_pod_name + title: v1beta2 Run Pod Name + type: text + default: "my-run-pod-name" + - name: v1beta2_preflight_collectors_run_pod_namespace + title: v1beta2 Run Pod Namespace + type: text + default: "my-run-pod-namespace" + - name: v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_name + title: v1beta2 Pod Spec Containers 0 Name + type: text + default: "my-pod-spec-containers-0-name" + - name: v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_image + title: v1beta2 Pod Spec Containers 0 Image + type: text + default: "my-pod-spec-containers-0-image" + - name: v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_security_context_allow_privilege_escalation + title: v1beta2 Pod Spec Containers 0 Security Context Allow Privilege Escalation + type: bool + default: "1" + - name: v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_command_0 + title: v1beta2 Pod Spec Containers 0 Command 0 + type: text + default: "my-pod-spec-containers-0-command-0" + - name: v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_args_0 + title: v1beta2 Pod Spec Containers 0 Args 0 + type: text + default: "my-pod-spec-containers-0-args-0" + - name: v1beta2_preflight_analyzers_text_analyze_check_name + title: v1beta2 Text Analyze Check Name + type: text + default: "my-text-analyze-check-name" + - name: v1beta2_preflight_analyzers_text_analyze_file_name + title: v1beta2 Text Analyze File Name + type: text + default: "my-text-analyze-file-name" + - name: v1beta2_preflight_analyzers_text_analyze_regex_groups + title: v1beta2 Text Analyze Regex Groups + type: text + default: "my-text-analyze-regex-groups" + - name: v1beta2_preflight_analyzers_text_analyze_outcomes_pass_when + title: v1beta2 Text Analyze Outcomes Pass When + type: text + default: "Loss < 5" + - name: v1beta2_preflight_analyzers_text_analyze_outcomes_pass_message + title: v1beta2 Text Analyze Outcomes Pass Message + type: text + default: "my-text-analyze-outcomes-pass-message" + - name: v1beta2_preflight_analyzers_text_analyze_outcomes_fail_message + title: v1beta2 Text Analyze Outcomes Fail Message + type: text + default: "my-text-analyze-outcomes-fail-message" diff --git a/pkg/tests/pull/cases/kotskinds/upstream/kots-preflight.yaml b/pkg/tests/pull/cases/kotskinds/upstream/kots-preflight.yaml new file mode 100644 index 0000000000..dcd23f54da --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/upstream/kots-preflight.yaml @@ -0,0 +1,30 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: Preflight +metadata: + name: my-app +spec: + collectors: + - runPod: + name: repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_name" }} + namespace: repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_namespace" }} + podSpec: + containers: + - name: repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_name" }} + image: repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_image" }} + securityContext: + allowPrivilegeEscalation: repl{{ ConfigOptionEquals "v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_security_context_allow_privilege_escalation" "1" }} + command: + - repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_command_0" }} + args: + - repl{{ ConfigOption "v1beta2_preflight_collectors_run_pod_pod_spec_containers_0_args_0" }} + analyzers: + - textAnalyze: + checkName: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_check_name" }} + fileName: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_file_name" }} + regexGroups: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_regex_groups" }} + outcomes: + - pass: + when: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_outcomes_pass_when" }} + message: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_outcomes_pass_message" }} + - fail: + message: repl{{ ConfigOption "v1beta2_preflight_analyzers_text_analyze_outcomes_fail_message" }} \ No newline at end of file diff --git a/pkg/tests/pull/cases/kotskinds/upstream/kots-supportbundle.yaml b/pkg/tests/pull/cases/kotskinds/upstream/kots-supportbundle.yaml new file mode 100644 index 0000000000..bc3e1147b4 --- /dev/null +++ b/pkg/tests/pull/cases/kotskinds/upstream/kots-supportbundle.yaml @@ -0,0 +1,55 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: my-app +spec: + collectors: + - exec: + collectorName: repl{{ ConfigOption "v1beta2_supportbundle_collectors_exec_collector_name" }} + name: repl{{ ConfigOption "v1beta2_supportbundle_collectors_exec_name" }} + selector: + - repl{{ ConfigOption "v1beta2_supportbundle_collectors_exec_selector_0" }} + command: + - repl{{ ConfigOption "v1beta2_supportbundle_collectors_exec_command_0" }} + timeout: repl{{ ConfigOption "v1beta2_supportbundle_collectors_exec_timeout" }} + - logs: + collectorName: repl{{ ConfigOption "v1beta2_supportbundle_collectors_logs_collector_name" }} + name: repl{{ ConfigOption "v1beta2_supportbundle_collectors_logs_name" }} + selector: + - repl{{ ConfigOption "v1beta2_supportbundle_collectors_logs_selector_0" }} + + analyzers: + - clusterVersion: + outcomes: + - fail: + when: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_cluster_version_outcomes_fail_when" }} + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_cluster_version_outcomes_fail_message" }} + - pass: + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_cluster_version_outcomes_pass_message" }} + - containerRuntime: + outcomes: + - fail: + when: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_container_runtime_outcomes_fail_when" }} + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_container_runtime_outcomes_fail_message" }} + - pass: + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_container_runtime_outcomes_pass_message" }} + - textAnalyze: + checkName: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_check_name" }} + exclude: repl{{ ConfigOptionEquals "v1beta2_supportbundle_analyzers_text_analyze_exclude" "1" }} + ignoreIfNoFiles: repl{{ ConfigOptionEquals "v1beta2_supportbundle_analyzers_text_analyze_ignore_if_no_files" "1" }} + fileName: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_file_name" }} + outcomes: + - fail: + when: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_outcomes_fail_when" }} + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_outcomes_fail_message" }} + - pass: + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_outcomes_pass_message" }} + regexGroups: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_text_analyze_regex_groups" }} + - nodeResources: + checkName: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_node_resources_check_name" }} + outcomes: + - fail: + when: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_node_resources_outcomes_fail_when" }} + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_node_resources_outcomes_fail_message" }} + - pass: + message: repl{{ ConfigOption "v1beta2_supportbundle_analyzers_node_resources_outcomes_pass_message" }} diff --git a/pkg/tests/pull/cases/kotskinds/upstream/my-chart-0.1.0.tgz b/pkg/tests/pull/cases/kotskinds/upstream/my-chart-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..dd79ddc8a78391a7e96ba81e4696f8382c47b46d GIT binary patch literal 407 zcmV;I0cidoiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PL2{YU3~vfPL0e%nQUB$;Mf9TXI=g3OV&YwUt!V`axDgL+QI0 zyCfwhB$T8H1$~DIjWV_e&nL@lC-pOhIGy*u`IbzZT^mHisw{^Q5yvC)naFrq7NV+n zfmMcF%!J4Qul^)QSa*>^h8N#!T-+~#yna-n)6PoR2u9xTr{71``3tv&v;OmPC8j2(tz@gw@-NA-Gz}x%1 zYe{^cQ~rC`T;)J;&VNykzyB4lc=G=i9QA+Unp5U`*Ram3x?ifDjijv#T|b)~*(5%f z832Gbv{Dl6O50b-b5RUY??N21`}}>v!i6Z|PdX