diff --git a/pkg/handlers/builders_lint.go b/pkg/handlers/builders_lint.go index ab59967..44c5d63 100644 --- a/pkg/handlers/builders_lint.go +++ b/pkg/handlers/builders_lint.go @@ -28,6 +28,8 @@ type LintBuildersReleaseResponse struct { func LintBuildersRelease(c *gin.Context) { log.Infof("Received builders lint request with content-length=%s, content-type=%s, client-ip=%s", c.GetHeader("content-length"), c.ContentType(), c.ClientIP()) + ctx := c.Request.Context() + specFiles := kots.SpecFiles{} numChartsRendered := 0 @@ -51,7 +53,7 @@ func LintBuildersRelease(c *gin.Context) { } log.Debugf("adding files for chart %s", header.Name) - files, err := kots.GetFilesFromChartReader(tarReader) + files, err := kots.GetFilesFromChartReader(ctx, tarReader) if err != nil { log.Infof("failed to get files from chart %s: %v", header.Name, err) lintExpressions = append(lintExpressions, kots.LintExpression{ @@ -63,11 +65,14 @@ func LintBuildersRelease(c *gin.Context) { continue } + troubleshootSpecs := kots.GetEmbeddedTroubleshootSpecs(ctx, files) + numChartsRendered += 1 specFiles = append(specFiles, files...) + specFiles = append(specFiles, troubleshootSpecs...) } } else if c.ContentType() == "application/gzip" { - files, err := kots.GetFilesFromChartReader(c.Request.Body) + files, err := kots.GetFilesFromChartReader(ctx, c.Request.Body) if err != nil { log.Infof("failed to get files from request: %v", err) lintExpressions = append(lintExpressions, kots.LintExpression{ @@ -76,8 +81,11 @@ func LintBuildersRelease(c *gin.Context) { Message: err.Error(), }) } else { + troubleshootSpecs := kots.GetEmbeddedTroubleshootSpecs(ctx, files) + numChartsRendered += 1 specFiles = append(specFiles, files...) + specFiles = append(specFiles, troubleshootSpecs...) } } else { c.JSON(http.StatusBadRequest, gin.H{"error": "content type must be application/gzip or application/tar"}) @@ -86,7 +94,7 @@ func LintBuildersRelease(c *gin.Context) { // Only lint if at least one chart was rendered, otherwise we get missing spec warnings/errors if numChartsRendered > 0 { - lint, err := kots.LintBuilders(c.Request.Context(), specFiles) + lint, err := kots.LintBuilders(ctx, specFiles) if err != nil { log.Errorf("failed to lint builders charts: %v", err) c.AbortWithError(http.StatusInternalServerError, err) diff --git a/pkg/handlers/lint.go b/pkg/handlers/lint.go index ee4b59b..e9a0231 100644 --- a/pkg/handlers/lint.go +++ b/pkg/handlers/lint.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "github.com/gin-gonic/gin" @@ -38,8 +38,10 @@ type LintReleaseResponse struct { func LintRelease(c *gin.Context) { log.Infof("Received lint request with content-length=%s, content-type=%s, client-ip=%s", c.GetHeader("content-length"), c.ContentType(), c.ClientIP()) + ctx := c.Request.Context() + // read before binding to check if body is a tar stream - data, err := ioutil.ReadAll(c.Request.Body) + data, err := io.ReadAll(c.Request.Body) c.Request.Body.Close() if err != nil { log.Errorf("failed to read request body: %v", err) @@ -58,7 +60,7 @@ func LintRelease(c *gin.Context) { specFiles = f } else { // restore request body to its original state to be able to bind it - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + c.Request.Body = io.NopCloser(bytes.NewBuffer(data)) var request LintReleaseParameters if err := c.Bind(&request.Body); err != nil { @@ -74,7 +76,7 @@ func LintRelease(c *gin.Context) { } } - lintExpressions, isComplete, err := kots.LintSpecFiles(specFiles) + lintExpressions, isComplete, err := kots.LintSpecFiles(ctx, specFiles) if err != nil { fmt.Printf("failed to lint spec files: %v", err) c.AbortWithError(http.StatusInternalServerError, err) diff --git a/pkg/kots/builders_lint_test.go b/pkg/kots/builders_lint_test.go index db23bae..eb90127 100644 --- a/pkg/kots/builders_lint_test.go +++ b/pkg/kots/builders_lint_test.go @@ -128,7 +128,7 @@ func TestLintBuilders(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotFiles, err := GetFilesFromChartReader(tt.chartReader()) + gotFiles, err := GetFilesFromChartReader(context.Background(), tt.chartReader()) if tt.isValidChart { assert.NoError(t, err) } else { diff --git a/pkg/kots/helm.go b/pkg/kots/helm.go index 7bdf076..f73c36e 100644 --- a/pkg/kots/helm.go +++ b/pkg/kots/helm.go @@ -1,6 +1,7 @@ package kots import ( + "context" _ "embed" "io" "path/filepath" @@ -14,7 +15,7 @@ import ( // GetFilesFromChartReader will render chart templates and return the resulting files // This function will ignore missing required values. // This function will also not validate value types. -func GetFilesFromChartReader(r io.Reader) (SpecFiles, error) { +func GetFilesFromChartReader(ctx context.Context, r io.Reader) (SpecFiles, error) { chart, err := loader.LoadArchive(r) if err != nil { return nil, errors.Wrap(err, "load chart archive") diff --git a/pkg/kots/lint.go b/pkg/kots/lint.go index e16d0e0..5bb40c7 100644 --- a/pkg/kots/lint.go +++ b/pkg/kots/lint.go @@ -135,7 +135,7 @@ func InitOPALinting() error { return nil } -func LintSpecFiles(specFiles SpecFiles) ([]LintExpression, bool, error) { +func LintSpecFiles(ctx context.Context, specFiles SpecFiles) ([]LintExpression, bool, error) { unnestedFiles := specFiles.unnest() tarGzFiles := SpecFiles{} @@ -149,6 +149,34 @@ func LintSpecFiles(specFiles SpecFiles) ([]LintExpression, bool, error) { } } + // Extract troubleshoot specs from ConfigMaps and Secrets, which may also be in Helm charts + troubleshootSpecs := GetEmbeddedTroubleshootSpecs(ctx, yamlFiles) + for _, tsSpec := range troubleshootSpecs { + yamlFiles = append(yamlFiles, SpecFile{ + Name: tsSpec.Name, + Path: tsSpec.Path, + Content: tsSpec.Content, + DocIndex: len(yamlFiles), + }) + } + + for _, tarGtarGzFile := range tarGzFiles { + files, err := GetFilesFromChartReader(ctx, bytes.NewReader([]byte(tarGtarGzFile.Content))) + if err != nil { + log.Debugf("failed to get files from tgz file %s: %v", tarGtarGzFile.Name, err) + continue + } + troubleshootSpecs := GetEmbeddedTroubleshootSpecs(ctx, files) + for _, tsSpec := range troubleshootSpecs { + yamlFiles = append(yamlFiles, SpecFile{ + Name: tsSpec.Name, + Path: tsSpec.Path, + Content: tsSpec.Content, + DocIndex: len(yamlFiles), + }) + } + } + // if there are yaml errors, end early there yamlLintExpressions := lintIsValidYAML(yamlFiles) if lintExpressionsHaveErrors(yamlLintExpressions) { diff --git a/pkg/kots/troubleshoot.go b/pkg/kots/troubleshoot.go new file mode 100644 index 0000000..fe682e9 --- /dev/null +++ b/pkg/kots/troubleshoot.go @@ -0,0 +1,117 @@ +package kots + +import ( + "context" + _ "embed" + "fmt" + "path" + "strings" + + troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme" + "github.com/replicatedhq/troubleshoot/pkg/constants" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +var decoder runtime.Decoder + +func init() { + _ = v1.AddToScheme(troubleshootscheme.Scheme) // for secrets and configmaps + decoder = troubleshootscheme.Codecs.UniversalDeserializer() +} + +func GetEmbeddedTroubleshootSpecs(ctx context.Context, specsFiles SpecFiles) SpecFiles { + tsSpecs := SpecFiles{} + + for _, specFile := range specsFiles { + fmt.Printf("++++looking for specs in %s / %s\n", specFile.Path, specFile.Name) + troubleshootSpecs := findTroubleshootSpecs(ctx, specFile.Content) + for _, tsSpec := range troubleshootSpecs { + tsSpecs = append(tsSpecs, SpecFile{ + Name: path.Join(specFile.Name, tsSpec.Name), + Path: specFile.Name, + Content: tsSpec.Content, + }) + } + } + + return tsSpecs +} + +// Extract troubleshoot specs from ConfigMap and Secret specs +func findTroubleshootSpecs(ctx context.Context, fileData string) SpecFiles { + tsSpecs := SpecFiles{} + + srcDocs := strings.Split(fileData, "\n---\n") + for _, srcDoc := range srcDocs { + obj, _, err := decoder.Decode([]byte(srcDoc), nil, nil) + if err != nil { + log.Debugf("failed to decode raw spec: %s", srcDoc) + continue + } + + switch v := obj.(type) { + case *v1.ConfigMap: + specs := getSpecFromConfigMap(v, fmt.Sprintf("%s-", v.Name)) + tsSpecs = append(tsSpecs, specs...) + case *v1.Secret: + specs := getSpecFromSecret(v, fmt.Sprintf("%s-", v.Name)) + tsSpecs = append(tsSpecs, specs...) + } + } + + return tsSpecs +} + +func getSpecFromConfigMap(cm *v1.ConfigMap, namePrefix string) SpecFiles { + possibleKeys := []string{ + constants.SupportBundleKey, + constants.RedactorKey, + constants.PreflightKey, + constants.PreflightKey2, + } + + specs := SpecFiles{} + for _, key := range possibleKeys { + str, ok := cm.Data[key] + if ok { + specs = append(specs, SpecFile{ + Name: namePrefix + key, + Content: str, + }) + } + } + + return specs +} + +func getSpecFromSecret(secret *v1.Secret, namePrefix string) SpecFiles { + possibleKeys := []string{ + constants.SupportBundleKey, + constants.RedactorKey, + constants.PreflightKey, + constants.PreflightKey2, + } + + specs := SpecFiles{} + for _, key := range possibleKeys { + data, ok := secret.Data[key] + if ok { + specs = append(specs, SpecFile{ + Name: namePrefix + key, + Content: string(data), + }) + } + + str, ok := secret.StringData[key] + if ok { + specs = append(specs, SpecFile{ + Name: namePrefix + key, + Content: str, + }) + } + } + + return specs +}