Skip to content

Commit

Permalink
Extract troubleshoot specs from secrets/config maps
Browse files Browse the repository at this point in the history
  • Loading branch information
divolgin committed Mar 22, 2024
1 parent 622bcae commit b64f017
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 10 deletions.
14 changes: 11 additions & 3 deletions pkg/handlers/builders_lint.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ type LintBuildersReleaseResponse struct {
func LintBuildersRelease(c *gin.Context) {
log.Infof("Received builders lint request with content-length=%s, content-type=%s, client-ip=%s", c.GetHeader("content-length"), c.ContentType(), c.ClientIP())

ctx := c.Request.Context()

specFiles := kots.SpecFiles{}
numChartsRendered := 0

Expand All @@ -51,7 +53,7 @@ func LintBuildersRelease(c *gin.Context) {
}

log.Debugf("adding files for chart %s", header.Name)
files, err := kots.GetFilesFromChartReader(tarReader)
files, err := kots.GetFilesFromChartReader(ctx, tarReader)
if err != nil {
log.Infof("failed to get files from chart %s: %v", header.Name, err)
lintExpressions = append(lintExpressions, kots.LintExpression{
Expand All @@ -63,11 +65,14 @@ func LintBuildersRelease(c *gin.Context) {
continue
}

troubleshootSpecs := kots.GetEmbeddedTroubleshootSpecs(ctx, files)

numChartsRendered += 1
specFiles = append(specFiles, files...)
specFiles = append(specFiles, troubleshootSpecs...)
}
} else if c.ContentType() == "application/gzip" {
files, err := kots.GetFilesFromChartReader(c.Request.Body)
files, err := kots.GetFilesFromChartReader(ctx, c.Request.Body)
if err != nil {
log.Infof("failed to get files from request: %v", err)
lintExpressions = append(lintExpressions, kots.LintExpression{
Expand All @@ -76,8 +81,11 @@ func LintBuildersRelease(c *gin.Context) {
Message: err.Error(),
})
} else {
troubleshootSpecs := kots.GetEmbeddedTroubleshootSpecs(ctx, files)

numChartsRendered += 1
specFiles = append(specFiles, files...)
specFiles = append(specFiles, troubleshootSpecs...)
}
} else {
c.JSON(http.StatusBadRequest, gin.H{"error": "content type must be application/gzip or application/tar"})
Expand All @@ -86,7 +94,7 @@ func LintBuildersRelease(c *gin.Context) {

// Only lint if at least one chart was rendered, otherwise we get missing spec warnings/errors
if numChartsRendered > 0 {
lint, err := kots.LintBuilders(c.Request.Context(), specFiles)
lint, err := kots.LintBuilders(ctx, specFiles)
if err != nil {
log.Errorf("failed to lint builders charts: %v", err)
c.AbortWithError(http.StatusInternalServerError, err)
Expand Down
10 changes: 6 additions & 4 deletions pkg/handlers/lint.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"

"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -38,8 +38,10 @@ type LintReleaseResponse struct {
func LintRelease(c *gin.Context) {
log.Infof("Received lint request with content-length=%s, content-type=%s, client-ip=%s", c.GetHeader("content-length"), c.ContentType(), c.ClientIP())

ctx := c.Request.Context()

// read before binding to check if body is a tar stream
data, err := ioutil.ReadAll(c.Request.Body)
data, err := io.ReadAll(c.Request.Body)
c.Request.Body.Close()
if err != nil {
log.Errorf("failed to read request body: %v", err)
Expand All @@ -58,7 +60,7 @@ func LintRelease(c *gin.Context) {
specFiles = f
} else {
// restore request body to its original state to be able to bind it
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data))
c.Request.Body = io.NopCloser(bytes.NewBuffer(data))

var request LintReleaseParameters
if err := c.Bind(&request.Body); err != nil {
Expand All @@ -74,7 +76,7 @@ func LintRelease(c *gin.Context) {
}
}

lintExpressions, isComplete, err := kots.LintSpecFiles(specFiles)
lintExpressions, isComplete, err := kots.LintSpecFiles(ctx, specFiles)
if err != nil {
fmt.Printf("failed to lint spec files: %v", err)
c.AbortWithError(http.StatusInternalServerError, err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/kots/builders_lint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func TestLintBuilders(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotFiles, err := GetFilesFromChartReader(tt.chartReader())
gotFiles, err := GetFilesFromChartReader(context.Background(), tt.chartReader())
if tt.isValidChart {
assert.NoError(t, err)
} else {
Expand Down
3 changes: 2 additions & 1 deletion pkg/kots/helm.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package kots

import (
"context"
_ "embed"
"io"
"path/filepath"
Expand All @@ -14,7 +15,7 @@ import (
// GetFilesFromChartReader will render chart templates and return the resulting files
// This function will ignore missing required values.
// This function will also not validate value types.
func GetFilesFromChartReader(r io.Reader) (SpecFiles, error) {
func GetFilesFromChartReader(ctx context.Context, r io.Reader) (SpecFiles, error) {
chart, err := loader.LoadArchive(r)
if err != nil {
return nil, errors.Wrap(err, "load chart archive")
Expand Down
30 changes: 29 additions & 1 deletion pkg/kots/lint.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func InitOPALinting() error {
return nil
}

func LintSpecFiles(specFiles SpecFiles) ([]LintExpression, bool, error) {
func LintSpecFiles(ctx context.Context, specFiles SpecFiles) ([]LintExpression, bool, error) {
unnestedFiles := specFiles.unnest()

tarGzFiles := SpecFiles{}
Expand All @@ -149,6 +149,34 @@ func LintSpecFiles(specFiles SpecFiles) ([]LintExpression, bool, error) {
}
}

// Extract troubleshoot specs from ConfigMaps and Secrets, which may also be in Helm charts
troubleshootSpecs := GetEmbeddedTroubleshootSpecs(ctx, yamlFiles)
for _, tsSpec := range troubleshootSpecs {
yamlFiles = append(yamlFiles, SpecFile{
Name: tsSpec.Name,
Path: tsSpec.Path,
Content: tsSpec.Content,
DocIndex: len(yamlFiles),
})
}

for _, tarGtarGzFile := range tarGzFiles {
files, err := GetFilesFromChartReader(ctx, bytes.NewReader([]byte(tarGtarGzFile.Content)))
if err != nil {
log.Debugf("failed to get files from tgz file %s: %v", tarGtarGzFile.Name, err)
continue
}
troubleshootSpecs := GetEmbeddedTroubleshootSpecs(ctx, files)
for _, tsSpec := range troubleshootSpecs {
yamlFiles = append(yamlFiles, SpecFile{
Name: tsSpec.Name,
Path: tsSpec.Path,
Content: tsSpec.Content,
DocIndex: len(yamlFiles),
})
}
}

// if there are yaml errors, end early there
yamlLintExpressions := lintIsValidYAML(yamlFiles)
if lintExpressionsHaveErrors(yamlLintExpressions) {
Expand Down
117 changes: 117 additions & 0 deletions pkg/kots/troubleshoot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package kots

import (
"context"
_ "embed"
"fmt"
"path"
"strings"

troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
"github.com/replicatedhq/troubleshoot/pkg/constants"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
)

var decoder runtime.Decoder

func init() {
_ = v1.AddToScheme(troubleshootscheme.Scheme) // for secrets and configmaps
decoder = troubleshootscheme.Codecs.UniversalDeserializer()
}

func GetEmbeddedTroubleshootSpecs(ctx context.Context, specsFiles SpecFiles) SpecFiles {
tsSpecs := SpecFiles{}

for _, specFile := range specsFiles {
fmt.Printf("++++looking for specs in %s / %s\n", specFile.Path, specFile.Name)
troubleshootSpecs := findTroubleshootSpecs(ctx, specFile.Content)
for _, tsSpec := range troubleshootSpecs {
tsSpecs = append(tsSpecs, SpecFile{
Name: path.Join(specFile.Name, tsSpec.Name),
Path: specFile.Name,
Content: tsSpec.Content,
})
}
}

return tsSpecs
}

// Extract troubleshoot specs from ConfigMap and Secret specs
func findTroubleshootSpecs(ctx context.Context, fileData string) SpecFiles {
tsSpecs := SpecFiles{}

srcDocs := strings.Split(fileData, "\n---\n")
for _, srcDoc := range srcDocs {
obj, _, err := decoder.Decode([]byte(srcDoc), nil, nil)
if err != nil {
log.Debugf("failed to decode raw spec: %s", srcDoc)
continue
}

switch v := obj.(type) {
case *v1.ConfigMap:
specs := getSpecFromConfigMap(v, fmt.Sprintf("%s-", v.Name))
tsSpecs = append(tsSpecs, specs...)
case *v1.Secret:
specs := getSpecFromSecret(v, fmt.Sprintf("%s-", v.Name))
tsSpecs = append(tsSpecs, specs...)
}
}

return tsSpecs
}

func getSpecFromConfigMap(cm *v1.ConfigMap, namePrefix string) SpecFiles {
possibleKeys := []string{
constants.SupportBundleKey,
constants.RedactorKey,
constants.PreflightKey,
constants.PreflightKey2,
}

specs := SpecFiles{}
for _, key := range possibleKeys {
str, ok := cm.Data[key]
if ok {
specs = append(specs, SpecFile{
Name: namePrefix + key,
Content: str,
})
}
}

return specs
}

func getSpecFromSecret(secret *v1.Secret, namePrefix string) SpecFiles {
possibleKeys := []string{
constants.SupportBundleKey,
constants.RedactorKey,
constants.PreflightKey,
constants.PreflightKey2,
}

specs := SpecFiles{}
for _, key := range possibleKeys {
data, ok := secret.Data[key]
if ok {
specs = append(specs, SpecFile{
Name: namePrefix + key,
Content: string(data),
})
}

str, ok := secret.StringData[key]
if ok {
specs = append(specs, SpecFile{
Name: namePrefix + key,
Content: str,
})
}
}

return specs
}

0 comments on commit b64f017

Please sign in to comment.