Skip to content

Commit

Permalink
feat(cmd): log test errors (#272)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shellcromancer
shellcromancer authored Nov 4, 2024
1 parent cf595c4 commit 917e29f
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 56 deletions.
45 changes: 45 additions & 0 deletions cmd/substation/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package main

import (
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"

"github.com/google/go-jsonnet"
Expand All @@ -17,6 +19,10 @@ var rootCmd = &cobra.Command{
Long: "'substation' is a tool for managing Substation configurations.",
}

// transformRe captures the transform ID from a Substation error message.
// Example: `transform 324f1035-10a51b9a: object_target_key: missing required option` -> `324f1035-10a51b9a`
var transformRe = regexp.MustCompile(`transform ([a-f0-9-]+):`)

const (
// confStdout is the default configuration used by
// read-like commands. It prints any results (<= 100MB)
Expand Down Expand Up @@ -156,6 +162,45 @@ func pathVars(p string) (string, string) {
return dir, fn
}

// transformErrStr returns a formatted string for transform errors.
//
// If the error is not a transform error, then the error message
// is returned as is.
func transformErrStr(err error, arg string, cfg customConfig) string {
r := transformRe.FindStringSubmatch(err.Error())

// Cannot determine which transform failed. This should almost
// never happen, unless something has modified the configuration
// after it was compiled by Jsonnet.
if len(r) == 0 {
// Substation uses the transform name as a static transform ID.
//
// Example: `vet.json: transform hash_sha256: object_target_key: missing required option``
return fmt.Sprintf("%s: %v\n", arg, err)
}

tfID := r[1] // The transform ID (e.g., `324f1035-10a51b9a`).

// Prioritize returning test errors.
for _, test := range cfg.Tests {
for idx, tf := range test.Transforms {
if tf.Settings["id"] == tfID {
// Example: `vet.json:3 transform 324f1035-10a51b9a: object_target_key: missing required option``
return fmt.Sprintf("%s:%d %v\n", arg, idx+1, err) + fmt.Sprintf(" %s\n\n", tf) // The line number is 1-based.
}
}
}

for idx, tf := range cfg.Config.Transforms {
if tf.Settings["id"] == tfID {
return fmt.Sprintf("%s:%d %v\n", arg, idx+1, err) + fmt.Sprintf(" %s\n\n", tf)
}
}

// This happens if the input is not a transform error.
return fmt.Sprintf("%s: %v\n", arg, err)
}

func main() {
err := rootCmd.Execute()
if err != nil {
Expand Down
38 changes: 30 additions & 8 deletions cmd/substation/playground.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,14 +175,16 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var output strings.Builder

arg := "config.jsonnet" // Used for consistency with the test and vet commands.

if len(cfg.Transforms) == 0 {
output.WriteString("?\t[config error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
sendJSONResponse(w, map[string]string{"output": output.String()})
return
}

if len(cfg.Tests) == 0 {
output.WriteString("?\t[no tests]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[no tests]\n", arg))
sendJSONResponse(w, map[string]string{"output": output.String()})
return
}
Expand All @@ -193,7 +195,9 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
for _, test := range cfg.Tests {
cnd, err := condition.New(ctx, test.Condition)
if err != nil {
output.WriteString("?\t[test error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}
Expand All @@ -202,28 +206,44 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
Transforms: test.Transforms,
})
if err != nil {
output.WriteString("?\t[test error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s", arg, test.Name), cfg))
} else {
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
}

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}

tester, err := substation.New(ctx, cfg.Config)
if err != nil {
output.WriteString("?\t[config error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
output.WriteString(transformErrStr(err, arg, cfg))
} else {
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
}

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}

sMsgs, err := setup.Transform(ctx, message.New().AsControl())
if err != nil {
output.WriteString("?\t[test error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}

tMsgs, err := tester.Transform(ctx, sMsgs...)
if err != nil {
output.WriteString("?\t[config error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}
Expand All @@ -236,7 +256,9 @@ func handleTest(w http.ResponseWriter, r *http.Request) {

ok, err := cnd.Condition(ctx, msg)
if err != nil {
output.WriteString("?\t[test error]\n")
output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))

sendJSONResponse(w, map[string]string{"output": output.String()})
return
}
Expand Down
51 changes: 30 additions & 21 deletions cmd/substation/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,16 @@ import (
"github.com/brexhq/substation/v2/message"
)

type testConfig struct {
Name string `json:"name"`
Transforms []config.Config `json:"transforms"`
Condition config.Config `json:"condition"`
}

// customConfig wraps the Substation config with support for tests.
type customConfig struct {
substation.Config

Tests []struct {
Name string `json:"name"`
Transforms []config.Config `json:"transforms"`
Condition config.Config `json:"condition"`
} `json:"tests"`
Tests []testConfig `json:"tests"`
}

func init() {
Expand Down Expand Up @@ -197,9 +198,9 @@ func testFile(arg string, extVars map[string]string) error {
// If the Jsonnet cannot compile, then the file is invalid.
mem, err := compileFile(arg, extVars)
if err != nil {
fmt.Printf("?\t%s\t[config error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))

//nolint:nilerr // errors should not disrupt the test.
return nil
}

Expand All @@ -220,7 +221,7 @@ func testFile(arg string, extVars map[string]string) error {

start := time.Now()

// These configurations are not valid.
// These are not valid Substation configs and must be ignored.
if len(cfg.Transforms) == 0 {
return nil
}
Expand All @@ -238,9 +239,9 @@ func testFile(arg string, extVars map[string]string) error {
// cnd asserts that the test is successful.
cnd, err := condition.New(ctx, test.Condition)
if err != nil {
fmt.Printf("FAIL\t%s\t[test error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))

//nolint:nilerr // errors should not disrupt the test.
return nil
}

Expand All @@ -249,9 +250,13 @@ func testFile(arg string, extVars map[string]string) error {
Transforms: test.Transforms,
})
if err != nil {
fmt.Printf("?\t%s\t[test error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s", arg, test.Name), cfg))
} else {
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
}

//nolint:nilerr // errors should not disrupt the test.
return nil
}

Expand All @@ -260,25 +265,29 @@ func testFile(arg string, extVars map[string]string) error {
// that there is no state shared between tests.
tester, err := substation.New(ctx, cfg.Config)
if err != nil {
fmt.Printf("?\t%s\t[config error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))
} else {
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
}

//nolint:nilerr // errors should not disrupt the test.
return nil
}

sMsgs, err := setup.Transform(ctx, message.New().AsControl())
if err != nil {
fmt.Printf("?\t%s\t[test error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))

//nolint:nilerr // errors should not disrupt the test.
return nil
}

tMsgs, err := tester.Transform(ctx, sMsgs...)
if err != nil {
fmt.Printf("?\t%s\t[config error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))

//nolint:nilerr // errors should not disrupt the test.
return nil
}

Expand All @@ -290,9 +299,9 @@ func testFile(arg string, extVars map[string]string) error {

ok, err := cnd.Condition(ctx, msg)
if err != nil {
fmt.Printf("?\t%s\t[test error]\n", arg)
fmt.Printf("?\t%s\t[error]\n", arg)
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))

//nolint:nilerr // errors should not disrupt the test.
return nil
}

Expand Down
32 changes: 5 additions & 27 deletions cmd/substation/vet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"

"github.com/spf13/cobra"

Expand All @@ -20,10 +19,6 @@ func init() {
vetCmd.PersistentFlags().SortFlags = false
}

// vetTransformRe captures the transform ID from a Substation error message.
// Example: `transform 324f1035-10a51b9a: object_target_key: missing required option` -> `324f1035-10a51b9a`
var vetTransformRe = regexp.MustCompile(`transform ([a-f0-9-]+):`)

var vetCmd = &cobra.Command{
Use: "vet [path]",
Short: "report config errors",
Expand Down Expand Up @@ -127,30 +122,13 @@ func vetFile(arg string, extVars map[string]string) error {

ctx := context.Background() // This doesn't need to be canceled.
if _, err := substation.New(ctx, cfg.Config); err != nil {
r := vetTransformRe.FindStringSubmatch(err.Error())

// Cannot determine which transform failed. This should almost
// never happen, unless something has modified the configuration
// after it was compiled by Jsonnet.
if len(r) == 0 {
// Substation uses the transform name as a static transform ID.
//
// Example: `vet.json: transform hash_sha256: object_target_key: missing required option``
fmt.Printf("%s: %v\n", arg, err)

return nil
if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))
} else {
fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
}

tfID := r[1] // The transform ID (e.g., `324f1035-10a51b9a`).
for idx, tf := range cfg.Config.Transforms {
if tf.Settings["id"] == tfID {
// Example: `vet.jsonnet:3 transform 324f1035-10a51b9a: object_target_key: missing required option``
fmt.Printf("%s:%d %v\n", arg, idx+1, err) // The line number is 1-based.
fmt.Printf("\n %s\n\n", tf)

return nil
}
}
return nil
}

// No errors were found.
Expand Down

0 comments on commit 917e29f

Please sign in to comment.