diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index e6cc1893..b43688c3 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -16,6 +16,13 @@ const ( MatchWildcard MatchType = "Wildcard" ) +type Strategy string + +const ( + StrategyAnywhere Strategy = "Anywhere" + StrategyExact Strategy = "Exact" +) + // Create embedded struct to implement custom DeepCopyInto method type RestConfig struct { RC *rest.Config @@ -133,7 +140,7 @@ type TestStep struct { // Useful to reuse a number of applies across tests / test steps. // all relative paths are relative to the folder the TestStep is defined in. Apply []Apply `json:"apply,omitempty"` - Assert []string `json:"assert,omitempty"` + Assert []Assert `json:"assert,omitempty"` Error []string `json:"error,omitempty"` // Objects to delete at the beginning of the test step. @@ -152,6 +159,25 @@ type TestStep struct { Kubeconfig string `json:"kubeconfig,omitempty"` } +type Assert struct { + // File specifies the relative or full path to the YAML containing the expected content. + File string `json:"file"` + ShouldFail bool `json:"shouldFail,omitempty"` + Options *Options `json:"options,omitempty"` +} + +type Options struct { + AssertArray []AssertArray `json:"arrays,omitempty"` +} + +// AssertArray specifies conditions for verifying content within a YAML against a Kubernetes resource. +type AssertArray struct { + // Path indicates the location within the YAML file to extract data for verification. + Path string `json:"path"` + // Strategy defines how the extracted data should be compared against the Kubernetes resource. + Strategy Strategy `json:"strategy"` +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // TestAssert represents the settings needed to verify the result of a test step. diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index be6fac06..49908f75 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -39,6 +39,43 @@ func (in *Apply) DeepCopy() *Apply { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Assert) DeepCopyInto(out *Assert) { + *out = *in + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(Options) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Assert. +func (in *Assert) DeepCopy() *Assert { + if in == nil { + return nil + } + out := new(Assert) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AssertArray) DeepCopyInto(out *AssertArray) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AssertArray. +func (in *AssertArray) DeepCopy() *AssertArray { + if in == nil { + return nil + } + out := new(AssertArray) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Command) DeepCopyInto(out *Command) { *out = *in @@ -126,6 +163,27 @@ func (in *ObjectReference) DeepCopy() *ObjectReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Options) DeepCopyInto(out *Options) { + *out = *in + if in.AssertArray != nil { + in, out := &in.AssertArray, &out.AssertArray + *out = make([]AssertArray, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Options. +func (in *Options) DeepCopy() *Options { + if in == nil { + return nil + } + out := new(Options) + in.DeepCopyInto(out) + return out +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestConfig. func (in *RestConfig) DeepCopy() *RestConfig { if in == nil { @@ -222,8 +280,10 @@ func (in *TestStep) DeepCopyInto(out *TestStep) { } if in.Assert != nil { in, out := &in.Assert, &out.Assert - *out = make([]string, len(*in)) - copy(*out, *in) + *out = make([]Assert, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Error != nil { in, out := &in.Error, &out.Error diff --git a/pkg/test/case.go b/pkg/test/case.go index 138f8c50..f9bc35a4 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -465,7 +465,7 @@ func (t *Case) LoadTestSteps() error { Index: int(index), SkipDelete: t.SkipDelete, Dir: t.Dir, - Asserts: []client.Object{}, + Asserts: []assertArray{}, Apply: []apply{}, Errors: []client.Object{}, } diff --git a/pkg/test/case_test.go b/pkg/test/case_test.go index 557c4b48..1159c8ac 100644 --- a/pkg/test/case_test.go +++ b/pkg/test/case_test.go @@ -49,10 +49,13 @@ func TestLoadTestSteps(t *testing.T) { }), }, }, - Asserts: []client.Object{ - testutils.WithStatus(t, testutils.NewPod("test", ""), map[string]interface{}{ - "qosClass": "BestEffort", - }), + Asserts: []assertArray{ + { + object: testutils.WithStatus(t, testutils.NewPod("test", ""), map[string]interface{}{ + "qosClass": "BestEffort", + }), + options: nil, + }, }, Errors: []client.Object{}, }, @@ -95,10 +98,10 @@ func TestLoadTestSteps(t *testing.T) { }), }, }, - Asserts: []client.Object{ - testutils.WithStatus(t, testutils.NewPod("test2", ""), map[string]interface{}{ + Asserts: []assertArray{ + {object: testutils.WithStatus(t, testutils.NewPod("test2", ""), map[string]interface{}{ "qosClass": "BestEffort", - }), + })}, }, Errors: []client.Object{}, }, @@ -127,10 +130,11 @@ func TestLoadTestSteps(t *testing.T) { }), }, }, - Asserts: []client.Object{ - testutils.WithStatus(t, testutils.NewPod("test3", ""), map[string]interface{}{ - "qosClass": "BestEffort", - }), + Asserts: []assertArray{ + { + object: testutils.WithStatus(t, testutils.NewPod("test3", ""), map[string]interface{}{ + "qosClass": "BestEffort", + })}, }, Errors: []client.Object{}, }, @@ -171,10 +175,10 @@ func TestLoadTestSteps(t *testing.T) { }), }, }, - Asserts: []client.Object{ - testutils.WithSpec(t, testutils.NewPod("test5", ""), map[string]interface{}{ + Asserts: []assertArray{ + {object: testutils.WithSpec(t, testutils.NewPod("test5", ""), map[string]interface{}{ "restartPolicy": "Never", - }), + })}, }, Errors: []client.Object{}, }, @@ -210,8 +214,8 @@ func TestLoadTestSteps(t *testing.T) { }, }, }, - Asserts: []client.Object{ - &unstructured.Unstructured{ + Asserts: []assertArray{ + {object: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "v1", "kind": "Pod", @@ -229,7 +233,7 @@ func TestLoadTestSteps(t *testing.T) { }, }, }, - }, + }}, }, Errors: []client.Object{}, }, diff --git a/pkg/test/step.go b/pkg/test/step.go index 0d5f4f1f..702f249c 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -23,6 +23,7 @@ import ( "github.com/kyverno/kuttl/pkg/env" kfile "github.com/kyverno/kuttl/pkg/file" "github.com/kyverno/kuttl/pkg/http" + "github.com/kyverno/kuttl/pkg/test/utils" testutils "github.com/kyverno/kuttl/pkg/test/utils" ) @@ -35,6 +36,12 @@ type apply struct { shouldFail bool } +type assertArray struct { + object client.Object + shouldfail bool + options *harness.Options +} + // A Step contains the name of the test step, its index in the test, // and all of the test step's settings (including objects to apply and assert on). type Step struct { @@ -47,7 +54,7 @@ type Step struct { Step *harness.TestStep Assert *harness.TestAssert - Asserts []client.Object + Asserts []assertArray Apply []apply Errors []client.Object @@ -283,7 +290,7 @@ func list(cl client.Client, gvk schema.GroupVersionKind, namespace string) ([]un } // CheckResource checks if the expected resource's state in Kubernetes is correct. -func (s *Step) CheckResource(expected runtime.Object, namespace string) []error { +func (s *Step) CheckResource(expected assertArray, namespace string) []error { cl, err := s.Client(false) if err != nil { return []error{err} @@ -296,12 +303,12 @@ func (s *Step) CheckResource(expected runtime.Object, namespace string) []error testErrors := []error{} - name, namespace, err := testutils.Namespaced(dClient, expected, namespace) + name, namespace, err := testutils.Namespaced(dClient, expected.object, namespace) if err != nil { return append(testErrors, err) } - gvk := expected.GetObjectKind().GroupVersionKind() + gvk := expected.object.GetObjectKind().GroupVersionKind() actuals := []unstructured.Unstructured{} @@ -325,25 +332,40 @@ func (s *Step) CheckResource(expected runtime.Object, namespace string) []error return append(testErrors, err) } - expectedObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(expected) + expectedObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(expected.object) if err != nil { return append(testErrors, err) } + testutils.StrategyFactory = func(path string) utils.ArrayComparisonStrategy { + for _, assertArr := range expected.options.AssertArray { + if assertArr.Path == path { + switch assertArr.Strategy { + case harness.StrategyExact: + return utils.StrategyExact(path) + case harness.StrategyAnywhere: + return utils.StrategyAnywhere(path) + } + } + } + // Default strategy if no match is found + return utils.StrategyExact(path) + } + for _, actual := range actuals { actual := actual tmpTestErrors := []error{} - if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent()); err != nil { - diff, diffErr := testutils.PrettyDiff(expected, &actual) + if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent(), "", testutils.StrategyFactory); err != nil { + diff, diffErr := testutils.PrettyDiff(expected.object, &actual) if diffErr == nil { tmpTestErrors = append(tmpTestErrors, fmt.Errorf(diff)) } else { tmpTestErrors = append(tmpTestErrors, diffErr) } - tmpTestErrors = append(tmpTestErrors, fmt.Errorf("resource %s: %s", testutils.ResourceID(expected), err)) + tmpTestErrors = append(tmpTestErrors, fmt.Errorf("resource %s: %s", testutils.ResourceID(expected.object), err)) } if len(tmpTestErrors) == 0 { @@ -407,7 +429,8 @@ func (s *Step) CheckResourceAbsent(expected runtime.Object, namespace string) er var unexpectedObjects []unstructured.Unstructured for _, actual := range actuals { - if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent()); err == nil { + // To be fixed later + if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent(), "", nil); err == nil { unexpectedObjects = append(unexpectedObjects, actual) } } @@ -436,7 +459,14 @@ func (s *Step) Check(namespace string, timeout int) []error { testErrors := []error{} for _, expected := range s.Asserts { - testErrors = append(testErrors, s.CheckResource(expected, namespace)...) + err := append(testErrors, s.CheckResource(expected, namespace)...) + if err != nil && !expected.shouldfail { + testErrors = append(testErrors, err...) + } + // if there was no error but we expected one + if err == nil && expected.shouldfail { + testErrors = append(testErrors, errors.New("an error was expected but didn't happen")) + } } if s.Assert != nil { @@ -543,9 +573,10 @@ func (s *Step) LoadYAML(file string) error { return fmt.Errorf("populating step: %v", err) } - asserts := []client.Object{} + asserts := []assertArray{} - for _, obj := range s.Asserts { + for _, assert := range s.Asserts { + obj := assert.object if obj.GetObjectKind().GroupVersionKind().Kind == "TestAssert" { if testAssert, ok := obj.DeepCopyObject().(*harness.TestAssert); ok { s.Assert = testAssert @@ -553,7 +584,7 @@ func (s *Step) LoadYAML(file string) error { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } } else { - asserts = append(asserts, obj) + asserts = append(asserts, assert) } } @@ -600,13 +631,15 @@ func (s *Step) LoadYAML(file string) error { } } // process configured step asserts - for _, assertPath := range s.Step.Assert { - exAssert := env.Expand(assertPath) + for _, aa := range s.Step.Assert { + exAssert := env.Expand(aa.File) assert, err := ObjectsFromPath(exAssert, s.Dir) if err != nil { return fmt.Errorf("step %q assert path %s: %w", s.Name, exAssert, err) } - asserts = append(asserts, assert...) + for _, a := range assert { + asserts = append(asserts, assertArray{object: a, shouldfail: aa.ShouldFail, options: aa.Options}) + } } // process configured errors for _, errorPath := range s.Step.Error { @@ -634,7 +667,9 @@ func (s *Step) populateObjectsByFileName(fileName string, objects []client.Objec switch fname := strings.ToLower(matches[1]); fname { case "assert": - s.Asserts = append(s.Asserts, objects...) + for _, obj := range objects { + s.Asserts = append(s.Asserts, assertArray{object: obj}) + } case "errors": s.Errors = append(s.Errors, objects...) default: @@ -704,7 +739,7 @@ func validateTestStep(ts *harness.TestStep, baseDir string) error { } // Check if referenced files in Assert exist for _, assert := range ts.Assert { - path := filepath.Join(baseDir, assert) + path := filepath.Join(baseDir, assert.File) if _, err := os.Stat(path); os.IsNotExist(err) { return fmt.Errorf("referenced file in Assert does not exist: %s", path) } diff --git a/pkg/test/step_test.go b/pkg/test/step_test.go index 10529eee..97329cf5 100644 --- a/pkg/test/step_test.go +++ b/pkg/test/step_test.go @@ -287,12 +287,13 @@ func TestRun(t *testing.T) { updateMethod func(*testing.T, client.Client) }{ { - testName: "successful run", Step: Step{ + testName: "successful run", + Step: Step{ Apply: []apply{ {object: testutils.NewPod("hello", "")}, }, - Asserts: []client.Object{ - testutils.NewPod("hello", ""), + Asserts: []assertArray{ + {object: testutils.NewPod("hello", "")}, }, }, }, @@ -301,10 +302,10 @@ func TestRun(t *testing.T) { Apply: []apply{ {object: testutils.NewPod("hello", "")}, }, - Asserts: []client.Object{ - testutils.WithStatus(t, testutils.NewPod("hello", ""), map[string]interface{}{ + Asserts: []assertArray{ + {object: testutils.WithStatus(t, testutils.NewPod("hello", ""), map[string]interface{}{ "phase": "Ready", - }), + })}, }, }, nil, }, @@ -313,10 +314,10 @@ func TestRun(t *testing.T) { Apply: []apply{ {object: testutils.NewPod("hello", "")}, }, - Asserts: []client.Object{ - testutils.WithStatus(t, testutils.NewPod("hello", ""), map[string]interface{}{ + Asserts: []assertArray{ + {object: testutils.WithStatus(t, testutils.NewPod("hello", ""), map[string]interface{}{ "phase": "Ready", - }), + })}, }, }, func(t *testing.T, client client.Client) { pod := testutils.NewPod("hello", testNamespace) diff --git a/pkg/test/test_data/teststep-assert/00-create.yaml b/pkg/test/test_data/teststep-assert/00-create.yaml index 0eb60fe0..c3057dd3 100644 --- a/pkg/test/test_data/teststep-assert/00-create.yaml +++ b/pkg/test/test_data/teststep-assert/00-create.yaml @@ -5,5 +5,7 @@ apply: - hello1-pod.yaml - hello2/hello2-pod.yaml assert: - - hello1-assert.yaml - - hello2/hello2-assert.yaml \ No newline at end of file + - file : hello1-assert.yaml + shouldFail : false + - file : hello2/hello2-assert.yaml + shouldFail : false \ No newline at end of file diff --git a/pkg/test/utils/subset.go b/pkg/test/utils/subset.go index 44212286..4fe25362 100644 --- a/pkg/test/utils/subset.go +++ b/pkg/test/utils/subset.go @@ -5,6 +5,11 @@ import ( "reflect" ) +type ArrayComparisonStrategyFactory func(path string) ArrayComparisonStrategy +type ArrayComparisonStrategy func(actualData, expectedData []interface{}) error + +var StrategyFactory ArrayComparisonStrategyFactory + // SubsetError is an error type used by IsSubset for tracking the path in the struct. type SubsetError struct { path []string @@ -36,7 +41,7 @@ func (e *SubsetError) Error() string { // IsSubset checks to see if `expected` is a subset of `actual`. A "subset" is an object that is equivalent to // the other object, but where map keys found in actual that are not defined in expected are ignored. -func IsSubset(expected, actual interface{}) error { +func IsSubset(expected, actual interface{}, currentPath string, strategyFactory ArrayComparisonStrategyFactory) error { if reflect.TypeOf(expected) != reflect.TypeOf(actual) { return &SubsetError{ message: fmt.Sprintf("type mismatch: %v != %v", reflect.TypeOf(expected), reflect.TypeOf(actual)), @@ -49,17 +54,21 @@ func IsSubset(expected, actual interface{}) error { switch reflect.TypeOf(expected).Kind() { case reflect.Slice: - if reflect.ValueOf(expected).Len() != reflect.ValueOf(actual).Len() { - return &SubsetError{ - message: fmt.Sprintf("slice length mismatch: %d != %d", reflect.ValueOf(expected).Len(), reflect.ValueOf(actual).Len()), - } + var strategy ArrayComparisonStrategy + if strategyFactory != nil { + strategy = strategyFactory(currentPath) + } else { + strategy = StrategyExact(currentPath) } + expectedData := make([]interface{}, reflect.ValueOf(expected).Len()) + actualData := make([]interface{}, reflect.ValueOf(actual).Len()) + for i := 0; i < reflect.ValueOf(expected).Len(); i++ { - if err := IsSubset(reflect.ValueOf(expected).Index(i).Interface(), reflect.ValueOf(actual).Index(i).Interface()); err != nil { - return err - } + expectedData[i] = reflect.ValueOf(expected).Index(i).Interface() + actualData[i] = reflect.ValueOf(actual).Index(i).Interface() } + return strategy(actualData, expectedData) case reflect.Map: iter := reflect.ValueOf(expected).MapRange() @@ -68,15 +77,16 @@ func IsSubset(expected, actual interface{}) error { if !actualValue.IsValid() { return &SubsetError{ - path: []string{iter.Key().String()}, + path: []string{currentPath}, message: "key is missing from map", } } - if err := IsSubset(iter.Value().Interface(), actualValue.Interface()); err != nil { + newPath := currentPath + "/" + iter.Key().String() + + if err := IsSubset(iter.Value().Interface(), actualValue.Interface(), newPath, strategyFactory); err != nil { subsetErr, ok := err.(*SubsetError) if ok { - subsetErr.AppendPath(iter.Key().String()) return subsetErr } return err @@ -90,3 +100,37 @@ func IsSubset(expected, actual interface{}) error { return nil } + +func StrategyAnywhere(path string) ArrayComparisonStrategy { + return func(actualData, expectedData []interface{}) error { + for i, expectedItem := range expectedData { + matched := false + for _, actualItem := range actualData { + newPath := path + fmt.Sprintf("[%d]", i) + if err := IsSubset(expectedItem, actualItem, newPath, StrategyFactory); err == nil { + matched = true + break + } + } + if !matched { + return &SubsetError{message: fmt.Sprintf("expected item %v not found in actual slice at path %s", expectedItem, path)} + } + } + return nil + } +} + +func StrategyExact(path string) ArrayComparisonStrategy { + return func(actualData, expectedData []interface{}) error { + if len(expectedData) != len(actualData) { + return &SubsetError{message: fmt.Sprintf("slice length mismatch at path %s: %d != %d", path, len(expectedData), len(actualData))} + } + for i, v := range expectedData { + newPath := path + fmt.Sprintf("[%d]", i) + if err := IsSubset(v, actualData[i], newPath, StrategyFactory); err != nil { + return err + } + } + return nil + } +}