metrics: implement test for the metrics generator

pkg/metrics/generate_integration_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2023 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"sigs.k8s.io/controller-tools/pkg/genall"
+	"sigs.k8s.io/controller-tools/pkg/loader"
+	"sigs.k8s.io/controller-tools/pkg/markers"
+	"sigs.k8s.io/controller-tools/pkg/metrics/internal/config"
+)
+
+func Test_Generate(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	optionsRegistry := &markers.Registry{}
+
+	metricGenerator := Generator{}
+	if err := metricGenerator.RegisterMarkers(optionsRegistry); err != nil {
+		t.Error(err)
+	}
+
+	out := &outputRule{
+		buf: &bytes.Buffer{},
+	}
+
+	// Load the passed packages as roots.
+	roots, err := loader.LoadRoots(path.Join(cwd, "testdata", "..."))
+	if err != nil {
+		t.Errorf("loading packages %v", err)
+	}
+
+	gen := Generator{}
+
+	generationContext := &genall.GenerationContext{
+		Collector:  &markers.Collector{Registry: optionsRegistry},
+		Roots:      roots,
+		Checker:    &loader.TypeChecker{},
+		OutputRule: out,
+	}
+
+	t.Log("Trying to generate a custom resource configuration from the loaded packages")
+
+	if err := gen.Generate(generationContext); err != nil {
+		t.Error(err)
+	}
+
+	output := strings.Split(out.buf.String(), "\n---\n")
+
+	header := fmt.Sprintf(headerText, "(devel)", config.KubeStateMetricsVersion)
+
+	if len(output) != 3 {
+		t.Error("Expected two output files, metrics configuration followed by rbac.")
+		return
+	}
+
+	generatedData := map[string]string{
+		"metrics.yaml": header + "---\n" + string(output[1]),
+		"rbac.yaml":    "---\n" + string(output[2]),
+	}
+
+	t.Log("Comparing output to testdata to check for regressions")
+
+	for _, golden := range []string{"metrics.yaml", "rbac.yaml"} {
+		// generatedRaw := strings.TrimSpace(output[i])
+
+		expectedRaw, err := os.ReadFile(path.Clean(path.Join(cwd, "testdata", golden)))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		// Remove leading `---` and trim newlines
+		generated := strings.TrimSpace(strings.TrimPrefix(generatedData[golden], "---"))
+		expected := strings.TrimSpace(strings.TrimPrefix(string(expectedRaw), "---"))
+
+		diff := cmp.Diff(expected, generated)
+		if diff != "" {
+			t.Log("generated:")
+			t.Log(generated)
+			t.Log("diff:")
+			t.Log(diff)
+			t.Logf("Expected output to match file `testdata/%s` but it does not.", golden)
+			t.Logf("If the change is intended, use `go generate ./pkg/metrics/testdata` to regenerate the `testdata/%s` file.", golden)
+			t.Errorf("Detected a diff between the output of the integration test and the file `testdata/%s`.", golden)
+			return
+		}
+	}
+}
+
+type outputRule struct {
+	buf *bytes.Buffer
+}
+
+func (o *outputRule) Open(_ *loader.Package, _ string) (io.WriteCloser, error) {
+	return nopCloser{o.buf}, nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (n nopCloser) Close() error {
+	return nil
+}

pkg/metrics/testdata/README.md

@@ -0,0 +1,64 @@
+# Testdata for generator tests
+
+The files in this directory are used for testing the `kube-state-metrics generate` command and to provide an example.
+
+## foo-config.yaml
+
+This file is used in the test at [generate_integration_test.go](../generate_integration_test.go) to verify that the resulting configuration does not change during changes in the codebase.
+
+If there are intended changes this file needs to get regenerated to make the test succeed again.
+This could be done via:
+
+```sh
+go run ./cmd/controller-gen metrics crd \
+    paths=./pkg/metrics/testdata \
+    output:dir=./pkg/metrics/testdata
+```
+
+Or by using the go:generate marker inside [foo_types.go](foo_types.go):
+
+```sh
+go generate ./pkg/metrics/testdata/
+```
+
+## Example files: metrics.yaml, rbac.yaml and example-metrics.txt
+
+There is also an example CR ([example-foo.yaml](example-foo.yaml)) and resulting example metrics ([example-metrics.txt](example-metrics.txt)).
+
+The example metrics file got created by:
+
+1. Generating a CustomResourceDefinition and Kube-State-Metrics configration file:
+
+    ```sh
+    go generate ./pkg/metrics/testdata/
+    ```
+
+2. Creating a cluster using [kind](https://kind.sigs.k8s.io/)
+
+    ```sh
+    kind create cluster
+    ```
+
+3. Applying the CRD and example CR to the cluster:
+
+    ```sh
+    kubectl apply -f ./pkg/metrics/testdata/bar.example.com_foos.yaml
+    kubectl apply -f ./pkg/metrics/testdata/example-foo.yaml
+    ```
+
+4. Running kube-state-metrics with the provided configuration file:
+
+    ```sh
+    docker run --net=host -ti --rm \
+        -v $HOME/.kube/config:/config \
+        -v $(pwd):/data \
+        registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0 \
+        --kubeconfig /config --custom-resource-state-only \
+        --custom-resource-state-config-file /data/pkg/metrics/testdata/foo-config.yaml
+    ```
+
+5. Querying the metrics endpoint in a second terminal:
+
+    ```sh
+    curl localhost:8080/metrics > ./pkg/metrics/testdata/foo-cr-example-metrics.txt
+    ```

pkg/metrics/testdata/bar.example.com_foos.yaml

@@ -0,0 +1,74 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  name: foos.bar.example.com
+spec:
+  group: bar.example.com
+  names:
+    kind: Foo
+    listKind: FooList
+    plural: foos
+    singular: foo
+  scope: Namespaced
+  versions:
+  - name: foo
+    schema:
+      openAPIV3Schema:
+        description: Foo is a test object.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec comments SHOULD appear in the CRD spec
+            properties:
+              someString:
+                description: SomeString is a string.
+                type: string
+            required:
+            - someString
+            type: object
+          status:
+            description: Status comments SHOULD appear in the CRD spec
+            properties:
+              conditions:
+                items:
+                  description: Condition is a test condition.
+                  properties:
+                    lastTransitionTime:
+                      description: LastTransitionTime of condition.
+                      format: date-time
+                      type: string
+                    status:
+                      description: Status of condition.
+                      type: string
+                    type:
+                      description: Type of condition.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - status
+                  - type
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true

pkg/metrics/testdata/example-foo.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: bar.example.com/foo
+kind: Foo
+metadata:
+  name: bar
+  ownerReferences:
+  - apiVersion: v1
+    kind: foo
+    controller: true
+    name: foo
+    uid: someuid
+spec:
+  someString: test
+status:
+  conditions:
+  - lastTransitionTime: "2023-10-12T13:59:02Z"
+    status: "True"
+    type: SomeType
+  - lastTransitionTime: "2023-10-12T13:59:02Z"
+    status: "False"
+    type: AnotherType

pkg/metrics/testdata/example-metrics.txt

@@ -0,0 +1,18 @@
+# HELP foo_created Unix creation timestamp.
+# TYPE foo_created gauge
+foo_created{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar"} 1.724940463e+09
+# HELP foo_owner Owner references.
+# TYPE foo_owner info
+foo_owner{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",owner_is_controller="true",owner_kind="foo",owner_name="foo",owner_uid="someuid"} 1
+# HELP foo_status_condition The condition of a foo.
+# TYPE foo_status_condition stateset
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="AnotherType"} 1
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="SomeType"} 0
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="AnotherType"} 0
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="SomeType"} 1
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="Unknown",type="AnotherType"} 0
+foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="Unknown",type="SomeType"} 0
+# HELP foo_status_condition_last_transition_time The condition last transition time of a foo.
+# TYPE foo_status_condition_last_transition_time gauge
+foo_status_condition_last_transition_time{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="AnotherType"} 1.697119142e+09
+foo_status_condition_last_transition_time{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="SomeType"} 1.697119142e+09

pkg/metrics/testdata/foo_types.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2023 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Changes to this file may require to regenerate the `foo-config.yaml`. Otherwise the
+// tests in ../generate_integration_test.go may fail.
+// The below marker can be used to regenerate the `foo-config.yaml` file by running
+// the following command:
+// $ go generate ./pkg/customresourcestate/generate/generator/testdata
+//go:generate sh -c "go run ../../../cmd/controller-gen crd metrics paths=./ output:dir=."
+
+// +groupName=bar.example.com
+package foo
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// FooSpec is the spec of Foo.
+type FooSpec struct {
+	// SomeString is a string.
+	SomeString string `json:"someString"`
+}
+
+// FooStatus is the status of Foo.
+type FooStatus struct {
+	// +Metrics:stateset:name="status_condition",help="The condition of a foo.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"}
+	// +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a foo.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"}
+	Conditions []Condition `json:"conditions,omitempty"`
+}
+
+// Foo is a test object.
+// +Metrics:gvk:namePrefix="foo"
+// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name"
+// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp."
+// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"}
+// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name
+type Foo struct {
+	// TypeMeta comments should NOT appear in the CRD spec
+	metav1.TypeMeta `json:",inline"`
+	// ObjectMeta comments should NOT appear in the CRD spec
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Spec comments SHOULD appear in the CRD spec
+	Spec FooSpec `json:"spec,omitempty"`
+	// Status comments SHOULD appear in the CRD spec
+	Status FooStatus `json:"status,omitempty"`
+}
+
+// Condition is a test condition.
+type Condition struct {
+	// Type of condition.
+	Type string `json:"type"`
+	// Status of condition.
+	Status string `json:"status"`
+	// LastTransitionTime of condition.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+}