-
Notifications
You must be signed in to change notification settings - Fork 94
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: node metrics collector (#1516)
* feat: node metrics collector A collector to collect node metrics served by the API server as per the documented API https://kubernetes.io/docs/reference/instrumentation/node-metrics/ * Update CRD schemas * Add tests * Remove clean from build target * Update comments * Commit missing tests * Remove unnecessary log in tests
- Loading branch information
Showing
14 changed files
with
365 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package collect | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| "k8s.io/client-go/rest" | ||
| "k8s.io/klog/v2" | ||
| ) | ||
|
|
||
| const ( | ||
| summaryUrlTemplate = "/api/v1/nodes/%s/proxy/stats/summary" | ||
| ) | ||
|
|
||
| type CollectNodeMetrics struct { | ||
| Collector *troubleshootv1beta2.NodeMetrics | ||
| BundlePath string | ||
| ClientConfig *rest.Config | ||
| Client kubernetes.Interface | ||
| Context context.Context | ||
| RBACErrors | ||
| } | ||
|
|
||
| func (c *CollectNodeMetrics) Title() string { | ||
| return getCollectorName(c) | ||
| } | ||
|
|
||
| func (c *CollectNodeMetrics) IsExcluded() (bool, error) { | ||
| return isExcluded(c.Collector.Exclude) | ||
| } | ||
|
|
||
| func (c *CollectNodeMetrics) Collect(progressChan chan<- interface{}) (CollectorResult, error) { | ||
| output := NewResult() | ||
| nodesMap := c.constructNodesMap() | ||
| if len(nodesMap) == 0 { | ||
| klog.V(2).Info("no nodes found to collect metrics for") | ||
| return output, nil | ||
| } | ||
|
|
||
| nodeNames := make([]string, 0, len(nodesMap)) | ||
| for nodeName := range nodesMap { | ||
| nodeNames = append(nodeNames, nodeName) | ||
| } | ||
|
|
||
| klog.V(2).Infof("collecting node metrics for [%s] nodes", strings.Join(nodeNames, ", ")) | ||
|
|
||
| for nodeName, endpoint := range nodesMap { | ||
| // Equivalent to `kubectl get --raw "/api/v1/nodes/<nodeName>/proxy/stats/summary"` | ||
| klog.V(2).Infof("querying: %+v\n", endpoint) | ||
| response, err := c.Client.CoreV1().RESTClient().Get().AbsPath(endpoint).DoRaw(c.Context) | ||
| if err != nil { | ||
| return output, errors.Wrapf(err, "could not query endpoint %s", endpoint) | ||
| } | ||
| err = output.SaveResult(c.BundlePath, fmt.Sprintf("node-metrics/%s.json", nodeName), bytes.NewBuffer(response)) | ||
| if err != nil { | ||
| klog.Errorf("failed to save node metrics for %s: %v", nodeName, err) | ||
| } | ||
|
|
||
| } | ||
| return output, nil | ||
| } | ||
|
|
||
| func (c *CollectNodeMetrics) constructNodesMap() map[string]string { | ||
| nodesMap := map[string]string{} | ||
|
|
||
| if c.Collector.NodeNames == nil && c.Collector.Selector == nil { | ||
| // If no node names or selectors are provided, collect all nodes | ||
| nodes, err := c.Client.CoreV1().Nodes().List(c.Context, metav1.ListOptions{}) | ||
| if err != nil { | ||
| klog.Errorf("failed to list nodes: %v", err) | ||
| } | ||
| for _, node := range nodes.Items { | ||
| nodesMap[node.Name] = fmt.Sprintf(summaryUrlTemplate, node.Name) | ||
| } | ||
| return nodesMap | ||
| } | ||
|
|
||
| for _, nodeName := range c.Collector.NodeNames { | ||
| nodesMap[nodeName] = fmt.Sprintf(summaryUrlTemplate, nodeName) | ||
| } | ||
|
|
||
| // Find nodes by label selector | ||
| if c.Collector.Selector != nil { | ||
| nodes, err := c.Client.CoreV1().Nodes().List(c.Context, metav1.ListOptions{ | ||
| LabelSelector: strings.Join(c.Collector.Selector, ","), | ||
| }) | ||
| if err != nil { | ||
| klog.Errorf("failed to list nodes by label selector: %v", err) | ||
| } | ||
| for _, node := range nodes.Items { | ||
| nodesMap[node.Name] = fmt.Sprintf(summaryUrlTemplate, node.Name) | ||
| } | ||
| } | ||
|
|
||
| return nodesMap | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package collect | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| v1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| testclient "k8s.io/client-go/kubernetes/fake" | ||
| ) | ||
|
|
||
| func TestCollectNodeMetrics_constructNodesMap(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| objectMetas []metav1.ObjectMeta | ||
| collector troubleshootv1beta2.NodeMetrics | ||
| want map[string]string | ||
| }{ | ||
| { | ||
| name: "default collector no nodes", | ||
| want: map[string]string{}, | ||
| }, | ||
| { | ||
| name: "default collector one node", | ||
| objectMetas: []metav1.ObjectMeta{ | ||
| { | ||
| Name: "node1", | ||
| }, | ||
| }, | ||
| want: map[string]string{ | ||
| "node1": "/api/v1/nodes/node1/proxy/stats/summary", | ||
| }, | ||
| }, | ||
| { | ||
| name: "collector with node list picking one node", | ||
| objectMetas: []metav1.ObjectMeta{ | ||
| { | ||
| Name: "node1", | ||
| }, | ||
| { | ||
| Name: "node2", | ||
| }, | ||
| }, | ||
| collector: troubleshootv1beta2.NodeMetrics{ | ||
| NodeNames: []string{"node2"}, | ||
| }, | ||
| want: map[string]string{ | ||
| "node2": "/api/v1/nodes/node2/proxy/stats/summary", | ||
| }, | ||
| }, | ||
| { | ||
| name: "collector with selector picking one node", | ||
| objectMetas: []metav1.ObjectMeta{ | ||
| { | ||
| Name: "node1", | ||
| Labels: map[string]string{ | ||
| "hostname": "node1.example.com", | ||
| }, | ||
| }, | ||
| { | ||
| Name: "node2", | ||
| }, | ||
| }, | ||
| collector: troubleshootv1beta2.NodeMetrics{ | ||
| Selector: []string{"hostname=node1.example.com"}, | ||
| }, | ||
| want: map[string]string{ | ||
| "node1": "/api/v1/nodes/node1/proxy/stats/summary", | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| client := testclient.NewSimpleClientset() | ||
| ctx := context.Background() | ||
| collector := tt.collector | ||
| c := &CollectNodeMetrics{ | ||
| Collector: &collector, | ||
| Client: client, | ||
| Context: ctx, | ||
| } | ||
|
|
||
| for _, objectMeta := range tt.objectMetas { | ||
| _, err := client.CoreV1().Nodes().Create(ctx, &v1.Node{ | ||
| ObjectMeta: objectMeta, | ||
| }, metav1.CreateOptions{}) | ||
| require.NoError(t, err) | ||
| } | ||
|
|
||
| got := c.constructNodesMap() | ||
| assert.Equalf(t, tt.want, got, "constructNodesMap() = %v, want %v", got, tt.want) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.