Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

VC-36547: Remove gabs library as a dependency #598

Merged
merged 1 commit into from
Oct 28, 2024
Merged

VC-36547: Remove gabs library as a dependency #598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,12 @@ module github.com/jetstack/preflight
go 1.22.0

require (
github.com/Jeffail/gabs/v2 v2.7.0
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/d4l3k/messagediff v1.2.1
github.com/fatih/color v1.17.0
github.com/google/uuid v1.6.0
github.com/hashicorp/go-multierror v1.1.1
github.com/jetstack/venafi-connection-lib v0.1.1-0.20240909145535-cd2301fd4e7c
github.com/json-iterator/go v1.1.12
github.com/maxatome/go-testdeep v1.14.0
github.com/microcosm-cc/bluemonday v1.0.27
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -44,6 +42,7 @@ require (
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/sosodev/duration v1.2.0 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg=
github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw=
github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
github.com/Venafi/vcert/v5 v5.7.1 h1:gUDbSuP6NE4yAslWp+D+ZoJlYOSRWhQora48oExuEN4=
Expand Down
169 changes: 57 additions & 112 deletions pkg/datagatherer/k8s/fieldfilter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,143 +1,88 @@
package k8s

import (
"fmt"
"strings"

"github.com/Jeffail/gabs/v2"
json "github.com/json-iterator/go"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

// SecretSelectedFields is the list of fields sent from Secret objects to the
// backend
var SecretSelectedFields = []string{
"kind",
"apiVersion",
"metadata.annotations",
"metadata.labels",
"metadata.name",
"metadata.namespace",
"metadata.ownerReferences",
"metadata.selfLink",
"metadata.uid",
"type",
"/data/tls.crt",
"/data/ca.crt",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did this original code use / as path separator for these two fields?

var SecretSelectedFields = []FieldPath{
{"kind"},
{"apiVersion"},
{"metadata", "annotations"},
{"metadata", "labels"},
{"metadata", "name"},
{"metadata", "namespace"},
{"metadata", "ownerReferences"},
{"metadata", "selfLink"},
{"metadata", "uid"},

{"type"},
{"data", "tls.crt"},
{"data", "ca.crt"},
}

// RouteSelectedFields is the list of fields sent from OpenShift Route objects to the
// backend
var RouteSelectedFields = []string{
"kind",
"apiVersion",
"metadata.annotations",
"metadata.name",
"metadata.namespace",
"metadata.ownerReferences",
"metadata.selfLink",
"metadata.uid",
"spec.host",
"spec.to.kind",
"spec.to.port",
"spec.to.name",
"spec.to.weight",
"spec.tls.termination",
"spec.tls.certificate",
"spec.tls.caCertificate",
"spec.tls.destinationCACertificate",
"spec.tls.insecureEdgeTerminationPolicy",
"spec.wildcardPolicy",
"status",
var RouteSelectedFields = []FieldPath{
{"kind"},
{"apiVersion"},
{"metadata", "annotations"},
{"metadata", "name"},
{"metadata", "namespace"},
{"metadata", "ownerReferences"},
{"metadata", "selfLink"},
{"metadata", "uid"},

{"spec", "host"},
{"spec", "to", "kind"},
{"spec", "to", "name"},
{"spec", "to", "weight"},
{"spec", "tls", "termination"},
{"spec", "tls", "certificate"},
{"spec", "tls", "caCertificate"},
{"spec", "tls", "destinationCACertificate"},
{"spec", "tls", "insecureEdgeTerminationPolicy"},
{"spec", "wildcardPolicy"},
{"status"},
}

// RedactFields are removed from all objects
var RedactFields = []string{
"metadata.managedFields",
"/metadata/annotations/kubectl.kubernetes.io~1last-applied-configuration",
inteon marked this conversation as resolved.
Show resolved Hide resolved
var RedactFields = []FieldPath{
{"metadata", "managedFields"},
{"metadata", "annotations", "kubectl.kubernetes.io/last-applied-configuration"},
}

// Select removes all but the supplied fields from the resource
func Select(fields []string, resource *unstructured.Unstructured) error {
// convert the object to JSON for field filtering
asJSON, err := json.Marshal(resource)
if err != nil {
return fmt.Errorf("failed to marshal json for resource: %s", err)
}
type FieldPath []string

// parse the JSON for processing in gabs
jsonParsed, err := gabs.ParseJSON(asJSON)
if err != nil {
return fmt.Errorf("failed to parse generated json for resource: %s", err)
// Select removes all but the supplied fields from the resource
func Select(fields []FieldPath, resource *unstructured.Unstructured) error {
newResource := unstructured.Unstructured{
Object: map[string]interface{}{},
}

// craft a new object containing only selected fields
filteredObject := gabs.New()
for _, v := range fields {
// also support JSONPointers for keys containing '.' chars
if strings.HasPrefix(v, "/") {
gObject, err := jsonParsed.JSONPointer(v)
if err != nil {
// fail to select field if missing, just continue
continue
}
pathComponents, err := gabs.JSONPointerToSlice(v)
if err != nil {
return fmt.Errorf("invalid JSONPointer: %s", v)
}
filteredObject.Set(gObject.Data(), pathComponents...)
} else {
if jsonParsed.ExistsP(v) {
filteredObject.SetP(jsonParsed.Path(v).Data(), v)
}
for _, field := range fields {
value, found, err := unstructured.NestedFieldNoCopy(resource.Object, field...)
if err != nil {
return err
}
if !found {
continue
}
if err := unstructured.SetNestedField(newResource.Object, value, field...); err != nil {
return err
}
}

// load the filtered JSON back into the resource
err = json.Unmarshal(filteredObject.Bytes(), resource)
if err != nil {
return fmt.Errorf("failed to update resource: %s", err)
}
resource.Object = newResource.Object

return nil
}

// Redact removes the supplied fields from the resource
func Redact(fields []string, resource *unstructured.Unstructured) error {
// convert the object to JSON for field filtering
asJSON, err := json.Marshal(resource)
if err != nil {
return fmt.Errorf("failed to marshal json for resource: %s", err)
}

// parse the JSON for processing in gabs
jsonParsed, err := gabs.ParseJSON(asJSON)
if err != nil {
return fmt.Errorf("failed to parse generated json for resource: %s", err)
}

// craft a new object excluding redacted fields
for _, v := range fields {
// also support JSONPointers for keys containing '.' chars
if strings.HasPrefix(v, "/") {
pathComponents, err := gabs.JSONPointerToSlice(v)
if err != nil {
return fmt.Errorf("invalid JSONPointer: %s", v)
}
if jsonParsed.Exists(pathComponents...) {
jsonParsed.Delete(pathComponents...)
}
} else {
if jsonParsed.ExistsP(v) {
jsonParsed.DeleteP(v)
}
}
}

// load the filtered JSON back into the resource
err = json.Unmarshal(jsonParsed.Bytes(), resource)
if err != nil {
return fmt.Errorf("failed to update resource: %s", err)
func Redact(fields []FieldPath, resource *unstructured.Unstructured) error {
for _, field := range fields {
unstructured.RemoveNestedField(resource.Object, field...)
}

return nil
Expand Down
38 changes: 18 additions & 20 deletions pkg/datagatherer/k8s/fieldfilter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ func TestSelect(t *testing.T) {
"metadata": map[string]interface{}{
"name": "example",
"namespace": "example",
"annotations": map[string]string{
"annotations": map[string]interface{}{
Copy link
Contributor Author

@inteon inteon Oct 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can expect all maps to be of type map[string]interface{}, that is the behavior for Kubernetes objects unmarshalled in unstructured objects. The helper unstructured functions will also make sure this stays true (eg. https://github.com/kubernetes/apimachinery/blob/master/pkg/apis/meta/v1/unstructured/helpers.go#L259).

"kubectl.kubernetes.io/last-applied-configuration": "secret",
},
"labels": map[string]string{
"labels": map[string]interface{}{
"foo": "bar",
},
},
Expand Down Expand Up @@ -61,16 +61,16 @@ func TestSelect(t *testing.T) {
"kind": "Route",
"metadata": map[string]interface{}{
"name": "example",
"annotations": map[string]string{
"annotations": map[string]interface{}{
"kubectl.kubernetes.io/last-applied-configuration": "secret",
},
"labels": map[string]string{
"labels": map[string]interface{}{
"foo": "bar",
},
},
"spec": map[string]interface{}{
"host": "www.example.com",
"to": map[string]string{
"to": map[string]interface{}{
"kind": "Service",
"name": "frontend",
},
Expand Down Expand Up @@ -112,7 +112,7 @@ func TestSelect(t *testing.T) {
))
}

func run_TestSelect(given map[string]interface{}, givenSelect []string, expect map[string]interface{}) func(*testing.T) {
func run_TestSelect(given map[string]interface{}, givenSelect []FieldPath, expect map[string]interface{}) func(*testing.T) {
return func(t *testing.T) {
t.Helper()
givenPtr := unstructured.Unstructured{Object: given}
Expand All @@ -130,10 +130,9 @@ func TestSelectMissingSelectedField(t *testing.T) {
},
}

fieldsToSelect := []string{
"kind", // required for unstructured unmarshal
"missing",
"/missing",
fieldsToSelect := []FieldPath{
{"kind"}, // required for unstructured unmarshal
{"missing"},
}

err := Select(fieldsToSelect, resource)
Expand All @@ -156,7 +155,7 @@ func TestRedactSecret(t *testing.T) {
"metadata": map[string]interface{}{
"name": "example",
"namespace": "example",
"annotations": map[string]string{
"annotations": map[string]interface{}{
"kubectl.kubernetes.io/last-applied-configuration": "secret",
},
"managedFields": nil,
Expand All @@ -169,10 +168,10 @@ func TestRedactSecret(t *testing.T) {
},
}

fieldsToRedact := []string{
"metadata.managedFields",
"/metadata/annotations/kubectl.kubernetes.io~1last-applied-configuration",
"/data/tls.key",
fieldsToRedact := []FieldPath{
{"metadata", "managedFields"},
{"metadata", "annotations", "kubectl.kubernetes.io/last-applied-configuration"},
{"data", "tls.key"},
}

err := Redact(fieldsToRedact, resource)
Expand Down Expand Up @@ -213,8 +212,8 @@ func TestRedactPod(t *testing.T) {
},
}

fieldsToRedact := []string{
"metadata.managedFields",
fieldsToRedact := []FieldPath{
{"metadata", "managedFields"},
}

err := Redact(fieldsToRedact, resource)
Expand Down Expand Up @@ -244,9 +243,8 @@ func TestRedactMissingField(t *testing.T) {
},
}

fieldsToRedact := []string{
"missing",
"/missing",
fieldsToRedact := []FieldPath{
{"missing"},
}

err := Redact(fieldsToRedact, resource)
Expand Down