diff --git a/cmd/cluster.go b/cmd/cluster.go index 6691d84..2fff392 100644 --- a/cmd/cluster.go +++ b/cmd/cluster.go @@ -42,55 +42,10 @@ import ( utiltaints "github.com/gardener/machine-controller-manager/pkg/util/taints" ) -type auditConfigOptionsMap map[string]struct { - Config *models.V1Audit - Description string -} - -func (a auditConfigOptionsMap) Names(withDescription bool) []string { - var names []string - for name, opt := range a { - if withDescription { - names = append(names, fmt.Sprintf("%s\t%s", name, opt.Description)) - } else { - names = append(names, name) - } - } - return names -} - -var ( - // options - auditConfigOptions = auditConfigOptionsMap{ - "off": { - Description: "turn off the kube-apiserver auditlog", - Config: &models.V1Audit{ - ClusterAudit: pointer.Pointer(false), - AuditToSplunk: pointer.Pointer(false), - }, - }, - "on": { - Description: "turn on the kube-apiserver auditlog, and expose it as container log of the audittailer deployment in the audit namespace", - Config: &models.V1Audit{ - ClusterAudit: pointer.Pointer(true), - AuditToSplunk: pointer.Pointer(false), - }, - }, - "splunk": { - Description: "also forward the auditlog to a splunk HEC endpoint. create a custom splunk config manifest with \"cloudctl cluster splunk-config-manifest\"", - Config: &models.V1Audit{ - ClusterAudit: pointer.Pointer(true), - AuditToSplunk: pointer.Pointer(true), - }, - }, - } -) - func newClusterCmd(c *config) *cobra.Command { clusterCmd := &cobra.Command{ Use: "cluster", Short: "manage clusters", - Long: "TODO", } clusterCreateCmd := &cobra.Command{ Use: "create", @@ -262,14 +217,6 @@ func newClusterCmd(c *config) *cobra.Command { ValidArgsFunction: c.comp.ClusterListCompletion, PreRun: bindPFlags, } - clusterSplunkConfigManifestCmd := &cobra.Command{ - Use: "splunk-config-manifest", - Short: "create a manifest for a custom splunk configuration, overriding the default settings for splunk auditing", - RunE: func(cmd *cobra.Command, args []string) error { - return c.clusterSplunkConfigManifest() - }, - PreRun: bindPFlags, - } clusterDNSManifestCmd := &cobra.Command{ Use: "dns-manifest ", Short: "create a manifest for an ingress or service type loadbalancer, creating a DNS entry and valid certificate within your cluster domain", @@ -304,7 +251,6 @@ func newClusterCmd(c *config) *cobra.Command { clusterCreateCmd.Flags().BoolP("allowprivileged", "", false, "allow privileged containers the cluster (this is achieved through pod security policies and has no effect anymore on clusters >= v1.25") clusterCreateCmd.Flags().String("default-pod-security-standard", "", "sets default pod security standard for clusters >= v1.23.x, defaults to restricted on clusters >= v1.25 (valid values: empty string, privileged, baseline, restricted)") clusterCreateCmd.Flags().BoolP("disable-pod-security-policies", "", false, "disable pod security policies") - clusterCreateCmd.Flags().String("audit", "on", "audit logging of cluster API access; can be off, on (default) or splunk (logging to a predefined or custom splunk endpoint). [optional]") clusterCreateCmd.Flags().Duration("healthtimeout", 0, "period (e.g. \"24h\") after which an unhealthy node is declared failed and will be replaced. [optional]") clusterCreateCmd.Flags().Duration("draintimeout", 0, "period (e.g. \"3h\") after which a draining node will be forcefully deleted. [optional]") clusterCreateCmd.Flags().Bool("encrypted-storage-classes", false, "enables the deployment of encrypted duros storage classes into the cluster. please refer to the user manual to properly use volume encryption. [optional]") @@ -342,10 +288,6 @@ func newClusterCmd(c *config) *cobra.Command { "cilium\tcilium networking plugin. please note that cilium support is still Alpha and we are happy to receive feedback.", }, cobra.ShellCompDirectiveNoFileComp })) - must(clusterCreateCmd.RegisterFlagCompletionFunc("audit", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return auditConfigOptions.Names(true), - cobra.ShellCompDirectiveNoFileComp - })) clusterDescribeCmd.Flags().Bool("no-machines", false, "does not return in the output") @@ -388,7 +330,6 @@ func newClusterCmd(c *config) *cobra.Command { clusterUpdateCmd.Flags().BoolP("allowprivileged", "", false, "allow privileged containers the cluster (this is achieved through pod security policies and has no effect anymore on clusters >=v1.25") clusterUpdateCmd.Flags().String("default-pod-security-standard", "", "set default pod security standard for cluster >=v 1.23.x, send empty string explicitly to disable pod security standards (valid values: empty string, privileged, baseline, restricted)") clusterUpdateCmd.Flags().BoolP("disable-pod-security-policies", "", false, "disable pod security policies") - clusterUpdateCmd.Flags().String("audit", "on", "audit logging of cluster API access; can be off, on or splunk (logging to a predefined or custom splunk endpoint).") clusterUpdateCmd.Flags().String("purpose", "", fmt.Sprintf("purpose of the cluster, can be one of %s. SLA is only given on production clusters.", strings.Join(completion.ClusterPurposes, "|"))) clusterUpdateCmd.Flags().StringSlice("egress", []string{}, "static egress ips per network, must be in the form :; e.g.: --egress internet:1.2.3.4;1.2.3.5 --egress extnet:123.1.1.1 [optional]. Use \"--egress none\" to remove all egress rules.") clusterUpdateCmd.Flags().StringSlice("external-networks", []string{}, "external networks of the cluster") @@ -418,23 +359,10 @@ func newClusterCmd(c *config) *cobra.Command { must(clusterUpdateCmd.RegisterFlagCompletionFunc("machineimage", c.comp.MachineImageListCompletion)) must(clusterUpdateCmd.RegisterFlagCompletionFunc("purpose", c.comp.ClusterPurposeListCompletion)) must(clusterUpdateCmd.RegisterFlagCompletionFunc("default-pod-security-standard", c.comp.PodSecurityListCompletion)) - must(clusterUpdateCmd.RegisterFlagCompletionFunc("audit", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return auditConfigOptions.Names(true), - cobra.ShellCompDirectiveNoFileComp - })) clusterInputsCmd.Flags().String("partition", "", "partition of the constraints.") must(clusterInputsCmd.RegisterFlagCompletionFunc("partition", c.comp.PartitionListCompletion)) - // Cluster splunk config manifest -------------------------------------------------------------------- - clusterSplunkConfigManifestCmd.Flags().String("token", "", "the hec token to use for this cluster's audit logs") - clusterSplunkConfigManifestCmd.Flags().String("index", "", "the splunk index to use for this cluster's audit logs") - clusterSplunkConfigManifestCmd.Flags().String("hechost", "", "the hostname or IP of the splunk HEC endpoint") - clusterSplunkConfigManifestCmd.Flags().Int("hecport", 0, "port on which the splunk HEC endpoint is listening") - clusterSplunkConfigManifestCmd.Flags().Bool("tls", false, "whether to use TLS encryption. You do need to specify a CA file.") - clusterSplunkConfigManifestCmd.Flags().String("cafile", "", "the path to the file containing the ca certificate (chain) for the splunk HEC endpoint") - clusterSplunkConfigManifestCmd.Flags().String("cabase64", "", "the base64-encoded ca certificate (chain) for the splunk HEC endpoint") - // Cluster dns manifest -------------------------------------------------------------------- clusterDNSManifestCmd.Flags().String("type", "ingress", "either of type ingress or service") clusterDNSManifestCmd.Flags().String("name", "", "the resource name") @@ -508,9 +436,9 @@ func newClusterCmd(c *config) *cobra.Command { clusterCmd.AddCommand(clusterMachineCmd) clusterCmd.AddCommand(clusterLogsCmd) clusterCmd.AddCommand(clusterIssuesCmd) - clusterCmd.AddCommand(clusterSplunkConfigManifestCmd) clusterCmd.AddCommand(clusterDNSManifestCmd) clusterCmd.AddCommand(clusterMonitoringSecretCmd) + clusterCmd.AddCommand(newClusterAuditCmd(c)) return clusterCmd } @@ -559,8 +487,6 @@ func (c *config) clusterCreate() error { disablePodSecurityPolicies = pointer.Pointer(viper.GetBool("disable-pod-security-policies")) } - audit := viper.GetString("audit") - labels := viper.GetStringSlice("labels") // FIXME helper and validation @@ -623,11 +549,6 @@ func (c *config) clusterCreate() error { log.Fatalf("provided cri:%s is not supported, only docker or containerd at the moment", cri) } - auditConfig, ok := auditConfigOptions[audit] - if !ok { - return fmt.Errorf("audit value %s is not supported; choose one of %v", audit, auditConfigOptions.Names(false)) - } - var customDefaultStorageClass *models.V1CustomDefaultStorageClass if viper.IsSet("default-storage-class") { class := viper.GetString("default-storage-class") @@ -662,7 +583,6 @@ func (c *config) clusterCreate() error { DefaultPodSecurityStandard: defaultPodSecurityStandard, DisablePodSecurityPolicies: disablePodSecurityPolicies, }, - Audit: auditConfig.Config, Maintenance: &models.V1Maintenance{ TimeWindow: &models.V1MaintenanceTimeWindow{ Begin: &maintenanceBegin, @@ -1326,16 +1246,6 @@ func (c *config) updateCluster(args []string) error { } cur.Kubernetes = k8s - - if viper.IsSet("audit") { - audit := viper.GetString("audit") - auditConfig, ok := auditConfigOptions[audit] - if !ok { - return fmt.Errorf("audit value %s is not supported; choose one of %v", audit, auditConfigOptions.Names(false)) - } - cur.Audit = auditConfig.Config - } - cur.EgressRules = makeEgressRules(egress) if viper.IsSet("enable-node-local-dns") { @@ -1581,56 +1491,6 @@ func (c *config) clusterInputs() error { return output.New().Print(sc) } -func (c *config) clusterSplunkConfigManifest() error { - secret := corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "splunk-config", Namespace: "kube-system"}, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{}, - Data: map[string][]byte{}, - } - if viper.IsSet("token") { - secret.StringData["hecToken"] = viper.GetString("token") - } - if viper.IsSet("index") { - secret.StringData["index"] = viper.GetString("index") - } - if viper.IsSet("hechost") { - secret.StringData["hecHost"] = viper.GetString("hechost") - } - if viper.IsSet("hecport") { - secret.StringData["hecPort"] = strconv.Itoa(viper.GetInt("hecport")) - } - if viper.IsSet("tls") { - if !viper.IsSet("cafile") && !viper.IsSet("cabase64") { - return fmt.Errorf("you need to supply a ca certificate when using TLS") - } - secret.StringData["tlsEnabled"] = strconv.FormatBool(viper.GetBool("tls")) - } - if viper.IsSet("cafile") { - if viper.IsSet("cabase64") { - return fmt.Errorf("specify ca certificate either through cafile or through cabase64, do not use both flags") - } - hecCAFile, err := os.ReadFile(viper.GetString("cafile")) - if err != nil { - return err - } - secret.StringData["hecCAFile"] = string(hecCAFile) - } - if viper.IsSet("cabase64") { - hecCAFileString := viper.GetString("cabase64") - _, err := base64.StdEncoding.DecodeString(hecCAFileString) - if err != nil { - return fmt.Errorf("unable to decode ca file string:%w", err) - } - secret.Data["hecCAFile"] = []byte(hecCAFileString) - } - - helper.MustPrintKubernetesResource(secret) - - return nil -} - func (c *config) clusterDNSManifest(args []string) error { ci, err := c.clusterID("dns-manifest", args) if err != nil { diff --git a/cmd/cluster_audit.go b/cmd/cluster_audit.go new file mode 100644 index 0000000..ef7a5aa --- /dev/null +++ b/cmd/cluster_audit.go @@ -0,0 +1,264 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/fi-ts/cloud-go/api/client/cluster" + "github.com/fi-ts/cloud-go/api/models" + "github.com/metal-stack/metal-lib/pkg/genericcli" + "github.com/metal-stack/metal-lib/pkg/pointer" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type auditCmd struct { + c *config +} + +func newClusterAuditCmd(c *config) *cobra.Command { + w := auditCmd{ + c: c, + } + + clusterAuditCmd := &cobra.Command{ + Use: "audit --cluster-id=", + Short: "configure a cluster's kube-apiserver audit configuration", + Long: `audit logs are captured through a webhook and buffered for up to 1GB next to the cluster's kube-apiserver. multiple backends are supported and can run simultaneously: +- Splunk +- Cluster Forwarding (not recommended for production use)`, + RunE: func(cmd *cobra.Command, args []string) error { + if viper.IsSet("disabled") { + return w.disable() + } + return fmt.Errorf("no command specified") + }, + PreRun: bindPFlags, + } + modeCmd := &cobra.Command{ + Use: "mode --cluster-id=", + Short: "set the audit webhook mode for this cluster", + Long: `the webhook mode of the cluster, one of: +- batch: Buffer events and asynchronously process them in batches. +- blocking: Block API server responses on processing each individual event. +- blocking-strict: Same as blocking, but when there is a failure during audit logging at the RequestReceived stage, the whole request to the kube-apiserver fails. This is the default. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return w.mode(args) + }, + ValidArgs: []string{ + "batch\tBuffer events and asynchronously process them in batches.", + "blocking\tBlock API server responses on processing each individual event.", + "blocking-strict\tSame as blocking, but when there is a failure during audit logging at the RequestReceived stage, the whole request to the kube-apiserver fails. This is the default.", + }, + PreRun: bindPFlags, + } + policyCmd := &cobra.Command{ + Use: "policy --cluster-id=", + Aliases: []string{"pol"}, + Short: "manage the audit policy for this cluster", + RunE: func(cmd *cobra.Command, args []string) error { + return w.auditPolicy() + }, + PreRun: bindPFlags, + } + splunkCmd := &cobra.Command{ + Use: "splunk --cluster-id=", + Short: "configure splunk as an audit backend, if enabled without any specific configuration, the provider's default configuration will be used", + RunE: func(cmd *cobra.Command, args []string) error { + return w.splunk() + }, + PreRun: bindPFlags, + } + clusterForwardingCmd := &cobra.Command{ + Use: "cluster-forwarding --cluster-id=", + Short: "configure forwarding the audit logs to an audittailer pod in the cluster (not recommended for production, see long help text)", + Long: "the approach has several downsides such as dependency on the stability of the VPN, possible corruption of the audit logs through malicious users in the cluster, etc. therefore this backend is not a recommended for production use-cases.", + RunE: func(cmd *cobra.Command, args []string) error { + return w.clusterForwarding() + }, + PreRun: bindPFlags, + } + + clusterAuditCmd.Flags().Bool("disabled", false, "disables the entire audit functionality, enable again with --disabled=false, requires --yes-i-really-mean-it flag") + + clusterAuditCmd.PersistentFlags().String("cluster-id", "", "the id of the cluster to apply the audit configuration to") + genericcli.Must(clusterAuditCmd.MarkPersistentFlagRequired("cluster-id")) + genericcli.Must(clusterAuditCmd.RegisterFlagCompletionFunc("cluster-id", c.comp.ClusterListCompletion)) + + policyCmd.Flags().String("from-file", "", "reads and applies the audit policy from the given file path") + policyCmd.Flags().Bool("remove", false, "removes the custom audit policy") + policyCmd.Flags().Bool("show", false, "shows the current audit policy") + policyCmd.MarkFlagsMutuallyExclusive("from-file", "remove", "show") + policyCmd.MarkFlagsOneRequired("from-file", "remove", "show") + + clusterForwardingCmd.Flags().Bool("enabled", false, "enables cluster-forwarding audit backend for this cluster.") + + splunkCmd.Flags().Bool("enabled", false, "enables splunk audit backend for this cluster, if enabled without any specific settings, the provider-default splunk backend will be used.") + splunkCmd.Flags().String("host", "", "the splunk host to configure.") + splunkCmd.Flags().String("index", "", "the splunk index to configure.") + splunkCmd.Flags().String("port", "", "the splunk port to configure.") + splunkCmd.Flags().String("token", "", "the splunk token used to authenticate against the splunk endpoint.") + splunkCmd.Flags().String("ca", "", "the path to a ca used for tls connection to splunk endpoint.") + + clusterAuditCmd.AddCommand(modeCmd, policyCmd, splunkCmd, clusterForwardingCmd) + + return clusterAuditCmd +} + +func (c *auditCmd) mode(args []string) error { + mode, err := genericcli.GetExactlyOneArg(args) + if err != nil { + return err + } + + _, err = c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: &models.V1Audit{ + WebhookMode: pointer.Pointer(mode), + }, + }), nil) + if err != nil { + return err + } + + return nil +} + +func (c *auditCmd) auditPolicy() error { + if viper.GetBool("show") { + resp, err := c.c.cloud.Cluster.GetAuditPolicy(cluster.NewGetAuditPolicyParams().WithID(viper.GetString("cluster-id")), nil) + if err != nil { + return err + } + + fmt.Println(pointer.SafeDeref(resp.Payload.Raw)) + + return nil + } + + if viper.GetBool("remove") { + _, err := c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: &models.V1Audit{ + AuditPolicy: pointer.Pointer(""), + }, + }), nil) + if err != nil { + return err + } + + return nil + } + + if viper.IsSet("from-file") { + policy, err := os.ReadFile(viper.GetString("from-file")) + if err != nil { + return err + } + + _, err = c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: &models.V1Audit{ + AuditPolicy: pointer.Pointer(string(policy)), + }, + }), nil) + if err != nil { + return err + } + + return nil + } + + return fmt.Errorf("either --show, --remove or --from-file needs to be used") +} + +func (c *auditCmd) disable() error { + disabled := viper.GetBool("disabled") + if disabled && !viper.GetBool("yes-i-really-mean-it") { + return fmt.Errorf("disabling cluster auditing requires --yes-i-really-mean-it") + } + + _, err := c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: &models.V1Audit{ + Disabled: pointer.Pointer(disabled), + }, + }), nil) + if err != nil { + return err + } + + return nil +} + +func (c *auditCmd) splunk() error { + auditConfigration := &models.V1Audit{} + + if auditConfigration.Backends == nil { + auditConfigration.Backends = &models.V1AuditBackends{} + } + if auditConfigration.Backends.Splunk == nil { + auditConfigration.Backends.Splunk = &models.V1AuditBackendSplunk{} + } + + if viper.IsSet("enabled") { + auditConfigration.Backends.Splunk.Enabled = pointer.Pointer(viper.GetBool("enabled")) + } + if viper.IsSet("host") { + auditConfigration.Backends.Splunk.Host = pointer.Pointer(viper.GetString("host")) + } + if viper.IsSet("index") { + auditConfigration.Backends.Splunk.Index = pointer.Pointer(viper.GetString("index")) + } + if viper.IsSet("port") { + auditConfigration.Backends.Splunk.Port = pointer.Pointer(viper.GetString("port")) + } + if viper.IsSet("token") { + auditConfigration.Backends.Splunk.Token = pointer.Pointer(viper.GetString("token")) + } + if viper.IsSet("ca") { + ca, err := os.ReadFile(viper.GetString("ca")) + if err != nil { + return err + } + + auditConfigration.Backends.Splunk.TLS = pointer.Pointer(true) + auditConfigration.Backends.Splunk.Ca = pointer.Pointer(string(ca)) + } + + _, err := c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: auditConfigration, + }), nil) + if err != nil { + return err + } + + return nil +} + +func (c *auditCmd) clusterForwarding() error { + auditConfigration := &models.V1Audit{} + + if auditConfigration.Backends == nil { + auditConfigration.Backends = &models.V1AuditBackends{} + } + if auditConfigration.Backends.ClusterForwarding == nil { + auditConfigration.Backends.ClusterForwarding = &models.V1AuditBackendClusterForwarding{} + } + + if viper.IsSet("enabled") { + auditConfigration.Backends.ClusterForwarding.Enabled = pointer.Pointer(viper.GetBool("enabled")) + } + + _, err := c.c.cloud.Cluster.UpdateCluster(cluster.NewUpdateClusterParams().WithBody(&models.V1ClusterUpdateRequest{ + ID: pointer.Pointer(viper.GetString("cluster-id")), + Audit: auditConfigration, + }), nil) + if err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 6a35a00..4b2cf45 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/dcorbe/termui-dpc v0.0.0-20211125210512-9d2673a82dd6 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.16.0 - github.com/fi-ts/cloud-go v0.22.6 + github.com/fi-ts/cloud-go v0.22.7-0.20231204123403-a023def89959 github.com/gardener/gardener v1.65.0 github.com/gardener/machine-controller-manager v0.50.1 github.com/go-openapi/strfmt v0.21.7 diff --git a/go.sum b/go.sum index 09142b0..613f0a9 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/fi-ts/cloud-go v0.22.6 h1:5MnxxyeJwiJwRL+xIa8VwC2nbdpcTT1RU1B/EgINaIc= -github.com/fi-ts/cloud-go v0.22.6/go.mod h1:z4ZWkDa0EiaAfHy1iiv7vlgoZgJQEsAgQnrvG2cJksw= +github.com/fi-ts/cloud-go v0.22.7-0.20231204123403-a023def89959 h1:21eRz8HqcbICU3tUjhYWRDp9rXA+cLVjRtfZ9U4J0wQ= +github.com/fi-ts/cloud-go v0.22.7-0.20231204123403-a023def89959/go.mod h1:z4ZWkDa0EiaAfHy1iiv7vlgoZgJQEsAgQnrvG2cJksw= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=