Skip to content

Commit

Permalink
feat: add support for embedded cluster updates
Browse files Browse the repository at this point in the history
TBD
  • Loading branch information
ricardomaraschini committed Dec 4, 2023
1 parent d505141 commit aaa2496
Show file tree
Hide file tree
Showing 9 changed files with 255 additions and 5 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ require (
github.com/go-ldap/ldap/v3 v3.4.4 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.4 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,7 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
Expand Down Expand Up @@ -1856,6 +1857,7 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
Expand All @@ -1872,6 +1874,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down
2 changes: 2 additions & 0 deletions migrations/tables/app_version.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,5 @@ spec:
type: text
- name: branding_archive
type: text
- name: embeddedcluster_config
type: text
72 changes: 72 additions & 0 deletions pkg/embeddedcluster/util.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package embeddedcluster

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"sort"
"time"

embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
"github.com/replicatedhq/kots/pkg/k8sutil"
Expand All @@ -13,6 +17,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

const configMapName = "embedded-cluster-config"
Expand Down Expand Up @@ -58,6 +64,23 @@ func ClusterID(client kubernetes.Interface) (string, error) {
return configMap.Data["embedded-cluster-id"], nil
}

// RequiresUpgrade returns true if the provided configuration differs from the latest active configuration.
func RequiresUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec) (bool, error) {
curcfg, err := ClusterConfig(ctx)
if err != nil {
return false, fmt.Errorf("failed to get current cluster config: %w", err)
}
serializedCur, err := json.Marshal(curcfg)
if err != nil {
return false, err
}
serializedNew, err := json.Marshal(newcfg)
if err != nil {
return false, err
}
return !bytes.Equal(serializedCur, serializedNew), nil
}

// ClusterConfig will get the list of installations, find the latest installation, and get that installation's config
func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, error) {
clientConfig, err := k8sutil.GetClusterConfig()
Expand All @@ -67,6 +90,10 @@ func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, err

scheme := runtime.NewScheme()
embeddedclusterv1beta1.AddToScheme(scheme)
k8slogger := zap.New(func(o *zap.Options) {
o.DestWriter = io.Discard
})
log.SetLogger(k8slogger)

kbClient, err := kbclient.New(clientConfig, kbclient.Options{
Scheme: scheme,
Expand All @@ -89,3 +116,48 @@ func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, err
latest := installationList.Items[0]
return latest.Spec.Config, nil
}

// StartClusterUpdate will create a new installation with the provided config.
func StartClusterUpdate(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec) error {
clientConfig, err := k8sutil.GetClusterConfig()
if err != nil {
return fmt.Errorf("failed to get cluster config: %w", err)
}
scheme := runtime.NewScheme()
embeddedclusterv1beta1.AddToScheme(scheme)
k8slogger := zap.New(func(o *zap.Options) {
o.DestWriter = io.Discard
})
log.SetLogger(k8slogger)
kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme})
if err != nil {
return fmt.Errorf("failed to get kubebuilder client: %w", err)
}
var installationList embeddedclusterv1beta1.InstallationList
err = kbClient.List(ctx, &installationList, &kbclient.ListOptions{})
if err != nil {
return fmt.Errorf("failed to list installations: %w", err)
}
sort.Slice(installationList.Items, func(i, j int) bool {
return installationList.Items[i].ObjectMeta.CreationTimestamp.After(installationList.Items[j].ObjectMeta.CreationTimestamp.Time)
})
if len(installationList.Items) == 0 {
return fmt.Errorf("no installations found")
}
latest := installationList.Items[0]
newins := embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
ClusterID: latest.Spec.ClusterID,
MetricsBaseURL: latest.Spec.MetricsBaseURL,
AirGap: latest.Spec.AirGap,
Config: &newcfg,
},
}
if err := kbClient.Create(ctx, &newins); err != nil {
return fmt.Errorf("failed to create installation: %w", err)
}
return nil
}
118 changes: 118 additions & 0 deletions pkg/handlers/embedded_cluster_update.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package handlers

import (
"net/http"

"github.com/gorilla/mux"

"github.com/replicatedhq/kots/pkg/embeddedcluster"
"github.com/replicatedhq/kots/pkg/k8sutil"
"github.com/replicatedhq/kots/pkg/logger"
"github.com/replicatedhq/kots/pkg/store"
)

// EmbeddedClusterUpdateCheckResponse is the response of the cluster update check. Contains the app
// slug the the verison to upgrade to.
type EmbeddedClusterUpdateCheckResponse struct {
Slug string `json:"app"`
Version string `json:"version"`
}

// EmbeddedClusterUpdateCheck is a handler func that checks if an update is available for the embedded
// cluster and returns the version to upgrade to if so.
func (h *Handler) EmbeddedClusterUpdateCheck(w http.ResponseWriter, r *http.Request) {
errout := func(err error) {
logger.Error(err)
w.WriteHeader(http.StatusInternalServerError)
}
ctx := r.Context()
appSlug := mux.Vars(r)["appSlug"]
client, err := k8sutil.GetClientset()
if err != nil {
errout(err)
return
}
if isEmbeddedCluster, err := embeddedcluster.IsEmbeddedCluster(client); err != nil {
errout(err)
return
} else if !isEmbeddedCluster {
w.WriteHeader(http.StatusNoContent)
return
}
app, err := store.GetStore().GetAppFromSlug(appSlug)
if err != nil {
errout(err)
return
}
version, err := store.GetStore().GetAppVersion(app.ID, app.CurrentSequence)
if err != nil {
errout(err)
return
}
if version.KOTSKinds == nil || version.KOTSKinds.EmbeddedClusterConfig == nil {
w.WriteHeader(http.StatusNoContent)
return
}
upgrade, err := embeddedcluster.RequiresUpgrade(ctx, version.KOTSKinds.EmbeddedClusterConfig.Spec)
if err != nil {
errout(err)
return
}
if !upgrade {
w.WriteHeader(http.StatusNoContent)
return
}
JSON(w, http.StatusOK, EmbeddedClusterUpdateCheckResponse{
Slug: app.Slug,
Version: version.KOTSKinds.EmbeddedClusterConfig.Spec.Version,
})
}

func (h *Handler) StartClusterUpdate(w http.ResponseWriter, r *http.Request) {
errout := func(err error) {
logger.Error(err)
w.WriteHeader(http.StatusInternalServerError)
}
ctx := r.Context()
appSlug := mux.Vars(r)["appSlug"]
client, err := k8sutil.GetClientset()
if err != nil {
errout(err)
return
}
if isEmbeddedCluster, err := embeddedcluster.IsEmbeddedCluster(client); err != nil {
errout(err)
return
} else if !isEmbeddedCluster {
w.WriteHeader(http.StatusNoContent)
return
}
app, err := store.GetStore().GetAppFromSlug(appSlug)
if err != nil {
errout(err)
return
}
version, err := store.GetStore().GetAppVersion(app.ID, app.CurrentSequence)
if err != nil {
errout(err)
return
}
if version.KOTSKinds == nil || version.KOTSKinds.EmbeddedClusterConfig == nil {
w.WriteHeader(http.StatusNoContent)
return
}
upgrade, err := embeddedcluster.RequiresUpgrade(ctx, version.KOTSKinds.EmbeddedClusterConfig.Spec)
if err != nil {
errout(err)
return
}
if !upgrade {
w.WriteHeader(http.StatusNoContent)
return
}
if err := embeddedcluster.StartClusterUpdate(ctx, version.KOTSKinds.EmbeddedClusterConfig.Spec); err != nil {
errout(err)
return
}
w.WriteHeader(http.StatusOK)
}
4 changes: 4 additions & 0 deletions pkg/handlers/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,10 @@ func RegisterSessionAuthRoutes(r *mux.Router, kotsStore store.Store, handler KOT
HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetEmbeddedClusterNode))
r.Name("GetEmbeddedClusterRoles").Path("/api/v1/embedded-cluster/roles").Methods("GET").
HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetEmbeddedClusterRoles))
r.Name("EmbeddedClusterUpdateCheck").Path("/api/v1/embedded-cluster/{appSlug}/updatecheck").Methods("GET").
HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.EmbeddedClusterUpdateCheck))
r.Name("EmbeddedClusterStartUpdate").Path("/api/v1/embedded-cluster/{appSlug}/update").Methods("POST").
HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.StartClusterUpdate))

// Prometheus
r.Name("SetPrometheusAddress").Path("/api/v1/prometheus").Methods("POST").
Expand Down
2 changes: 2 additions & 0 deletions pkg/handlers/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ type KOTSHandler interface {
GetEmbeddedClusterNodes(w http.ResponseWriter, r *http.Request)
GetEmbeddedClusterNode(w http.ResponseWriter, r *http.Request)
GetEmbeddedClusterRoles(w http.ResponseWriter, r *http.Request)
EmbeddedClusterUpdateCheck(w http.ResponseWriter, r *http.Request)
StartClusterUpdate(w http.ResponseWriter, r *http.Request)

// Prometheus
SetPrometheusAddress(w http.ResponseWriter, r *http.Request)
Expand Down
30 changes: 30 additions & 0 deletions pkg/kotsutil/kots.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/blang/semver"
"github.com/pkg/errors"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
"github.com/replicatedhq/kots/pkg/archives"
"github.com/replicatedhq/kots/pkg/binaries"
"github.com/replicatedhq/kots/pkg/buildversion"
Expand Down Expand Up @@ -51,6 +52,7 @@ func init() {
velerov1.AddToScheme(scheme.Scheme)
kurlscheme.AddToScheme(scheme.Scheme)
applicationv1beta1.AddToScheme(scheme.Scheme)
embeddedclusterv1beta1.AddToScheme(scheme.Scheme)
}

var (
Expand Down Expand Up @@ -105,6 +107,8 @@ type KotsKinds struct {
Installer *kurlv1beta1.Installer

LintConfig *kotsv1beta1.LintConfig

EmbeddedClusterConfig *embeddedclusterv1beta1.Config
}

func IsKotsKind(apiVersion string, kind string) bool {
Expand All @@ -129,6 +133,10 @@ func IsKotsKind(apiVersion string, kind string) bool {
if apiVersion == "kurl.sh/v1beta1" {
return true
}
// In addition to kotskinds, we exclude the embedded cluster configuration.
if apiVersion == "embeddedcluster.replicated.com/v1beta1" {
return true
}
// In addition to kotskinds, we exclude the application crd for now
if apiVersion == "app.k8s.io/v1beta1" {
return true
Expand Down Expand Up @@ -448,6 +456,14 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) {
}
}

if g == "embeddedcluster.replicated.com" && v == "v1beta1" && k == "Config" {
var b bytes.Buffer
if err := s.Encode(o.EmbeddedClusterConfig, &b); err != nil {
return "", errors.Wrap(err, "failed to encode embedded cluster config")
}
return string(b.Bytes()), nil
}

return "", errors.Errorf("unknown gvk %s/%s, Kind=%s", g, v, k)
}

Expand Down Expand Up @@ -528,6 +544,8 @@ func (k *KotsKinds) addKotsKinds(content []byte) error {
k.Installer = decoded.(*kurlv1beta1.Installer)
case "app.k8s.io/v1beta1, Kind=Application":
k.Application = decoded.(*applicationv1beta1.Application)
case "embeddedcluster.replicated.com/v1beta1, Kind=Config":
k.EmbeddedClusterConfig = decoded.(*embeddedclusterv1beta1.Config)
}
}

Expand Down Expand Up @@ -913,6 +931,18 @@ func LoadLicenseFromBytes(data []byte) (*kotsv1beta1.License, error) {
return obj.(*kotsv1beta1.License), nil
}

func LoadEmbeddedClusterConfigFromBytes(data []byte) (*embeddedclusterv1beta1.Config, error) {
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, gvk, err := decode([]byte(data), nil, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to decode embedded cluster config data")
}
if gvk.Group != "embeddedcluster.replicated.com" || gvk.Version != "v1beta1" || gvk.Kind != "Config" {
return nil, errors.Errorf("unexpected GVK: %s", gvk.String())
}
return obj.(*embeddedclusterv1beta1.Config), nil
}

func LoadConfigValuesFromFile(configValuesFilePath string) (*kotsv1beta1.ConfigValues, error) {
configValuesData, err := ioutil.ReadFile(configValuesFilePath)
if err != nil {
Expand Down
Loading

0 comments on commit aaa2496

Please sign in to comment.