-
Notifications
You must be signed in to change notification settings - Fork 93
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add support for embedded cluster updates (#4201)
* feat: add support for embedded cluster updates we now support embedded cluster upgrades together with the kots app upgrade. users can provide their own embedded cluster config object that is then applied to the cluster when it differs from the current active one. * trigger cluster upgrades on app version deploy not on poll * make embedded_cluster_status table * eventually exit cluster state update loop, only persist to DB on state change * trigger a single check of the cluster status at startup * don't prevent logging * only attempt to get embedded cluser installation state on embedded cluster systems * fix js missing property * unlock mutex, don't log about embedded-cluster on non-ec installs * when starting up, watch the embedded cluster state until it is Installed --------- Co-authored-by: Andrew Lavery <[email protected]>
- Loading branch information
1 parent
72d34c8
commit 8452ca5
Showing
18 changed files
with
482 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,3 +71,5 @@ spec: | |
| type: text | ||
| - name: branding_archive | ||
| type: text | ||
| - name: embeddedcluster_config | ||
| type: text | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| apiVersion: schemas.schemahero.io/v1alpha4 | ||
| kind: Table | ||
| metadata: | ||
| name: embedded-cluster-status | ||
| spec: | ||
| name: embedded_cluster_status | ||
| requires: [] | ||
| schema: | ||
| rqlite: | ||
| strict: true | ||
| primaryKey: | ||
| - updated_at | ||
| columns: | ||
| - name: updated_at | ||
| type: integer | ||
| - name: status | ||
| type: text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| package embeddedcluster | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" | ||
| "github.com/replicatedhq/kots/pkg/logger" | ||
| "github.com/replicatedhq/kots/pkg/store" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| var stateMut = sync.Mutex{} | ||
|
|
||
| // MaybeStartClusterUpgrade checks if the embedded cluster is in a state that requires an upgrade. If so, | ||
| // it starts the upgrade process. We only start an upgrade if the following conditions are met: | ||
| // - The app has an embedded cluster configuration. | ||
| // - The app embedded cluster configuration differs from the current embedded cluster config. | ||
| func MaybeStartClusterUpgrade(ctx context.Context, client kubernetes.Interface, store store.Store, conf *v1beta1.Config) error { | ||
| if conf == nil { | ||
| return nil | ||
| } | ||
|
|
||
| isEC, err := IsEmbeddedCluster(client) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to check if embedded cluster is enabled: %w", err) | ||
| } | ||
| if !isEC { | ||
| return nil | ||
| } | ||
|
|
||
| spec := conf.Spec | ||
| if upgrade, err := RequiresUpgrade(ctx, spec); err != nil { | ||
| return fmt.Errorf("failed to check if upgrade is required: %w", err) | ||
| } else if !upgrade { | ||
| return nil | ||
| } | ||
| if err := startClusterUpgrade(ctx, spec); err != nil { | ||
| return fmt.Errorf("failed to start cluster upgrade: %w", err) | ||
| } | ||
|
|
||
| go watchClusterState(ctx, store) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // InitClusterState initializes the cluster state in the database. This should be called when the | ||
| // server launches. | ||
| func InitClusterState(ctx context.Context, client kubernetes.Interface, store store.Store) error { | ||
| isEC, err := IsEmbeddedCluster(client) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to check if embedded cluster is enabled: %w", err) | ||
| } | ||
| if isEC { | ||
| go watchClusterState(ctx, store) | ||
| return nil | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // watchClusterState checks the status of the installation object and updates the cluster state | ||
| // after the cluster state has been 'installed' for 30 seconds, it will exit the loop. | ||
| // this function is blocking and should be run in a goroutine. | ||
| // if it is called multiple times, only one instance will run. | ||
| func watchClusterState(ctx context.Context, store store.Store) { | ||
| stateMut.Lock() | ||
| defer stateMut.Unlock() | ||
| numReady := 0 | ||
| lastState := "" | ||
| for numReady < 6 { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return | ||
| case <-time.After(time.Second * 5): | ||
| } | ||
| state, err := updateClusterState(ctx, store, lastState) | ||
| if err != nil { | ||
| logger.Errorf("embeddedcluster monitor: fail updating state: %v", err) | ||
| } | ||
|
|
||
| if state == v1beta1.InstallationStateInstalled { | ||
| numReady++ | ||
| } else { | ||
| numReady = 0 | ||
| } | ||
| lastState = state | ||
| } | ||
| } | ||
|
|
||
| // updateClusterState updates the cluster state in the database. Gets the state from the cluster | ||
| // by reading the latest embedded cluster installation CRD. | ||
| // If the lastState is the same as the current state, it will not update the database. | ||
| func updateClusterState(ctx context.Context, store store.Store, lastState string) (string, error) { | ||
| installation, err := GetCurrentInstallation(ctx) | ||
| if err != nil { | ||
| return "", fmt.Errorf("failed to get current installation: %w", err) | ||
| } | ||
| state := v1beta1.InstallationStateUnknown | ||
| if installation.Status.State != "" { | ||
| state = installation.Status.State | ||
| } | ||
| // only update the state if it has changed | ||
| if state != lastState { | ||
| if err := store.SetEmbeddedClusterState(state); err != nil { | ||
| return "", fmt.Errorf("failed to update embedded cluster state: %w", err) | ||
| } | ||
| } | ||
| return state, nil | ||
| } |
Oops, something went wrong.