-
Notifications
You must be signed in to change notification settings - Fork 55
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
✨ Add upgrade E2E #1003
✨ Add upgrade E2E #1003
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would it make sense to put this under the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This could be done later (if we desire) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
help="pre-upgrade-setup.sh is used to create some basic resources | ||
which will later be used in upgrade testing. | ||
|
||
Usage: | ||
post-upgrade-checks.sh [TEST_CATALOG_IMG] [TEST_CATALOG_NAME] [TEST_CLUSTER_EXTENSION_NAME] | ||
" | ||
|
||
if [[ "$#" -ne 3 ]]; then | ||
echo "Illegal number of arguments passed" | ||
echo "${help}" | ||
exit 1 | ||
fi | ||
|
||
TEST_CATALOG_IMG=$1 | ||
TEST_CLUSTER_CATALOG_NAME=$2 | ||
TEST_CLUSTER_EXTENSION_NAME=$3 | ||
|
||
kubectl apply -f - << EOF | ||
apiVersion: catalogd.operatorframework.io/v1alpha1 | ||
kind: ClusterCatalog | ||
metadata: | ||
name: ${TEST_CLUSTER_CATALOG_NAME} | ||
spec: | ||
source: | ||
type: image | ||
image: | ||
ref: ${TEST_CATALOG_IMG} | ||
pollInterval: 24h | ||
insecureSkipTLSVerify: true | ||
EOF | ||
|
||
|
||
kubectl apply -f - << EOF | ||
apiVersion: olm.operatorframework.io/v1alpha1 | ||
kind: ClusterExtension | ||
metadata: | ||
name: ${TEST_CLUSTER_EXTENSION_NAME} | ||
spec: | ||
installNamespace: default | ||
packageName: prometheus | ||
version: 1.0.0 | ||
serviceAccount: | ||
name: default | ||
EOF | ||
|
||
kubectl wait --for=condition=Unpacked --timeout=60s ClusterCatalog $TEST_CLUSTER_CATALOG_NAME | ||
kubectl wait --for=condition=Installed --timeout=60s ClusterExtension $TEST_CLUSTER_EXTENSION_NAME |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,135 @@ | ||
package upgradee2e | ||
|
||
import ( | ||
"bufio" | ||
"context" | ||
"fmt" | ||
"strings" | ||
"testing" | ||
"time" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
appsv1 "k8s.io/api/apps/v1" | ||
corev1 "k8s.io/api/core/v1" | ||
apimeta "k8s.io/apimachinery/pkg/api/meta" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/labels" | ||
"k8s.io/apimachinery/pkg/types" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1" | ||
|
||
ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" | ||
) | ||
|
||
func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { | ||
t.Log("Starting checks after OLM upgrade") | ||
ctx := context.Background() | ||
|
||
managerLabelSelector := labels.Set{"control-plane": "controller-manager"} | ||
|
||
t.Log("Checking that the controller-manager deployment is updated") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
var managerDeployments appsv1.DeploymentList | ||
assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) | ||
assert.Len(ct, managerDeployments.Items, 1) | ||
managerDeployment := managerDeployments.Items[0] | ||
|
||
assert.True(ct, | ||
managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, | ||
) | ||
}, time.Minute, time.Second) | ||
|
||
var managerPods corev1.PodList | ||
t.Log("Waiting for only one controller-manager Pod to remain") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) | ||
assert.Len(ct, managerPods.Items, 1) | ||
}, time.Minute, time.Second) | ||
|
||
t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") | ||
// Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes | ||
logCtx, cancel := context.WithTimeout(ctx, time.Minute) | ||
defer cancel() | ||
substring := fmt.Sprintf(`"ClusterExtension": {"name":"%s"}`, testClusterExtensionName) | ||
found, err := watchPodLogsForSubstring(logCtx, &managerPods.Items[0], "manager", substring) | ||
require.NoError(t, err) | ||
require.True(t, found) | ||
|
||
t.Log("Checking that the ClusterCatalog is unpacked") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
var clusterCatalog catalogdv1alpha1.ClusterCatalog | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) | ||
cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogdv1alpha1.TypeUnpacked) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, metav1.ConditionTrue, cond.Status) | ||
assert.Equal(ct, catalogdv1alpha1.ReasonUnpackSuccessful, cond.Reason) | ||
}, time.Minute, time.Second) | ||
|
||
t.Log("Checking that the ClusterExtension is installed") | ||
var clusterExtension ocv1alpha1.ClusterExtension | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) | ||
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, metav1.ConditionTrue, cond.Status) | ||
assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) | ||
assert.Contains(ct, cond.Message, "Instantiated bundle") | ||
assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle) | ||
assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle.Version) | ||
}, time.Minute, time.Second) | ||
|
||
previousVersion := clusterExtension.Status.InstalledBundle.Version | ||
|
||
t.Log("Updating the ClusterExtension to change version") | ||
// Make sure that after we upgrade OLM itself we can still reconcile old objects if we change them | ||
clusterExtension.Spec.Version = "1.0.1" | ||
require.NoError(t, c.Update(ctx, &clusterExtension)) | ||
|
||
t.Log("Checking that the ClusterExtension installs successfully") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) | ||
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) | ||
assert.Contains(ct, cond.Message, "Instantiated bundle") | ||
assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.ResolvedBundle) | ||
assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.InstalledBundle) | ||
assert.NotEqual(ct, previousVersion, clusterExtension.Status.InstalledBundle.Version) | ||
}, time.Minute, time.Second) | ||
} | ||
|
||
func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container, substring string) (bool, error) { | ||
podLogOpts := corev1.PodLogOptions{ | ||
Follow: true, | ||
Container: container, | ||
} | ||
|
||
req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) | ||
podLogs, err := req.Stream(ctx) | ||
if err != nil { | ||
return false, err | ||
} | ||
defer podLogs.Close() | ||
|
||
scanner := bufio.NewScanner(podLogs) | ||
for scanner.Scan() { | ||
line := scanner.Text() | ||
|
||
if strings.Contains(line, substring) { | ||
return true, nil | ||
} | ||
} | ||
|
||
return false, scanner.Err() | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
package upgradee2e | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
"testing" | ||
|
||
"k8s.io/client-go/kubernetes" | ||
ctrl "sigs.k8s.io/controller-runtime" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
"github.com/operator-framework/operator-controller/pkg/scheme" | ||
) | ||
|
||
const ( | ||
testClusterCatalogNameEnv = "TEST_CLUSTER_CATALOG_NAME" | ||
testClusterExtensionNameEnv = "TEST_CLUSTER_EXTENSION_NAME" | ||
) | ||
|
||
var ( | ||
c client.Client | ||
kclientset kubernetes.Interface | ||
|
||
testClusterCatalogName string | ||
testClusterExtensionName string | ||
) | ||
|
||
func TestMain(m *testing.M) { | ||
var ok bool | ||
testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) | ||
if !ok { | ||
fmt.Printf("%q is not set", testClusterCatalogNameEnv) | ||
os.Exit(1) | ||
} | ||
testClusterExtensionName, ok = os.LookupEnv(testClusterExtensionNameEnv) | ||
if !ok { | ||
fmt.Printf("%q is not set", testClusterExtensionNameEnv) | ||
os.Exit(1) | ||
} | ||
|
||
cfg := ctrl.GetConfigOrDie() | ||
|
||
var err error | ||
c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) | ||
if err != nil { | ||
fmt.Printf("failed to create client: %s\n", err) | ||
os.Exit(1) | ||
} | ||
|
||
kclientset, err = kubernetes.NewForConfig(cfg) | ||
if err != nil { | ||
fmt.Printf("failed to create kubernetes clientset: %s\n", err) | ||
os.Exit(1) | ||
} | ||
|
||
os.Exit(m.Run()) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we also run the standard e2e after an upgrade as well?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We were considering this, but decided not to because: