diff --git a/test/e2e/pkg/work_eviction_test.go b/test/e2e/pkg/work_eviction_test.go new file mode 100644 index 00000000..8f108738 --- /dev/null +++ b/test/e2e/pkg/work_eviction_test.go @@ -0,0 +1,191 @@ +package e2e_test + +import ( + "fmt" + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/openshift-online/maestro/pkg/api/openapi" +) + +// go test -v ./test/e2e/pkg -args -api-server=$api_server -consumer-name=$consumer.Name -consumer-kubeconfig=$consumer_kubeconfig -ginkgo.focus "Applied Manifestwork Eviction" +var _ = Describe("Applied Manifestwork Eviction", Ordered, Label("e2e-tests-work-eviction"), func() { + var resource *openapi.Resource + var maestroServerReplicas int + + Context("Agent Appliedmanifestwork Eviction Grace Period Tests", func() { + It("post the nginx resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, 1) + var resp *http.Response + var err error + resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusCreated)) + Expect(*resource.Id).ShouldNot(BeEmpty()) + Expect(*resource.Version).To(Equal(int32(1))) + + Eventually(func() error { + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + if err != nil { + return err + } + if *deploy.Spec.Replicas != 1 { + return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + } + return nil + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("shut down maestro server", func() { + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro").Get(ctx, "maestro", metav1.GetOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + maestroServerReplicas = int(*deploy.Spec.Replicas) + + // patch maestro server replicas to 0 + deploy, err = consumer.ClientSet.AppsV1().Deployments("maestro").Patch(ctx, "maestro", types.MergePatchType, []byte(`{"spec":{"replicas":0}}`), metav1.PatchOptions{ + FieldManager: "testConsumer.ClientSet", + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(*deploy.Spec.Replicas).To(Equal(int32(0))) + + // ensure no running maestro server pods + Eventually(func() error { + pods, err := consumer.ClientSet.CoreV1().Pods("maestro").List(ctx, metav1.ListOptions{ + LabelSelector: "app=maestro", + }) + if err != nil { + return err + } + if len(pods.Items) > 0 { + return fmt.Errorf("maestro server pods still running") + } + return nil + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("restart maestro agent", func() { + // patch maestro agent to restart it + restartPatchData := fmt.Sprintf(`{"spec": {"template": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": "%s"}}}}}`, time.Now().Format("20060102150405")) + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro-agent").Patch(ctx, "maestro-agent", types.StrategicMergePatchType, []byte(restartPatchData), metav1.PatchOptions{ + FieldManager: "testconsumer.ClientSet", + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(*deploy.Spec.Replicas).To(Equal(int32(1))) + + // ensure maestro agent is ready + Eventually(func() error { + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro-agent").Get(ctx, "maestro-agent", metav1.GetOptions{}) + if err != nil { + return err + } + if deploy != nil && deploy.Spec.Replicas != nil && *deploy.Spec.Replicas == deploy.Status.ReadyReplicas { + return nil + } + return fmt.Errorf("maestro agent deploy not ready") + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("ensure the nginx resource is evicted", func() { + Eventually(func() error { + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("nginx deployment still exists") + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("start maestro server", func() { + // patch maestro server replicas to 1 + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro").Patch(ctx, "maestro", types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, maestroServerReplicas)), metav1.PatchOptions{ + FieldManager: "testConsumer.ClientSet", + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(*deploy.Spec.Replicas).To(Equal(int32(maestroServerReplicas))) + + // ensure maestro server pod is up and running + Eventually(func() error { + pods, err := consumer.ClientSet.CoreV1().Pods("maestro").List(ctx, metav1.ListOptions{ + LabelSelector: "app=maestro", + }) + if err != nil { + return err + } + if len(pods.Items) != maestroServerReplicas { + return fmt.Errorf("unexpected maestro server pod count, expected %d, got %d", maestroServerReplicas, len(pods.Items)) + } + for _, pod := range pods.Items { + if pod.Status.Phase != "Running" { + return fmt.Errorf("maestro server pod not in running state") + } + if pod.Status.ContainerStatuses[0].State.Running == nil { + return fmt.Errorf("maestro server container not in running state") + } + } + return nil + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("restart maestro agent again", func() { + // patch maestro agent to restart it + restartPatchData := fmt.Sprintf(`{"spec": {"template": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": "%s"}}}}}`, time.Now().Format("20060102150405")) + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro-agent").Patch(ctx, "maestro-agent", types.StrategicMergePatchType, []byte(restartPatchData), metav1.PatchOptions{ + FieldManager: "testconsumer.ClientSet", + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(*deploy.Spec.Replicas).To(Equal(int32(1))) + + // ensure maestro agent is ready + Eventually(func() error { + deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro-agent").Get(ctx, "maestro-agent", metav1.GetOptions{}) + if err != nil { + return err + } + if deploy != nil && deploy.Spec.Replicas != nil && *deploy.Spec.Replicas == deploy.Status.ReadyReplicas { + return nil + } + return fmt.Errorf("maestro agent deploy not ready") + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("ensure the nginx resource is resynced", func() { + Eventually(func() error { + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + if err != nil { + return err + } + if *deploy.Spec.Replicas != 1 { + return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + } + return nil + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + + It("delete the nginx resource", func() { + resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource.Id).Execute() + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + + Eventually(func() error { + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("nginx deployment still exists") + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }) + }) +}) + diff --git a/test/e2e/setup/e2e_setup.sh b/test/e2e/setup/e2e_setup.sh index 1f488d7b..8950585c 100755 --- a/test/e2e/setup/e2e_setup.sh +++ b/test/e2e/setup/e2e_setup.sh @@ -209,7 +209,7 @@ EOF # create secret containing the client certs to mqtt broker and patch the maestro-agent deployment kubectl create secret generic maestro-agent-certs -n ${agent_namespace} --from-file=ca.crt=${certDir}/ca.crt --from-file=client.crt=${certDir}/agent-client.crt --from-file=client.key=${certDir}/agent-client.key -kubectl patch deploy/maestro-agent -n ${agent_namespace} --type='json' -p='[{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mqtt-certs","secret":{"secretName":"maestro-agent-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mqtt-certs","mountPath":"/secrets/mqtt-certs"}}]' +kubectl patch deploy/maestro-agent -n ${agent_namespace} --type='json' -p='[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--appliedmanifestwork-eviction-grace-period=30s"},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mqtt-certs","secret":{"secretName":"maestro-agent-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mqtt-certs","mountPath":"/secrets/mqtt-certs"}}]' kubectl wait deploy/maestro-agent -n ${agent_namespace} --for condition=Available=True --timeout=200s # remove the certs