[artemiscloud#574] Use k8s jobs to do scaledown

gaohoward · Apr 24, 2023 · 245988f · 245988f
1 parent cd0330a
commit 245988f
Show file tree

Hide file tree

Showing 9 changed files with 440 additions and 230 deletions.
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -41,6 +41,14 @@ rules:
   - deployments/finalizers
   verbs:
   - update
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - list
 - apiGroups:
   - broker.amq.io
   resources:

diff --git a/controllers/activemqartemisscaledown_controller.go b/controllers/activemqartemisscaledown_controller.go
@@ -51,6 +51,7 @@ type ActiveMQArtemisScaledownReconciler struct {
 //+kubebuilder:rbac:groups=broker.amq.io,namespace=activemq-artemis-operator,resources=activemqartemisscaledowns,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=broker.amq.io,namespace=activemq-artemis-operator,resources=activemqartemisscaledowns/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=broker.amq.io,namespace=activemq-artemis-operator,resources=activemqartemisscaledowns/finalizers,verbs=update
+//+kubebuilder:rbac:groups=batch,namespace=activemq-artemis-operator,resources=jobs,verbs=create;list;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -99,6 +100,14 @@ func (r *ActiveMQArtemisScaledownReconciler) Reconcile(ctx context.Context, requ
 	return ctrl.Result{}, nil
 }
 
+//for testing
+func (r *ActiveMQArtemisScaledownReconciler) GetDrainController(namespace string) *draincontroller.Controller {
+	if inst, ok := controllers[namespace]; ok {
+		return inst
+	}
+	return nil
+}
+
 func (r *ActiveMQArtemisScaledownReconciler) getDrainController(localOnly bool, namespace string, kubeClient *kubernetes.Clientset, instance *brokerv1beta1.ActiveMQArtemisScaledown) (kubeinformers.SharedInformerFactory, *draincontroller.Controller, bool) {
 	var kubeInformerFactory kubeinformers.SharedInformerFactory
 	var controllerInstance *draincontroller.Controller

diff --git a/controllers/activemqartemisscaledown_controller_test.go b/controllers/activemqartemisscaledown_controller_test.go
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -42,6 +42,7 @@ import (
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -91,14 +92,15 @@ const (
 )
 
 var (
-	resCount   int64
-	specCount  int64
-	currentDir string
-	k8sClient  client.Client
-	restConfig *rest.Config
-	testEnv    *envtest.Environment
-	ctx        context.Context
-	cancel     context.CancelFunc
+	resCount     int64
+	specCount    int64
+	currentDir   string
+	k8sClient    client.Client
+	k8sClientSet *kubernetes.Clientset
+	restConfig   *rest.Config
+	testEnv      *envtest.Environment
+	ctx          context.Context
+	cancel       context.CancelFunc
 
 	// the cluster url
 	clusterUrl *url.URL
@@ -112,6 +114,7 @@ var (
 
 	brokerReconciler   *ActiveMQArtemisReconciler
 	securityReconciler *ActiveMQArtemisSecurityReconciler
+	scaleDownRconciler *ActiveMQArtemisScaledownReconciler
 
 	oprRes = []string{
 		"../deploy/service_account.yaml",
@@ -373,7 +376,7 @@ func createControllerManager(disableMetrics bool, watchNamespace string) {
 	err = addressReconciler.SetupWithManager(k8Manager, managerCtx)
 	Expect(err).ToNot(HaveOccurred(), "failed to create address reconciler")
 
-	scaleDownRconciler := &ActiveMQArtemisScaledownReconciler{
+	scaleDownRconciler = &ActiveMQArtemisScaledownReconciler{
 		Client: k8Manager.GetClient(),
 		Scheme: k8Manager.GetScheme(),
 		Config: k8Manager.GetConfig(),
@@ -580,6 +583,9 @@ func setUpK8sClient() {
 	k8sClient, err = client.New(restConfig, client.Options{Scheme: scheme.Scheme})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())
+
+	k8sClientSet, err = kubernetes.NewForConfig(restConfig)
+	Expect(err).To(BeNil())
 }
 
 var _ = BeforeSuite(func() {

diff --git a/deploy/cluster_role.yaml b/deploy/cluster_role.yaml
@@ -38,6 +38,14 @@ rules:
   - deployments/finalizers
   verbs:
   - update
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - list
 - apiGroups:
   - broker.amq.io
   resources:

diff --git a/deploy/role.yaml b/deploy/role.yaml
@@ -38,6 +38,14 @@ rules:
   - deployments/finalizers
   verbs:
   - update
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - list
 - apiGroups:
   - broker.amq.io
   resources: