add extra check to verify node to test is still schedulable

FundingCircle · Apr 25, 2024 · 03470a4 · 03470a4
1 parent a9e0a57
commit 03470a4
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 12 deletions.
diff --git a/cmd/storage-check/run_check.go b/cmd/storage-check/run_check.go
@@ -70,7 +70,7 @@ func runStorageCheck() {
 	// Create a storage resource.
 	storageConfig := createStorageConfig(checkStorageName)
 	log.Infoln("Created Storage resource.")
-	log.Infof("It looks like: %v", storageConfig)
+	log.Debugf("It looks like: %v", storageConfig)
 
 	// Apply the storage struct manifest to the cluster.
 	var storageResult StorageResult
@@ -100,7 +100,7 @@ func runStorageCheck() {
 	//initStorageConfig := initializeStorageConfig(checkStorageName+"-init", checkStorageName)
 	initStorageConfig := initializeStorageConfig(checkStorageName+"-init-job", checkStorageName)
 	log.Infoln("Created Storage Initialiazer resource.")
-	log.Infof("It looks like: %v", initStorageConfig)
+	log.Debugf("It looks like: %v", initStorageConfig)
 
 	// Initialize the storage
 	var initStorageResult InitStorageResult
@@ -160,10 +160,10 @@ func runStorageCheck() {
 			log.Infoln("Error on getting nodes..not sure what to do", err)
 		}
 
-		log.Infoln("Nodes are ", nodes)
+		log.Debugln("Nodes are ", nodes)
 		for _, n := range nodes.Items {
 			log.Infoln("Node.name=", n.Name)
-			log.Infoln("Status=", n.Status)
+			log.Debugln("Status=", n.Status)
 
 			node := new(Node)
 			node.name = n.Name
@@ -179,7 +179,7 @@ func runStorageCheck() {
 				}
 				log.Printf("Adding node %s with taints %s to %s scheduled for check", n.Name, formatTaints(n.Spec.Taints), status)
 			} else {
-				log.Infoln("Adding untainted node ", n.Name, " to be scheduled for check")
+				log.Infoln("Adding untainted node", n.Name, "to be scheduled for check")
 				node.schedulable = true
 			}
 			checkNodes[node.name] = node
@@ -221,10 +221,15 @@ func runStorageCheck() {
 
 	for nodeName, node := range checkNodes {
 		if node.schedulable == true {
-			log.Infof("Creating config for %s %+v", nodeName, node)
+			if !verifyStillSchedulable(nodeName) {
+				log.Infof("Node %s is no longer schedulable, skipping check", nodeName)
+				continue
+			}
+
+			log.Infof("Creating config for %s", nodeName)
 			config := checkNodeConfig(checkStorageName+"-check-job", checkStorageName, nodeName)
 			log.Infoln("Created config.")
-			log.Infof("It looks like: %+v", config)
+			log.Debugf("It looks like: %+v", config)
 
 			// Initialize the storage
 			var checkStorageResult CheckStorageResult
@@ -401,3 +406,26 @@ func formatTaints(taints []v1.Taint) string {
 
 	return strings.Join(taintStrings, ",")
 }
+
+func verifyStillSchedulable(nodeName string) bool {
+	log.Infof("Checking if %s is still schedulable", nodeName)
+
+	nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		log.Errorln("Error on getting nodes... not sure what to do", err)
+	}
+
+	schedulable := false
+
+	for _, n := range nodes.Items {
+		if n.Name == nodeName {
+			if len(n.Spec.Taints) > 0 {
+				schedulable = toleratesAllTaints(tolerations, n.Spec.Taints)
+			} else {
+				schedulable = true
+			}
+		}
+	}
+
+	return schedulable
+}
diff --git a/cmd/storage-check/storage.go b/cmd/storage-check/storage.go
@@ -94,7 +94,7 @@ func createStorageConfig(pvcname string) *corev1.PersistentVolumeClaim {
 	// Add the storage spec to the storage.
 	pvc.Spec = pvcSpec
 
-	log.Infoln("PVC ", pvcname, " is", pvc, "namespace environment variables:chris", additionalEnvVars)
+	log.Debugln("PVC", pvcname, "is", pvc, "namespace environment variables:chris", additionalEnvVars)
 	return pvc
 }
 
@@ -154,7 +154,7 @@ func initializeStorageConfig(jobName string, pvcName string) *batchv1.Job {
 	// Add the storage spec to the storage.
 	job.Spec = jobSpec
 
-	log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars)
+	log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars)
 	return job
 }
 
@@ -164,7 +164,7 @@ func checkNodeConfig(jobName string, pvcName string, node string) *batchv1.Job {
 	// Make a Job
 	job := &batchv1.Job{}
 
-	log.Infoln("Creating a job", jobName, " in", checkNamespace, "namespace environment variables:", additionalEnvVars)
+	log.Infoln("Creating a job", jobName, "in", checkNamespace, "namespace environment variables:", additionalEnvVars)
 
 	// Make labels for Job
 	labels := make(map[string]string, 0)
@@ -623,7 +623,7 @@ func deleteStorageCheckAndWait(ctx context.Context) error {
 	// Send a delete on the storage.
 	err := deleteStorageCheckJob(jobName)
 	if err != nil {
-		log.Infoln("Could not delete storage init job :", checkStorageName)
+		log.Infoln("Could not delete storage init job:", checkStorageName)
 	}
 
 	return <-deleteChan
@@ -689,7 +689,7 @@ func deleteStorageInitJobAndWait(ctx context.Context) error {
 	// Send a delete on the storage.
 	err := deleteStorageInitJob(jobName)
 	if err != nil {
-		log.Infoln("Could not delete storage init job :", checkStorageName)
+		log.Infoln("Could not delete storage init job:", checkStorageName)
 	}
 
 	return <-deleteChan