From 92a76b89404b7ad2d06dd062932a8f551f874deb Mon Sep 17 00:00:00 2001
From: Maciej Moleda <maciej.moleda@fundingcircle.com>
Date: Thu, 25 Apr 2024 22:28:04 +0100
Subject: [PATCH] add extra check to verify node to test is still schedulable

---
 cmd/storage-check/run_check.go | 39 ++++++++++++++++++++++++++++------
 cmd/storage-check/storage.go   |  6 +++---
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/cmd/storage-check/run_check.go b/cmd/storage-check/run_check.go
index ac0653a..71bce7a 100644
--- a/cmd/storage-check/run_check.go
+++ b/cmd/storage-check/run_check.go
@@ -70,7 +70,7 @@ func runStorageCheck() {
 	// Create a storage resource.
 	storageConfig := createStorageConfig(checkStorageName)
 	log.Infoln("Created Storage resource.")
-	log.Infof("It looks like: %v", storageConfig)
+	log.Debugf("It looks like: %v", storageConfig)
 
 	// Apply the storage struct manifest to the cluster.
 	var storageResult StorageResult
@@ -100,7 +100,7 @@ func runStorageCheck() {
 	//initStorageConfig := initializeStorageConfig(checkStorageName+"-init", checkStorageName)
 	initStorageConfig := initializeStorageConfig(checkStorageName+"-init-job", checkStorageName)
 	log.Infoln("Created Storage Initialiazer resource.")
-	log.Infof("It looks like: %v", initStorageConfig)
+	log.Debugf("It looks like: %v", initStorageConfig)
 
 	// Initialize the storage
 	var initStorageResult InitStorageResult
@@ -157,13 +157,13 @@ func runStorageCheck() {
 			// LabelSelector: defaultLabelKey + "=" + defaultLabelValueBase + strconv.Itoa(int(now.Unix())),
 		})
 		if err != nil {
-			log.Infoln("Error on getting nodes..not sure what to do", err)
+			log.Errorln("Error on getting nodes.. not sure what to do", err)
 		}
 
-		log.Infoln("Nodes are ", nodes)
+		log.Debugln("Nodes are ", nodes)
 		for _, n := range nodes.Items {
 			log.Infoln("Node.name=", n.Name)
-			log.Infoln("Status=", n.Status)
+			log.Debugln("Status=", n.Status)
 
 			node := new(Node)
 			node.name = n.Name
@@ -221,10 +221,15 @@ func runStorageCheck() {
 
 	for nodeName, node := range checkNodes {
 		if node.schedulable == true {
-			log.Infof("Creating config for %s %+v", nodeName, node)
+			if !verifyStillSchedulable(nodeName) {
+				log.Infof("Node %s is no longer schedulable, skipping check", nodeName)
+				continue
+			}
+
+			log.Infof("Creating config for %s", nodeName)
 			config := checkNodeConfig(checkStorageName+"-check-job", checkStorageName, nodeName)
 			log.Infoln("Created config.")
-			log.Infof("It looks like: %+v", config)
+			log.Debugf("It looks like: %+v", config)
 
 			// Initialize the storage
 			var checkStorageResult CheckStorageResult
@@ -401,3 +406,23 @@ func formatTaints(taints []v1.Taint) string {
 
 	return strings.Join(taintStrings, ",")
 }
+
+func verifyStillSchedulable(nodeName string) bool {
+	log.Infof("Checking if %s is still schedulable", nodeName)
+
+	nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		log.Errorln("Error on getting nodes... not sure what to do", err)
+	}
+
+	schedulable := false
+
+	for _, n := range nodes.Items {
+		if n.Name == nodeName {
+			schedulable = true
+			break
+		}
+	}
+
+	return schedulable
+}
diff --git a/cmd/storage-check/storage.go b/cmd/storage-check/storage.go
index a83bbc9..43ce8cf 100644
--- a/cmd/storage-check/storage.go
+++ b/cmd/storage-check/storage.go
@@ -94,7 +94,7 @@ func createStorageConfig(pvcname string) *corev1.PersistentVolumeClaim {
 	// Add the storage spec to the storage.
 	pvc.Spec = pvcSpec
 
-	log.Infoln("PVC ", pvcname, " is", pvc, "namespace environment variables:chris", additionalEnvVars)
+	log.Debugln("PVC", pvcname, "is", pvc, "namespace environment variables:chris", additionalEnvVars)
 	return pvc
 }
 
@@ -154,7 +154,7 @@ func initializeStorageConfig(jobName string, pvcName string) *batchv1.Job {
 	// Add the storage spec to the storage.
 	job.Spec = jobSpec
 
-	log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars)
+	log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars)
 	return job
 }
 
@@ -214,7 +214,7 @@ func checkNodeConfig(jobName string, pvcName string, node string) *batchv1.Job {
 	// Add the storage spec to the storage.
 	job.Spec = jobSpec
 
-	log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars)
+	log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars)
 	return job
 }