diff --git a/cmd/storage-check/run_check.go b/cmd/storage-check/run_check.go index ac0653a..6ba5b09 100644 --- a/cmd/storage-check/run_check.go +++ b/cmd/storage-check/run_check.go @@ -70,7 +70,7 @@ func runStorageCheck() { // Create a storage resource. storageConfig := createStorageConfig(checkStorageName) log.Infoln("Created Storage resource.") - log.Infof("It looks like: %v", storageConfig) + log.Debugf("It looks like: %v", storageConfig) // Apply the storage struct manifest to the cluster. var storageResult StorageResult @@ -100,7 +100,7 @@ func runStorageCheck() { //initStorageConfig := initializeStorageConfig(checkStorageName+"-init", checkStorageName) initStorageConfig := initializeStorageConfig(checkStorageName+"-init-job", checkStorageName) log.Infoln("Created Storage Initialiazer resource.") - log.Infof("It looks like: %v", initStorageConfig) + log.Debugf("It looks like: %v", initStorageConfig) // Initialize the storage var initStorageResult InitStorageResult @@ -157,13 +157,13 @@ func runStorageCheck() { // LabelSelector: defaultLabelKey + "=" + defaultLabelValueBase + strconv.Itoa(int(now.Unix())), }) if err != nil { - log.Infoln("Error on getting nodes..not sure what to do", err) + log.Errorln("Error on getting nodes.. not sure what to do", err) } - log.Infoln("Nodes are ", nodes) + log.Debugln("Nodes are ", nodes) for _, n := range nodes.Items { log.Infoln("Node.name=", n.Name) - log.Infoln("Status=", n.Status) + log.Debugln("Status=", n.Status) node := new(Node) node.name = n.Name @@ -179,7 +179,7 @@ func runStorageCheck() { } log.Printf("Adding node %s with taints %s to %s scheduled for check", n.Name, formatTaints(n.Spec.Taints), status) } else { - log.Infoln("Adding untainted node ", n.Name, " to be scheduled for check") + log.Infoln("Adding untainted node", n.Name, "to be scheduled for check") node.schedulable = true } checkNodes[node.name] = node @@ -221,10 +221,15 @@ func runStorageCheck() { for nodeName, node := range checkNodes { if node.schedulable == true { - log.Infof("Creating config for %s %+v", nodeName, node) + if !verifyStillSchedulable(nodeName) { + log.Infof("Node %s is no longer schedulable, skipping check", nodeName) + continue + } + + log.Infof("Creating config for %s", nodeName) config := checkNodeConfig(checkStorageName+"-check-job", checkStorageName, nodeName) log.Infoln("Created config.") - log.Infof("It looks like: %+v", config) + log.Debugf("It looks like: %+v", config) // Initialize the storage var checkStorageResult CheckStorageResult @@ -401,3 +406,23 @@ func formatTaints(taints []v1.Taint) string { return strings.Join(taintStrings, ",") } + +func verifyStillSchedulable(nodeName string) bool { + log.Infof("Checking if %s is still schedulable", nodeName) + + nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + log.Errorln("Error on getting nodes... not sure what to do", err) + } + + schedulable := false + + for _, n := range nodes.Items { + if n.Name == nodeName { + schedulable = true + break + } + } + + return schedulable +} diff --git a/cmd/storage-check/storage.go b/cmd/storage-check/storage.go index a83bbc9..0873ef9 100644 --- a/cmd/storage-check/storage.go +++ b/cmd/storage-check/storage.go @@ -94,7 +94,7 @@ func createStorageConfig(pvcname string) *corev1.PersistentVolumeClaim { // Add the storage spec to the storage. pvc.Spec = pvcSpec - log.Infoln("PVC ", pvcname, " is", pvc, "namespace environment variables:chris", additionalEnvVars) + log.Debugln("PVC", pvcname, "is", pvc, "namespace environment variables:chris", additionalEnvVars) return pvc } @@ -154,7 +154,7 @@ func initializeStorageConfig(jobName string, pvcName string) *batchv1.Job { // Add the storage spec to the storage. job.Spec = jobSpec - log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars) + log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars) return job } @@ -164,7 +164,7 @@ func checkNodeConfig(jobName string, pvcName string, node string) *batchv1.Job { // Make a Job job := &batchv1.Job{} - log.Infoln("Creating a job", jobName, " in", checkNamespace, "namespace environment variables:", additionalEnvVars) + log.Infoln("Creating a job", jobName, "in", checkNamespace, "namespace environment variables:", additionalEnvVars) // Make labels for Job labels := make(map[string]string, 0) @@ -623,7 +623,7 @@ func deleteStorageCheckAndWait(ctx context.Context) error { // Send a delete on the storage. err := deleteStorageCheckJob(jobName) if err != nil { - log.Infoln("Could not delete storage init job :", checkStorageName) + log.Infoln("Could not delete storage init job:", checkStorageName) } return <-deleteChan @@ -689,7 +689,7 @@ func deleteStorageInitJobAndWait(ctx context.Context) error { // Send a delete on the storage. err := deleteStorageInitJob(jobName) if err != nil { - log.Infoln("Could not delete storage init job :", checkStorageName) + log.Infoln("Could not delete storage init job:", checkStorageName) } return <-deleteChan