CHAOS-181: Try to recover better from stuck nodeFailure disruptions (#…

…453)

* CHAOS-181: If running a nodefailure, check for missing nodes when clenaing

* Log the targetNodeName in the injector

* make deps

LICENSE-3rdparty.csv

@@ -185,8 +185,6 @@ github.com/markbates/pkger,github.com/markbates/pkger,MIT
 github.com/markbates/pkger,github.com/markbates/pkger/here,MIT
 github.com/markbates/pkger,github.com/markbates/pkger/internal/maps,MIT
 github.com/markbates/pkger,github.com/markbates/pkger/pkging,MIT
-github.com/markbates/pkger,github.com/markbates/pkger/pkging/embed,MIT
-github.com/markbates/pkger,github.com/markbates/pkger/pkging/mem,MIT
 github.com/markbates/pkger,github.com/markbates/pkger/pkging/stdos,MIT
 github.com/mattn/go-colorable,github.com/mattn/go-colorable,MIT
 github.com/mattn/go-isatty,github.com/mattn/go-isatty,MIT

api/disruption_kind.go

@@ -30,6 +30,7 @@ type DisruptionArgs struct {
 	DisruptionName                  string
 	DisruptionNamespace             string
 	TargetName                      string
+	TargetNodeName                  string
 	DNSServer                       string
 	KubeDNS                         string
 	InjectorServiceAccountNamespace string
@@ -51,6 +52,7 @@ func AppendArgs(args []string, xargs DisruptionArgs) []string {
 		"--log-context-disruption-name", xargs.DisruptionName,
 		"--log-context-disruption-namespace", xargs.DisruptionNamespace,
 		"--log-context-target-name", xargs.TargetName,
+		"--log-context-target-node-name", xargs.TargetNodeName,
 	)
 
 	// enable dry-run mode

cli/injector/main.go

@@ -57,6 +57,7 @@ var (
 	disruptionNamespace string
 	chaosNamespace      string
 	targetName          string
+	targetNodeName      string
 	onInit              bool
 	handlerPID          uint32
 	configs             []injector.Config
@@ -91,6 +92,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&disruptionName, "log-context-disruption-name", "", "Log value: current disruption name")
 	rootCmd.PersistentFlags().StringVar(&disruptionNamespace, "log-context-disruption-namespace", "", "Log value: current disruption namespace")
 	rootCmd.PersistentFlags().StringVar(&targetName, "log-context-target-name", "", "Log value: current target name")
+	rootCmd.PersistentFlags().StringVar(&targetNodeName, "log-context-target-node-name", "", "Log value: node hosting the current target pod")
 
 	_ = cobra.MarkFlagRequired(rootCmd.PersistentFlags(), "level")
 	cobra.OnInitialize(initLogger)
@@ -130,6 +132,7 @@ func initLogger() {
 		"disruptionName", disruptionName,
 		"disruptionNamespace", disruptionNamespace,
 		"targetName", targetName,
+		"targetNodeName", targetNodeName,
 	)
 }
 
@@ -245,6 +248,7 @@ func initConfig() {
 			Level:           chaostypes.DisruptionLevel(level),
 			TargetContainer: ctns[i],
 			TargetPodIP:     targetPodIP,
+			TargetNodeName:  targetNodeName,
 			Cgroup:          cgroupMgr,
 			Netns:           netnsMgr,
 			K8sClient:       clientset,

controllers/disruption_controller.go

@@ -998,6 +998,7 @@ func (r *DisruptionReconciler) generateChaosPods(instance *chaosv1beta1.Disrupti
 			Kind:                            kind,
 			TargetContainerIDs:              targetContainerIDs,
 			TargetName:                      targetName,
+			TargetNodeName:                  targetNodeName,
 			TargetPodIP:                     targetPodIP,
 			DryRun:                          instance.Spec.DryRun,
 			DisruptionName:                  instance.Name,

injector/injector.go

@@ -32,6 +32,7 @@ type Config struct {
 	Level           types.DisruptionLevel
 	TargetContainer container.Container
 	TargetPodIP     string
+	TargetNodeName  string
 	Cgroup          cgroup.Manager
 	Netns           netns.Manager
 	K8sClient       kubernetes.Interface

targetselector/running_target_selector.go

@@ -153,6 +153,14 @@ func (r runningTargetSelector) TargetIsHealthy(target string, c client.Client, i
 		if p.Status.Phase != corev1.PodRunning {
 			return errors.New("pod is not Running")
 		}
+
+		// check if pod's node is gone in the case that this was a node failure
+		if instance.Spec.NodeFailure != nil {
+			var n corev1.Node
+			if err := c.Get(context.Background(), client.ObjectKey{Name: p.Spec.NodeName}, &n); err != nil {
+				return err
+			}
+		}
 	case chaostypes.DisruptionLevelNode:
 		var n corev1.Node
 		if err := c.Get(context.Background(), client.ObjectKey{Name: target}, &n); err != nil {