improve graceful shutdown

cmd/kail/main.go

@@ -91,9 +91,8 @@ func main() {
 }
 
 func watchSignals(ctx context.Context, cancel context.CancelFunc) {
-	// NOTE: ignoring SIGINT to improve responsiveness Ctrl-C responsiveness
 	sigch := make(chan os.Signal, 1)
-	signal.Notify(sigch, syscall.SIGHUP, syscall.SIGQUIT)
+	signal.Notify(sigch, syscall.SIGINT, syscall.SIGHUP)
 	go func() {
 		select {
 		case <-ctx.Done():

controller.go

@@ -116,12 +116,14 @@ func (c *controller) run(initial []*v1.Pod) {
 
 		case <-shutdownch:
 			c.log.Debugf("shutdown requested")
+
+			c.lc.ShutdownInitiated()
 			shutdownch = nil
 			draining = true
 
 			for _, pms := range c.monitors {
 				for _, pm := range pms {
-					go pm.Shutdown()
+					pm.Shutdown()
 				}
 			}
 
@@ -166,7 +168,7 @@ func (c *controller) handlePodEvent(ev pod.Event) {
 	if ev.Type() == kcache.EventTypeDelete {
 		if pms, ok := c.monitors[id]; ok {
 			for _, pm := range pms {
-				go pm.Shutdown()
+				pm.Shutdown()
 			}
 		}
 		return
@@ -193,7 +195,7 @@ func (c *controller) ensureMonitorsForPod(pod *v1.Pod) {
 	if pms, ok := c.monitors[id]; ok {
 		for source, pm := range pms {
 			if !sources[source] {
-				go pm.Shutdown()
+				pm.Shutdown()
 			}
 		}
 	}
@@ -219,15 +221,23 @@ func (c *controller) ensureMonitorsForPod(pod *v1.Pod) {
 
 func (c *controller) createMonitor(source eventSource) monitor {
 	defer c.log.Un(c.log.Trace("createMonitor(%v)", source))
+
 	m := newMonitor(c, &source)
+
 	go func() {
-		<-m.Done()
 		select {
-		case c.monitorch <- source:
+		case <-m.Done():
 		case <-c.lc.Done():
 			c.log.Warnf("done before monitor %v complete", source)
+			return
+		}
+		select {
+		case c.monitorch <- source:
+		case <-c.lc.Done():
+			c.log.Warnf("done before monitor %v unregistered", source)
 		}
 	}()
+
 	return m
 }
 

monitor.go

@@ -55,7 +55,7 @@ type _monitor struct {
 }
 
 func (m *_monitor) Shutdown() {
-	m.lc.Shutdown()
+	m.lc.ShutdownAsync()
 }
 
 func (m *_monitor) Done() <-chan struct{} {
@@ -72,23 +72,19 @@ func (m *_monitor) run() {
 
 	go m.mainloop(ctx, donech)
 
-	go func() {
-		<-m.lc.ShutdownRequest()
-		m.lc.ShutdownInitiated()
-		cancel()
-	}()
+	<-m.lc.ShutdownRequest()
+	m.lc.ShutdownInitiated()
+	cancel()
 
 	<-donech
 }
 
 func (m *_monitor) mainloop(ctx context.Context, donech chan struct{}) {
 	defer m.log.Un(m.log.Trace("mainloop"))
 	defer close(donech)
-	defer func() {
-		go m.lc.Shutdown()
-	}()
+	defer m.lc.ShutdownAsync()
 
-	// todo: backoff
+	// todo: backoff handled by k8 client?
 
 	for ctx.Err() == nil {
 		err := m.readloop(ctx)

vendor/vendor.json

@@ -37,10 +37,10 @@
 			"revisionTime": "2015-10-22T06:55:26Z"
 		},
 		{
-			"checksumSHA1": "DUjGn2/bouPgKEb5xw7ns/rh1y0=",
+			"checksumSHA1": "mZEz6eJo2f7XdUQ2miTxtZkbHTI=",
 			"path": "github.com/boz/go-lifecycle",
-			"revision": "b4a83b8a1dd3240842922bfc5bfba1bcbadb294a",
-			"revisionTime": "2017-05-02T06:09:24Z"
+			"revision": "431131c8f55fdf5cef669d1f527522113abdc5b8",
+			"revisionTime": "2017-08-16T11:26:27Z"
 		},
 		{
 			"checksumSHA1": "usaZ1e18JqJu5wqLcFTmVgUschY=",