Merge pull request #172 from calvinmclean/feature/notify-down

Notify if Garden is not healthy

garden-app/integration_tests/main_test.go

@@ -247,7 +247,7 @@ func GardenTests(t *testing.T) {
 
 		// The health status timing can be inconsistent, so it should be retried
 		retries := 1
-		for g.Health.Status != "UP" && retries <= 5 {
+		for g.Health.Status != pkg.HealthStatusUp && retries <= 5 {
 			time.Sleep(time.Duration(retries) * time.Second)
 
 			status, err := makeRequest(http.MethodGet, "/gardens/"+gardenID, http.NoBody, &g)
@@ -257,7 +257,7 @@ func GardenTests(t *testing.T) {
 			retries++
 		}
 
-		assert.Equal(t, "UP", g.Health.Status)
+		assert.Equal(t, pkg.HealthStatusUp, g.Health.Status)
 		assert.Equal(t, 50.0, g.TemperatureHumidityData.TemperatureCelsius)
 		assert.Equal(t, 50.0, g.TemperatureHumidityData.HumidityPercentage)
 	})

garden-app/pkg/garden.go

@@ -12,6 +12,14 @@ import (
 	"github.com/calvinmclean/babyapi"
 )
 
+type HealthStatus string
+
+const (
+	HealthStatusDown    HealthStatus = "DOWN"
+	HealthStatusUp      HealthStatus = "UP"
+	HealthStatusUnknown HealthStatus = "N/A"
+)
+
 // Garden is the representation of a single garden-controller device
 type Garden struct {
 	Name                      string         `json:"name" yaml:"name,omitempty"`
@@ -35,9 +43,9 @@ func (g *Garden) String() string {
 
 // GardenHealth holds information about the Garden controller's health status
 type GardenHealth struct {
-	Status      string     `json:"status,omitempty"`
-	Details     string     `json:"details,omitempty"`
-	LastContact *time.Time `json:"last_contact,omitempty"`
+	Status      HealthStatus `json:"status,omitempty"`
+	Details     string       `json:"details,omitempty"`
+	LastContact *time.Time   `json:"last_contact,omitempty"`
 }
 
 // Health returns a GardenHealth struct after querying InfluxDB for the Garden controller's last contact time
@@ -52,7 +60,7 @@ func (g *Garden) Health(ctx context.Context, influxdbClient influxdb.Client) *Ga
 
 	if lastContact.IsZero() {
 		return &GardenHealth{
-			Status:  "DOWN",
+			Status:  HealthStatusDown,
 			Details: "no last contact time available",
 		}
 	}
@@ -61,9 +69,9 @@ func (g *Garden) Health(ctx context.Context, influxdbClient influxdb.Client) *Ga
 	between := time.Since(lastContact)
 	up := between < 5*time.Minute
 
-	status := "UP"
+	status := HealthStatusUp
 	if !up {
-		status = "DOWN"
+		status = HealthStatusDown
 	}
 
 	return &GardenHealth{

garden-app/pkg/garden_test.go

@@ -17,31 +17,31 @@ func TestHealth(t *testing.T) {
 		name            string
 		lastContactTime time.Time
 		err             error
-		expectedStatus  string
+		expectedStatus  HealthStatus
 	}{
 		{
 			"GardenIsUp",
 			time.Now(),
 			nil,
-			"UP",
+			HealthStatusUp,
 		},
 		{
 			"GardenIsDown",
 			time.Now().Add(-5 * time.Minute),
 			nil,
-			"DOWN",
+			HealthStatusDown,
 		},
 		{
 			"InfluxDBError",
 			time.Time{},
 			errors.New("influxdb error"),
-			"N/A",
+			HealthStatusUnknown,
 		},
 		{
 			"ZeroTime",
 			time.Time{},
 			nil,
-			"DOWN",
+			HealthStatusDown,
 		},
 	}
 

garden-app/pkg/notifications/fake/fake.go

@@ -22,9 +22,8 @@ type Message struct {
 }
 
 var (
-	// lastMessage allows checking the last message that was sent
-	lastMessage    = Message{}
-	lastMessageMtx = sync.Mutex{}
+	messages    = []Message{}
+	messagesMtx = sync.Mutex{}
 )
 
 func NewClient(options map[string]interface{}) (*Client, error) {
@@ -46,21 +45,33 @@ func (c *Client) SendMessage(title, message string) error {
 	if c.SendMessageError != "" {
 		return errors.New(c.SendMessageError)
 	}
-	lastMessageMtx.Lock()
-	lastMessage = Message{title, message}
-	lastMessageMtx.Unlock()
+	messagesMtx.Lock()
+	messages = append(messages, Message{title, message})
+	messagesMtx.Unlock()
 	return nil
 }
 
 func LastMessage() Message {
-	lastMessageMtx.Lock()
-	result := lastMessage
-	lastMessageMtx.Unlock()
+	messagesMtx.Lock()
+	defer messagesMtx.Unlock()
+
+	if len(messages) == 0 {
+		return Message{}
+	}
+	result := messages[len(messages)-1]
+	return result
+}
+
+func Messages() []Message {
+	messagesMtx.Lock()
+	result := make([]Message, len(messages))
+	copy(result, messages)
+	messagesMtx.Unlock()
 	return result
 }
 
 func ResetLastMessage() {
-	lastMessageMtx.Lock()
-	lastMessage = Message{}
-	lastMessageMtx.Unlock()
+	messagesMtx.Lock()
+	messages = []Message{}
+	messagesMtx.Unlock()
 }

garden-app/worker/notifications.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"time"
 
 	"github.com/calvinmclean/automated-garden/garden-app/pkg"
 )
@@ -14,10 +15,23 @@ func (w *Worker) sendLightActionNotification(g *pkg.Garden, state pkg.LightState
 	}
 
 	title := fmt.Sprintf("%s: Light %s", g.Name, state.String())
-	w.sendNotificationWithClientID(g.LightSchedule.GetNotificationClientID(), title, "Successfully executed LightAction", logger)
+	w.sendNotification(g.LightSchedule.GetNotificationClientID(), title, "Successfully executed LightAction", logger)
 }
 
-func (w *Worker) sendNotificationWithClientID(clientID, title, msg string, logger *slog.Logger) {
+func (w *Worker) sendDownNotification(g *pkg.Garden, clientID, actionName string) {
+	health := g.Health(context.Background(), w.influxdbClient)
+	if health.Status != pkg.HealthStatusUp {
+		w.sendNotification(
+			clientID,
+			fmt.Sprintf("%s: %s", g.Name, health.Status),
+			fmt.Sprintf(`Attempting to execute %s Action, but last contact was %s.
+Details: %s`, actionName, health.LastContact.Format(time.DateTime), health.Details),
+			w.logger,
+		)
+	}
+}
+
+func (w *Worker) sendNotification(clientID, title, msg string, logger *slog.Logger) {
 	ncLogger := logger.With("notification_client_id", clientID)
 
 	notificationClient, err := w.storageClient.NotificationClientConfigs.Get(context.Background(), clientID)

garden-app/worker/scheduler.go

@@ -76,7 +76,7 @@ func (w *Worker) ScheduleWaterAction(waterSchedule *pkg.WaterSchedule) error {
 						jobLogger.Error("error executing scheduled water action", "error", err, "zone_id", zg.Zone.ID.String())
 						schedulerErrors.WithLabelValues(zoneLabels(zg.Zone)...).Inc()
 						if ws.GetNotificationClientID() != "" {
-							go w.sendNotificationWithClientID(
+							go w.sendNotification(
 								ws.GetNotificationClientID(),
 								fmt.Sprintf("%s: Water Action Error", ws.Name),
 								err.Error(),
@@ -91,7 +91,7 @@ func (w *Worker) ScheduleWaterAction(waterSchedule *pkg.WaterSchedule) error {
 				jobLogger.Error("error executing schedule WaterAction", "error", err)
 				schedulerErrors.WithLabelValues(waterScheduleLabels(waterSchedule)...).Inc()
 				if waterSchedule.GetNotificationClientID() != "" {
-					w.sendNotificationWithClientID(
+					w.sendNotification(
 						waterSchedule.GetNotificationClientID(),
 						fmt.Sprintf("%s: Water Action Error", waterSchedule.Name),
 						err.Error(),
@@ -480,13 +480,18 @@ func waterScheduleLabels(ws *pkg.WaterSchedule) []string {
 func (w *Worker) executeLightActionInScheduledJob(g *pkg.Garden, input *action.LightAction, actionLogger *slog.Logger) {
 	actionLogger = actionLogger.With("state", input.State.String())
 	actionLogger.Info("executing LightAction")
+
+	if g.LightSchedule.GetNotificationClientID() != "" {
+		w.sendDownNotification(g, g.LightSchedule.GetNotificationClientID(), "Light")
+	}
+
 	err := w.ExecuteLightAction(g, input)
 	if err != nil {
 		actionLogger.Error("error executing scheduled LightAction", "error", err)
 		schedulerErrors.WithLabelValues(gardenLabels(g)...).Inc()
 
 		if g.LightSchedule.GetNotificationClientID() != "" {
-			w.sendNotificationWithClientID(g.LightSchedule.GetNotificationClientID(), fmt.Sprintf("%s: Light Action Error", g.Name), err.Error(), actionLogger)
+			w.sendNotification(g.LightSchedule.GetNotificationClientID(), fmt.Sprintf("%s: Light Action Error", g.Name), err.Error(), actionLogger)
 		}
 		return
 	}