From eeae03015699f3b0b5fdca1849049975db71c2c3 Mon Sep 17 00:00:00 2001
From: Stuart Douglas <stuart.w.douglas@gmail.com>
Date: Tue, 1 Oct 2024 12:32:46 +1000
Subject: [PATCH] fix: prevent possible busy loop

Failure to remove old deployments should not be a fatal error.

fixes: #2885
---
 backend/runner/runner.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/backend/runner/runner.go b/backend/runner/runner.go
index 335f9c3753..e98aba5aac 100644
--- a/backend/runner/runner.go
+++ b/backend/runner/runner.go
@@ -174,7 +174,8 @@ func manageDeploymentDirectory(logger *log.Logger, config Config) error {
 
 			err := os.RemoveAll(old)
 			if err != nil {
-				return fmt.Errorf("failed to remove old deployment: %w", err)
+				// This is not a fatal error, just log it.
+				logger.Errorf(err, "Failed to remove old deployment: %s", deployment.Name())
 			}
 		}
 	}
@@ -212,8 +213,16 @@ func (s *Service) Call(ctx context.Context, req *connect.Request[ftlv1.CallReque
 	}
 	response, err := deployment.plugin.Client.Call(ctx, req)
 	if err != nil {
+		deploymentLogger := s.getDeploymentLogger(ctx, deployment.key)
+		deploymentLogger.Errorf(err, "Call to deployments %s failed to perform gRPC call", deployment.key)
 		return nil, connect.NewError(connect.CodeOf(err), err)
+	} else if response.Msg.GetError() != nil {
+		// This is a user level error (i.e. something wrong in the users app)
+		// Log it to the deployment logger
+		deploymentLogger := s.getDeploymentLogger(ctx, deployment.key)
+		deploymentLogger.Errorf(fmt.Errorf("%v", response.Msg.GetError().GetMessage()), "Call to deployments %s failed", deployment.key)
 	}
+
 	return connect.NewResponse(response.Msg), nil
 }
 
@@ -413,6 +422,7 @@ func (s *Service) registrationLoop(ctx context.Context, send func(request *ftlv1
 
 func (s *Service) streamLogsLoop(ctx context.Context, send func(request *ftlv1.StreamDeploymentLogsRequest) error) error {
 	delay := time.Millisecond * 500
+	logger := log.FromContext(ctx)
 
 	select {
 	case entry := <-s.deploymentLogQueue:
@@ -430,6 +440,9 @@ func (s *Service) streamLogsLoop(ctx context.Context, send func(request *ftlv1.S
 		if reqStr, ok := entry.Attributes["request"]; ok {
 			request = &reqStr
 		}
+		// We also just output the log normally, so it shows up in the pod logs and gets synced to DD etc.
+		// It's not clear if we should output this here on or on the controller side.
+		logger.Log(entry)
 
 		err := send(&ftlv1.StreamDeploymentLogsRequest{
 			RequestKey:    request,