Handle service CF stack failure

app/jobs/deploy_runner_job.rb

@@ -40,6 +40,6 @@ def perform(heritage, without_before_deploy:, description: "")
 
   def other_deploy_in_progress?(heritage)
     return false if heritage.version == 1
-    heritage.services.map { |s| !s.deployment_finished?(nil) }.any?
+    heritage.services.map { |s| s.deployment_status(nil) == :in_progress }.any?
   end
 end

app/jobs/monitor_deployment_job.rb

@@ -2,8 +2,11 @@ class MonitorDeploymentJob < ActiveJob::Base
   queue_as :default
 
   def perform(service, count: 0, deployment_id: nil)
-    if service.deployment_finished?(deployment_id)
-      service.heritage.events.create!(level: :good, message: "#{service.name} service deployed")
+    status = service.deployment_status
+    if status == :complete
+      service.heritage.events.create!(level: :good, message: "#{service.name} service has been deployed")
+    elsif status == :failed
+      service.heritage.events.create!(level: :error, message: "#{service.name} service deployment failed")
     elsif count > 200
       # deploys not finished after 1000 seconds are marked as timeout
       service.heritage.events.create!(level: :error, message: "Deploying #{service.name} service has not finished for a while.")

app/models/backend/ecs/v1/adapter.rb

@@ -45,20 +45,22 @@ def endpoint
       end
     end
 
-    def deployment_finished?(deployment_id)
+    def deployment_status(deployment_id)
       deployment = ecs_service.deployment(deployment_id)
       # deployment being nil means the deployment finished and
       # another newer deployment takes in place as PRIMARY
       # http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Deployment.html
-      return true if deployment.nil? || deployment.status == "INACTIVE"
+      return :complete if deployment.nil? || deployment.status == "INACTIVE"
 
       # A deployment is considered as finished when
       # 1) There is only one PRIMARY deployment, and
       # 2) The number of running tasks deployed by PRIMARY deployment
       #    reaches to service's desired task count
-      ecs_service.deployments.count == 1 &&
-        deployment.status == "PRIMARY"   &&
-        deployment.desired_count == deployment.running_count
+      if ecs_service.deployments.count == 1 && deployment.status == "PRIMARY" && deployment.desired_count == deployment.running_count
+        :complete
+      else
+        :in_progress
+      end
     end
 
     def ecs_service

app/models/backend/ecs/v2/adapter.rb

@@ -31,8 +31,18 @@ def endpoint
       nil
     end
 
-    def deployment_finished?(_)
-      !cf_executor.in_progress?
+    def deployment_status(_)
+      status = cf_executor.stack_status
+      case status
+      when "CREATE_COMPLETE", "UPDATE_COMPLETE"
+        :complete
+      when "CREATE_FAILED", /^ROLLBACK_/, /^UPDATE_ROLLBACK_/
+        # regardless of a specific rollback status,
+        # if rollback has happened barcelona handles it as "failed"
+        :failed
+      when "CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS"
+        :in_progress
+      end
     end
 
     private

app/models/service.rb

@@ -76,8 +76,8 @@ def https_port_mapping
     port_mappings.find_by(protocol: 'https')
   end
 
-  def deployment_finished?(deployment_id)
-    backend.deployment_finished?(deployment_id)
+  def deployment_status(deployment_id)
+    backend.deployment_status(deployment_id)
   end
 
   private