From 7feb471f2038bef2daead8fb6095ff154b0ccbb0 Mon Sep 17 00:00:00 2001
From: Kazunori Kajihiro <kazunori.kajihiro@gmail.com>
Date: Sat, 31 Dec 2016 19:09:43 +0900
Subject: [PATCH] Handle service CF stack failure

---
 app/jobs/deploy_runner_job.rb        |  2 +-
 app/jobs/monitor_deployment_job.rb   |  7 +++++--
 app/models/backend/ecs/v1/adapter.rb | 12 +++++++-----
 app/models/backend/ecs/v2/adapter.rb | 14 ++++++++++++--
 app/models/service.rb                |  4 ++--
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/app/jobs/deploy_runner_job.rb b/app/jobs/deploy_runner_job.rb
index 42cad0dd..3e168c01 100644
--- a/app/jobs/deploy_runner_job.rb
+++ b/app/jobs/deploy_runner_job.rb
@@ -40,6 +40,6 @@ def perform(heritage, without_before_deploy:, description: "")
 
   def other_deploy_in_progress?(heritage)
     return false if heritage.version == 1
-    heritage.services.map { |s| !s.deployment_finished?(nil) }.any?
+    heritage.services.map { |s| s.deployment_status(nil) == :in_progress }.any?
   end
 end
diff --git a/app/jobs/monitor_deployment_job.rb b/app/jobs/monitor_deployment_job.rb
index 6fcd779e..82361452 100644
--- a/app/jobs/monitor_deployment_job.rb
+++ b/app/jobs/monitor_deployment_job.rb
@@ -2,8 +2,11 @@ class MonitorDeploymentJob < ActiveJob::Base
   queue_as :default
 
   def perform(service, count: 0, deployment_id: nil)
-    if service.deployment_finished?(deployment_id)
-      service.heritage.events.create!(level: :good, message: "#{service.name} service deployed")
+    status = service.deployment_status
+    if status == :complete
+      service.heritage.events.create!(level: :good, message: "#{service.name} service has been deployed")
+    elsif status == :failed
+      service.heritage.events.create!(level: :error, message: "#{service.name} service deployment failed")
     elsif count > 200
       # deploys not finished after 1000 seconds are marked as timeout
       service.heritage.events.create!(level: :error, message: "Deploying #{service.name} service has not finished for a while.")
diff --git a/app/models/backend/ecs/v1/adapter.rb b/app/models/backend/ecs/v1/adapter.rb
index 95a47ac7..18ef3dd5 100644
--- a/app/models/backend/ecs/v1/adapter.rb
+++ b/app/models/backend/ecs/v1/adapter.rb
@@ -45,20 +45,22 @@ def endpoint
       end
     end
 
-    def deployment_finished?(deployment_id)
+    def deployment_status(deployment_id)
       deployment = ecs_service.deployment(deployment_id)
       # deployment being nil means the deployment finished and
       # another newer deployment takes in place as PRIMARY
       # http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Deployment.html
-      return true if deployment.nil? || deployment.status == "INACTIVE"
+      return :complete if deployment.nil? || deployment.status == "INACTIVE"
 
       # A deployment is considered as finished when
       # 1) There is only one PRIMARY deployment, and
       # 2) The number of running tasks deployed by PRIMARY deployment
       #    reaches to service's desired task count
-      ecs_service.deployments.count == 1 &&
-        deployment.status == "PRIMARY"   &&
-        deployment.desired_count == deployment.running_count
+      if ecs_service.deployments.count == 1 && deployment.status == "PRIMARY" && deployment.desired_count == deployment.running_count
+        :complete
+      else
+        :in_progress
+      end
     end
 
     def ecs_service
diff --git a/app/models/backend/ecs/v2/adapter.rb b/app/models/backend/ecs/v2/adapter.rb
index d19e5b59..598661df 100644
--- a/app/models/backend/ecs/v2/adapter.rb
+++ b/app/models/backend/ecs/v2/adapter.rb
@@ -31,8 +31,18 @@ def endpoint
       nil
     end
 
-    def deployment_finished?(_)
-      !cf_executor.in_progress?
+    def deployment_status(_)
+      status = cf_executor.stack_status
+      case status
+      when "CREATE_COMPLETE", "UPDATE_COMPLETE"
+        :complete
+      when "CREATE_FAILED", /^ROLLBACK_/, /^UPDATE_ROLLBACK_/
+        # regardless of a specific rollback status,
+        # if rollback has happened barcelona handles it as "failed"
+        :failed
+      when "CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS"
+        :in_progress
+      end
     end
 
     private
diff --git a/app/models/service.rb b/app/models/service.rb
index 5e7f3f98..1ee28cda 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -76,8 +76,8 @@ def https_port_mapping
     port_mappings.find_by(protocol: 'https')
   end
 
-  def deployment_finished?(deployment_id)
-    backend.deployment_finished?(deployment_id)
+  def deployment_status(deployment_id)
+    backend.deployment_status(deployment_id)
   end
 
   private